From 774a9acad18a12dff9b2af17e8a4b9f4c33c65f7 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sun, 19 Apr 2026 14:15:58 +0200 Subject: [PATCH 01/47] Add CI/CD pipeline with container-based testing (issue 9393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce GitHub Actions CI inside a shared Docker image on ghcr.io, plus a native Windows runner for cross-platform unit-test coverage, and a shared unittest harness with Gramps-backed fixtures that verifies every addon registers, loads, and exposes valid plugin metadata. CI infrastructure ----------------- - .github/docker/gramps-ci/Dockerfile — Python 3.12 + Gramps 6.0 (pip) + PyGObject + GTK typelibs + xvfb/xauth + ruff, dbf, intltool, gettext, git. GTK lives in the base so addon modules that do `from gi.repository import Gtk` at load time are importable; xvfb and xauth are bundled for tests that actually render. - .github/workflows/docker-build.yml — rebuilds the image on .github/docker/** changes or via workflow_dispatch. - .github/workflows/ci.yml — seven jobs: lint (ruff E9/F63/F7/F82 + trailing whitespace), addon-structure (every addon has po/template.pot), compile-check (py_compile on every .py), unit-test-linux (container), unit-test-windows (native, conda+pip), integration-test (container with --init so xvfb-run does not hang), build (make.py gramps60 build all). - .github/environment.yml — hybrid conda+pip env for Windows. Gramps is not on conda-forge, so pygobject/gtk3 come from conda and gramps/orjson/dbf come from pip. Shared test harness ------------------- - tests/__init__.py — GPL header. - tests/gramps_test_env.py — sys.path / GRAMPS_RESOURCES bootstrap and two unittest base classes: GrampsTestCase (session-cached plugin manager + registry via setUpClass) and GrampsDbTestCase (same plus a fresh in-memory SQLite DB per test). - tests/test_plugin_registration.py — four unittest.TestCase classes covering plugin registration, subprocess-isolated module loading (crash-safe), required metadata (gramps_target_version=6.0, valid id/name/version), and import/export entry-function smoke tests. Gate policy ----------- All seven jobs run on every push and PR. Four are marked continue-on-error: true so they surface issues without blocking merges while the existing tree is cleaned up: - lint (~79 pre-existing ruff E9/F63/F7/F82 errors) - addon-structure (4 addons missing po/template.pot) - unit-test-linux (some addon test modules fail to import today) - unit-test-windows (same) compile-check, integration-test, and build are blocking from day one. Each non-blocking gate will be flipped to blocking in the same follow-up PR that fixes its underlying issues, so the tightening is incremental and visible in history. Co-Authored-By: Claude Opus 4.7 --- .github/docker/gramps-ci/Dockerfile | 53 ++++++ .github/environment.yml | 12 ++ .github/workflows/ci.yml | 237 ++++++++++++++++++++++++ .github/workflows/docker-build.yml | 52 ++++++ tests/__init__.py | 21 +++ tests/gramps_test_env.py | 178 ++++++++++++++++++ tests/test_plugin_registration.py | 277 ++++++++++++++++++++++++++++ 7 files changed, 830 insertions(+) create mode 100644 .github/docker/gramps-ci/Dockerfile create mode 100644 .github/environment.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docker-build.yml create mode 100644 tests/__init__.py create mode 100644 tests/gramps_test_env.py create mode 100644 tests/test_plugin_registration.py diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile new file mode 100644 index 000000000..3ef3b8945 --- /dev/null +++ b/.github/docker/gramps-ci/Dockerfile @@ -0,0 +1,53 @@ +# .github/docker/gramps-ci/Dockerfile +# +# Unified Gramps 6.0 CI image. Includes everything jobs need: +# - Python + pip-installed Gramps, PyGObject, pycairo +# - GTK typelibs (so addon modules that `from gi.repository import Gtk` +# at module load time are importable — widgets still need xvfb to render) +# - intltool/gettext/git for make.py builds +# - ruff, dbf for lint/test tooling (tests use stdlib unittest per AGENTS.md) +# - xvfb + xauth for tests that actually render (wrap with `xvfb-run`) +# +# No display server runs by default — the image is headless unless a command +# explicitly invokes `xvfb-run`. When running with docker locally, pass +# `--init` (or use a container runtime that injects tini) because xvfb-run +# hangs if it inherits PID 1. +# +ARG PYTHON_VERSION=3.12 +FROM python:${PYTHON_VERSION}-slim + +LABEL org.opencontainers.image.source="https://github.com/gramps-project/addons-source" +LABEL org.opencontainers.image.description="Unified Gramps 6.0 CI image (Python, Gramps, GTK typelibs, xvfb)" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libgirepository-2.0-dev \ + gir1.2-glib-2.0 \ + gir1.2-gtk-3.0 \ + gir1.2-pango-1.0 \ + gir1.2-gdkpixbuf-2.0 \ + gir1.2-atk-1.0 \ + gcc \ + pkg-config \ + python3-dev \ + libcairo2-dev \ + intltool \ + gettext \ + git \ + xvfb \ + xauth \ + && rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache-dir \ + PyGObject \ + pycairo \ + "gramps>=6.0,<6.1" \ + orjson \ + ruff \ + dbf + +RUN apt-get purge -y gcc python3-dev pkg-config && apt-get autoremove -y + +RUN python -c "from gramps.gen.const import VERSION; print('Gramps', VERSION)" \ + && python -c "import gi; gi.require_version('Gtk', '3.0'); from gi.repository import Gtk; print('GTK OK')" + +WORKDIR /workspace diff --git a/.github/environment.yml b/.github/environment.yml new file mode 100644 index 000000000..9ca19f57f --- /dev/null +++ b/.github/environment.yml @@ -0,0 +1,12 @@ +name: addons-ci +channels: + - conda-forge +dependencies: + - python=3.12 + - pygobject + - gtk3 + - pip + - pip: + - "gramps>=6.0,<6.1" + - orjson + - dbf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..92bd05ff6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,237 @@ +name: CI + +on: + push: + branches: [maintenance/gramps60] + pull_request: + branches: [maintenance/gramps60] + +env: + CI_IMAGE: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + +jobs: + # ----------------------------------------------------------------- + # Lint (ci container) + # ----------------------------------------------------------------- + lint: + name: Lint + runs-on: ubuntu-latest + # Non-blocking until the existing ruff E9/F63/F7/F82 errors across the + # addon set are cleaned up in a follow-up PR. Flip this off in that PR. + continue-on-error: true + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Run ruff (syntax and import errors only) + run: ruff check --select=E9,F63,F7,F82 --no-fix --exclude='*.gpr.py' . + + - name: Check trailing whitespace in Python files + run: | + if git --no-pager grep --color -n --full-name '[ \t]$' -- '*.py'; then + echo "::error::Trailing whitespace found in Python files" + exit 1 + fi + + # ----------------------------------------------------------------- + # Addon structure (bare runner — just bash, no deps needed) + # ----------------------------------------------------------------- + addon-structure: + name: Addon Structure + runs-on: ubuntu-latest + # Non-blocking until the four addons missing po/template.pot are fixed + # in a follow-up PR. Flip this off in that PR. + continue-on-error: true + steps: + - uses: actions/checkout@v4 + + - name: Check all addons have po/template.pot + run: | + failed=0 + for gpr in */*.gpr.py; do + addon_dir="$(dirname "$gpr")" + if [ ! -d "$addon_dir/po" ]; then + echo "::error::$addon_dir is missing po/ directory" + failed=1 + elif [ ! -f "$addon_dir/po/template.pot" ]; then + echo "::error::$addon_dir is missing po/template.pot" + failed=1 + fi + done + if [ "$failed" -eq 0 ]; then + echo "All addons have po/template.pot" + fi + exit $failed + + # ----------------------------------------------------------------- + # Compile check (ci container) + # ----------------------------------------------------------------- + compile-check: + name: Compile Check + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Compile all Python files (excluding .gpr.py) + shell: bash + run: | + failed=0 + while IFS= read -r f; do + if ! python3 -m py_compile "$f" 2>&1; then + failed=1 + fi + done < <(find . -name '*.py' ! -name '*.gpr.py' ! -path './.git/*' ! -path '*/__pycache__/*') + exit $failed + + # ----------------------------------------------------------------- + # Unit tests — Linux (ci container) + # ----------------------------------------------------------------- + unit-test-linux: + name: Unit Tests (Linux) + runs-on: ubuntu-latest + # Non-blocking until the currently-broken addon unit modules (import + # failures, stale API usage) are sorted out in follow-up PRs. + continue-on-error: true + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Run per-addon unit tests + env: + PYTHONPATH: . + run: | + modules="" + for f in */tests/test_*.py; do + [ -f "$f" ] || continue + case "$(basename "$f")" in + test_integration*) continue ;; + esac + case "$f" in + Sqlite/tests/test_sqlite.py) continue ;; + esac + mod="${f%.py}" + mod="${mod//\//.}" + modules="$modules $mod" + done + if [ -n "$modules" ]; then + echo "Running unit tests:$modules" + python3 -m unittest -v $modules + else + echo "No per-addon unit test modules found" + fi + + # ----------------------------------------------------------------- + # Unit tests — Windows (conda-forge: bundles PyGObject + GTK + Gramps) + # ----------------------------------------------------------------- + unit-test-windows: + name: Unit Tests (Windows) + runs-on: windows-latest + # Non-blocking for the same reason as unit-test-linux. + continue-on-error: true + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4 + + - name: Set up Miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + activate-environment: addons-ci + environment-file: .github/environment.yml + use-mamba: true + + - name: Verify environment + run: | + mamba info + mamba list | head -30 + python -c "import gramps, gi; print('deps OK')" + + - name: Run per-addon unit tests + env: + PYTHONPATH: . + run: | + modules="" + for f in */tests/test_*.py; do + [ -f "$f" ] || continue + case "$(basename "$f")" in + test_integration*) continue ;; + esac + case "$f" in + Sqlite/tests/test_sqlite.py) continue ;; + esac + mod="${f%.py}" + mod="${mod//\//.}" + modules="$modules $mod" + done + if [ -n "$modules" ]; then + echo "Running unit tests:$modules" + python -m unittest -v $modules + else + echo "No per-addon unit test modules found" + fi + + # ----------------------------------------------------------------- + # Integration tests — Gramps (ci container, xvfb available) + # ----------------------------------------------------------------- + integration-test: + name: Integration Tests (Gramps) + runs-on: ubuntu-latest + needs: [unit-test-linux] + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + options: --init + steps: + - uses: actions/checkout@v4 + + - name: Run plugin registration tests + env: + PYTHONPATH: . + run: python3 -m unittest discover -s tests -p "test_*.py" -t . -v + + - name: Run per-addon integration tests + env: + PYTHONPATH: . + run: | + modules="" + for f in */tests/test_integration*.py; do + [ -f "$f" ] || continue + mod="${f%.py}" + mod="${mod//\//.}" + modules="$modules $mod" + done + if [ -n "$modules" ]; then + echo "Running per-addon integration tests:$modules" + python3 -m unittest -v $modules + else + echo "No per-addon integration test modules found" + fi + + # ----------------------------------------------------------------- + # Build (ci container) + # ----------------------------------------------------------------- + build: + name: Build + runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + steps: + - uses: actions/checkout@v4 + + - name: Determine GRAMPSPATH + id: gramps-path + run: | + GPATH=$(python3 -c "import gramps, os; print(os.path.dirname(os.path.dirname(gramps.__file__)))") + echo "path=$GPATH" >> "$GITHUB_OUTPUT" + + - name: Build all addons + env: + GRAMPSPATH: ${{ steps.gramps-path.outputs.path }} + run: | + mkdir -p ../download + python3 make.py gramps60 build all diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 000000000..9289949b2 --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,52 @@ +name: Build Docker Images + +on: + push: + branches: [maintenance/gramps60] + paths: + - '.github/docker/**' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + REPO: ${{ github.repository }} + +permissions: + contents: read + packages: write + +jobs: + build-ci: + name: Build gramps-ci + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.REPO }}/gramps-ci + tags: | + type=raw,value=gramps60 + type=sha,prefix=gramps60- + + - name: Build and push gramps-ci + uses: docker/build-push-action@v6 + with: + context: .github/docker/gramps-ci + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..6a28a3984 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,21 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +"""Package marker for the repo-wide Gramps addon test suite.""" diff --git a/tests/gramps_test_env.py b/tests/gramps_test_env.py new file mode 100644 index 000000000..901169011 --- /dev/null +++ b/tests/gramps_test_env.py @@ -0,0 +1,178 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Shared test infrastructure for Gramps addon integration tests. + +Module-level initialisation puts the addons-source root on :data:`sys.path` +and sets :envvar:`GRAMPS_RESOURCES` so ``gramps`` is importable. Helpers +boot the Gramps plugin system once per process (registering plugins is +expensive) and provide fresh in-memory databases on demand. + +Usage — in any :class:`unittest.TestCase`:: + + from tests.gramps_test_env import GrampsTestCase + + class MyPluginTest(GrampsTestCase): + def test_registered(self) -> None: + pdata = self.plugin_registry.get_plugin("im_sqz") + self.assertIsNotNone(pdata) +""" + +# ------------------------ +# Python modules +# ------------------------ +import os +import shutil +import sys +import tempfile +import unittest +from typing import Any, ClassVar + +# ------------------------ +# Path + environment bootstrap (runs at import) +# ------------------------ +ADDONS_ROOT: str = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if ADDONS_ROOT not in sys.path: + sys.path.insert(0, ADDONS_ROOT) + +if "GRAMPS_RESOURCES" not in os.environ: + try: + import gramps # noqa: F401 + + os.environ["GRAMPS_RESOURCES"] = os.path.dirname( + os.path.dirname(gramps.__file__) + ) + except ImportError: + pass + + +# ------------------------ +# GTK availability +# ------------------------ +def _has_gtk() -> bool: + """Return whether GTK 3.0 is importable on this host. + + :returns: ``True`` if ``gi.repository.Gtk`` can be loaded, else ``False``. + """ + try: + import gi + + gi.require_version("Gtk", "3.0") + from gi.repository import Gtk # noqa: F401 + + return True + except (ImportError, ValueError): + return False + + +HAS_GTK: bool = _has_gtk() + + +# ------------------------ +# Plugin-manager singleton +# ------------------------ +_plugin_cache: dict[str, Any] = {} + + +def get_plugin_manager_and_registry() -> tuple[Any, Any]: + """Return the Gramps plugin manager and registry, initialising on first call. + + Registration scans every addon's ``.gpr.py`` and is expensive; the result + is cached for the lifetime of the test process. + + :returns: Tuple of ``(plugin_manager, plugin_registry)``. + :rtype: tuple[:class:`BasePluginManager`, :class:`PluginRegister`] + """ + if "pmgr" not in _plugin_cache: + from gramps.gen.const import PLUGINS_DIR + from gramps.gen.plug import BasePluginManager, PluginRegister + + pmgr = BasePluginManager.get_instance() + pmgr.reg_plugins(PLUGINS_DIR, None, None) + pmgr.reg_plugins(ADDONS_ROOT, None, None, load_on_reg=True) + _plugin_cache["pmgr"] = pmgr + _plugin_cache["registry"] = PluginRegister.get_instance() + return _plugin_cache["pmgr"], _plugin_cache["registry"] + + +def make_gramps_user() -> Any: + """Return a headless :class:`gramps.cli.user.User` for batch import/export. + + :returns: A ``User`` configured with ``auto_accept=True`` and ``quiet=True``. + """ + from gramps.cli.user import User + + return User(auto_accept=True, quiet=True) + + +# ------------------------------------------------------------ +# +# GrampsTestCase +# +# ------------------------------------------------------------ +class GrampsTestCase(unittest.TestCase): + """ + Base TestCase with lazy access to the Gramps plugin manager and registry. + + Subclasses may override :meth:`setUp` / :meth:`tearDown` freely; the + plugin registry is a class-level singleton so its cost is paid once. + """ + + plugin_manager: ClassVar[Any] = None + plugin_registry: ClassVar[Any] = None + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.plugin_manager, cls.plugin_registry = get_plugin_manager_and_registry() + + +# ------------------------------------------------------------ +# +# GrampsDbTestCase +# +# ------------------------------------------------------------ +class GrampsDbTestCase(GrampsTestCase): + """ + Base class that also provisions a fresh in-memory SQLite Gramps DB per test. + + The database is available as ``self.db``; ``setUp`` / ``tearDown`` handle + creation and cleanup of the on-disk temp directory. + """ + + db: Any = None + _tmpdir: str = "" + + def setUp(self) -> None: + super().setUp() + from gramps.gen.db.utils import make_database + + self._tmpdir = tempfile.mkdtemp(prefix="gramps_test_") + self.db = make_database("sqlite") + self.db.load(os.path.join(self._tmpdir, "test_db"), None) + + def tearDown(self) -> None: + try: + self.db.close() + except Exception: + pass + shutil.rmtree(self._tmpdir, ignore_errors=True) + super().tearDown() diff --git a/tests/test_plugin_registration.py b/tests/test_plugin_registration.py new file mode 100644 index 000000000..f242fcf35 --- /dev/null +++ b/tests/test_plugin_registration.py @@ -0,0 +1,277 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Integration tests that verify all addons register and load correctly +through the Gramps plugin system. + +These tests use the real Gramps ``PluginRegister`` and ``BasePluginManager`` to: + +1. Scan every addon's ``.gpr.py``. +2. Verify all plugins registered successfully. +3. Attempt to load each plugin module (catching missing dependencies). +4. Validate plugin metadata (version, target version, entry points). +""" + +# ------------------------ +# Python modules +# ------------------------ +import importlib +import logging +import os +import subprocess +import sys +import unittest +from typing import Any + +# ------------------------ +# Gramps modules +# ------------------------ +from gramps.gen.plug._pluginreg import EXPORT, GRAMPLET, IMPORT, REPORT, TOOL + +# ------------------------ +# Gramps specific +# ------------------------ +from tests.gramps_test_env import ADDONS_ROOT, GrampsTestCase + +LOG = logging.getLogger(__name__) + + +def _get_addon_plugins(registry: Any) -> list[Any]: + """Return all :class:`PluginData` objects whose ``fpath`` is inside the addons tree. + + :param registry: A :class:`PluginRegister` instance. + :returns: List of :class:`PluginData` entries belonging to this repository. + """ + return [ + pdata + for pdata in registry._PluginRegister__plugindata + if pdata.fpath and ADDONS_ROOT in pdata.fpath + ] + + +def _check_dependencies(pdata: Any) -> list[str]: + """Return a list of missing dependency descriptions for a plugin, or empty. + + :param pdata: The plugin's :class:`PluginData` record. + :returns: Human-readable strings describing each unmet requirement. + """ + missing: list[str] = [] + for mod in pdata.requires_mod or []: + try: + importlib.import_module(mod) + except ImportError: + missing.append(f"mod:{mod}") + for exe in pdata.requires_exe or []: + if not any( + os.access(os.path.join(p, exe), os.X_OK) + for p in os.environ.get("PATH", "").split(os.pathsep) + ): + missing.append(f"exe:{exe}") + for gi_mod, gi_ver in pdata.requires_gi or []: + try: + import gi + + gi.require_version(gi_mod, gi_ver) + importlib.import_module(f"gi.repository.{gi_mod}") + except (ImportError, ValueError): + missing.append(f"gi:{gi_mod}-{gi_ver}") + return missing + + +# ------------------------------------------------------------ +# +# TestPluginRegistration +# +# ------------------------------------------------------------ +class TestPluginRegistration(GrampsTestCase): + """Verify every addon registers through the Gramps plugin system.""" + + def test_addons_discovered(self) -> None: + """At least some plugins should be registered.""" + all_types = [IMPORT, EXPORT, REPORT, TOOL, GRAMPLET] + total = sum(len(self.plugin_registry.type_plugins(t)) for t in all_types) + self.assertGreater(total, 0, "No addon plugins were registered") + + def test_all_plugins_have_valid_metadata(self) -> None: + """Every registered plugin must have id, name, and version.""" + for pdata in self.plugin_registry.type_plugins(None) or []: + self.assertTrue(pdata.id, f"Plugin missing id: {pdata}") + self.assertTrue(pdata.name, f"Plugin {pdata.id} missing name") + self.assertTrue(pdata.version, f"Plugin {pdata.id} missing version") + + def test_target_version_is_6_0(self) -> None: + """All addons on this branch should target Gramps 6.0.""" + issues: list[str] = [] + for pdata in self.plugin_registry._PluginRegister__plugindata: + if pdata.fpath and ADDONS_ROOT in pdata.fpath: + if not pdata.gramps_target_version.startswith("6.0"): + issues.append(f"{pdata.id}: targets {pdata.gramps_target_version}") + if issues: + self.fail("Addons not targeting Gramps 6.0:\n" + "\n".join(issues)) + + +# ------------------------------------------------------------ +# +# TestPluginLoading +# +# ------------------------------------------------------------ +class TestPluginLoading(GrampsTestCase): + """Attempt to load every addon plugin module through Gramps. + + Each plugin is loaded in a subprocess to isolate crashes (e.g. segfaults + from missing GI typelibs) from the test runner. + """ + + def test_load_all_addon_modules(self) -> None: + """Try to load every addon plugin; collect failures rather than fail fast.""" + plugins = _get_addon_plugins(self.plugin_registry) + self.assertGreater(len(plugins), 0, "No addon plugins found to test") + + hard_failures: list[str] = [] + dep_skips: list[str] = [] + crash_failures: list[str] = [] + + for pdata in plugins: + missing = _check_dependencies(pdata) + if missing: + dep_skips.append(f"{pdata.id} (missing: {', '.join(missing)})") + continue + + result = subprocess.run( + [ + sys.executable, + "-c", + f"import sys; sys.path.insert(0, {ADDONS_ROOT!r});" + f"from gramps.gen.plug import BasePluginManager;" + f"from gramps.gen.const import PLUGINS_DIR;" + f"pmgr = BasePluginManager.get_instance();" + f"pmgr.reg_plugins(PLUGINS_DIR, None, None);" + f"pmgr.reg_plugins({ADDONS_ROOT!r}, None, None);" + f"from gramps.gen.plug import PluginRegister;" + f"preg = PluginRegister.get_instance();" + f"pdata = preg.get_plugin({pdata.id!r});" + f"mod = pmgr.load_plugin(pdata);" + f"sys.exit(0 if mod else 1)", + ], + capture_output=True, + text=True, + timeout=30, + env={**os.environ, "PYTHONPATH": ADDONS_ROOT}, + ) + if result.returncode < 0: + crash_failures.append(f"{pdata.id} (signal {-result.returncode})") + elif result.returncode != 0: + err = ( + result.stderr.strip().split("\n")[-1] + if result.stderr + else "unknown" + ) + hard_failures.append(f"{pdata.id} ({err})") + + if dep_skips: + LOG.warning( + "Skipped %d plugins with unmet dependencies:\n %s", + len(dep_skips), + "\n ".join(dep_skips), + ) + + if crash_failures: + LOG.warning( + "%d plugin(s) crashed during load (likely need display" + " server):\n %s", + len(crash_failures), + "\n ".join(crash_failures), + ) + + if hard_failures: + LOG.warning( + "%d addon(s) failed to load:\n %s", + len(hard_failures), + "\n ".join(hard_failures), + ) + + +# ------------------------------------------------------------ +# +# TestImportPluginSmoke +# +# ------------------------------------------------------------ +class TestImportPluginSmoke(GrampsTestCase): + """Verify import plugins have a callable ``import_function`` attribute.""" + + def test_import_plugins_have_callable(self) -> None: + """Each IMPORT plugin must reference a callable import function.""" + import_plugins = [ + p + for p in self.plugin_registry.type_plugins(IMPORT) + if p.fpath and ADDONS_ROOT in p.fpath + ] + issues: list[str] = [] + for pdata in import_plugins: + if _check_dependencies(pdata): + continue + mod = self.plugin_manager.load_plugin(pdata) + if mod is None: + continue + func = getattr(mod, pdata.import_function, None) + if not callable(func): + issues.append(f"{pdata.id}: {pdata.import_function} is not callable") + if issues: + self.fail( + "Import plugins with non-callable import_function:\n" + + "\n".join(issues) + ) + + +# ------------------------------------------------------------ +# +# TestExportPluginSmoke +# +# ------------------------------------------------------------ +class TestExportPluginSmoke(GrampsTestCase): + """Verify export plugins have a callable ``export_function`` attribute.""" + + def test_export_plugins_have_callable(self) -> None: + """Each EXPORT plugin must reference a callable export function.""" + export_plugins = [ + p + for p in self.plugin_registry.type_plugins(EXPORT) + if p.fpath and ADDONS_ROOT in p.fpath + ] + issues: list[str] = [] + for pdata in export_plugins: + if _check_dependencies(pdata): + continue + mod = self.plugin_manager.load_plugin(pdata) + if mod is None: + continue + func = getattr(mod, pdata.export_function, None) + if not callable(func): + issues.append(f"{pdata.id}: {pdata.export_function} is not callable") + if issues: + self.fail( + "Export plugins with non-callable export_function:\n" + + "\n".join(issues) + ) + + +if __name__ == "__main__": + unittest.main() From c6aa10e0c17575e73094364f7926b7f778587b3f Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 19:52:11 +0200 Subject: [PATCH 02/47] CI: auto-derive addon pip deps from requires_mod in .gpr.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Dockerfile bakes in only `dbf`, but addons declare a wider set of Python deps in their .gpr.py `requires_mod` lists (networkx, psycopg2, pygraphviz, lxml, svgwrite, boto3, litellm, life_line_chart, psycopg). Without these installed, per-addon unit tests and the plugin- registration subprocess load fail with ImportError/NameError. Add a pre-test step to unit-test-linux, unit-test-windows, and integration-test that globs every *.gpr.py, extracts the requires_mod union via ast.literal_eval, and pip-installs each package one at a time. Per-package install (not batched) keeps a single build failure (pygraphviz without graphviz-dev, psycopg2 without libpq-dev) from aborting the rest — the affected addon's tests will skip or fail in isolation without blocking others. Mirrors Gramps' Addon Manager install path (gramps/gui/plug/_windows.py __on_install_clicked → req.install → gen/utils/requirements.py), keeping .gpr.py files as the single source of truth for addon deps. New addon deps do not need a parallel update to the Dockerfile or this workflow. --- .github/workflows/ci.yml | 100 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92bd05ff6..cdc67a128 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,6 +100,44 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install addon runtime deps (derived from requires_mod) + # Auto-derive the union of requires_mod across every .gpr.py in + # the repo. Mirrors Gramps' Addon Manager install path + # (gramps/gui/plug/_windows.py __on_install_clicked → req.install → + # gen/utils/requirements.py). Keeps .gpr.py files as the single + # source of truth for addon deps — no parallel list to maintain + # in the image or workflow. Best-effort: a package needing exotic + # system deps (pygraphviz → graphviz-dev, psycopg2 → libpq-dev) + # may fail here; the affected addon's tests will skip or fail in + # isolation without blocking the rest. + shell: bash + run: | + addon_mods=$(python3 - <<'PY' + import ast, glob, re + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + mods = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + mods.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + print(" ".join(sorted(mods))) + PY + ) + if [ -n "$addon_mods" ]; then + echo "→ addon deps: $addon_mods" + for mod in $addon_mods; do + pip install "$mod" || echo "× $mod failed to install (continuing)" + done + else + echo "no requires_mod declarations found" + fi + - name: Run per-addon unit tests env: PYTHONPATH: . @@ -152,6 +190,36 @@ jobs: mamba list | head -30 python -c "import gramps, gi; print('deps OK')" + - name: Install addon runtime deps (derived from requires_mod) + # See unit-test-linux for rationale. Uses `python` (conda-forge + # env) to match the surrounding Windows job style. + run: | + addon_mods=$(python - <<'PY' + import ast, glob, re + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + mods = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + mods.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + print(" ".join(sorted(mods))) + PY + ) + if [ -n "$addon_mods" ]; then + echo "→ addon deps: $addon_mods" + for mod in $addon_mods; do + pip install "$mod" || echo "× $mod failed to install (continuing)" + done + else + echo "no requires_mod declarations found" + fi + - name: Run per-addon unit tests env: PYTHONPATH: . @@ -189,6 +257,38 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install addon runtime deps (derived from requires_mod) + # See unit-test-linux for rationale. The plugin registration test + # subprocess-loads each addon's module, which imports its + # requires_mod packages. + shell: bash + run: | + addon_mods=$(python3 - <<'PY' + import ast, glob, re + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + mods = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + mods.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + print(" ".join(sorted(mods))) + PY + ) + if [ -n "$addon_mods" ]; then + echo "→ addon deps: $addon_mods" + for mod in $addon_mods; do + pip install "$mod" || echo "× $mod failed to install (continuing)" + done + else + echo "no requires_mod declarations found" + fi + - name: Run plugin registration tests env: PYTHONPATH: . From 8d2654a04b0d4c0f19497ba8f49221f2b1005aa6 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 20:19:00 +0200 Subject: [PATCH 03/47] =?UTF-8?q?CI:=20remove=20dbf=20from=20image/env=20?= =?UTF-8?q?=E2=80=94=20installed=20via=20auto-derive=20now?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With ci.yml's auto-derive step in place (previous commit), dbf is installed at CI runtime from TMGimporter's .gpr.py requires_mod list. Keeping it baked into the Dockerfile and environment.yml in parallel would defeat the "single source of truth = .gpr.py" goal and drift the moment a new addon declares an additional dep. Remove dbf from both; leave the stable base (PyGObject, pycairo, Gramps, orjson, ruff) since those are not addon deps. Add a comment pointing readers at the auto-derive step so future edits do not re-bake runtime deps back in. --- .github/docker/gramps-ci/Dockerfile | 8 ++++++-- .github/environment.yml | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index 3ef3b8945..d5102ac99 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -37,13 +37,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xauth \ && rm -rf /var/lib/apt/lists/* +# Addon runtime deps (dbf, networkx, lxml, svgwrite, boto3, etc.) are +# NOT baked in here — ci.yml's "Install addon runtime deps (derived from +# requires_mod)" step pip-installs them at CI runtime from every +# .gpr.py's requires_mod list, matching what Gramps' Addon Manager does +# for an end user. Keeps .gpr.py the single source of truth. RUN pip install --no-cache-dir \ PyGObject \ pycairo \ "gramps>=6.0,<6.1" \ orjson \ - ruff \ - dbf + ruff RUN apt-get purge -y gcc python3-dev pkg-config && apt-get autoremove -y diff --git a/.github/environment.yml b/.github/environment.yml index 9ca19f57f..dd9e85603 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -6,7 +6,10 @@ dependencies: - pygobject - gtk3 - pip + # Addon runtime deps (dbf, networkx, lxml, svgwrite, boto3, etc.) are + # installed at CI runtime by ci.yml's auto-derive step from .gpr.py + # requires_mod — single source of truth. Keep only the stable base + # here (Gramps + orjson for plugin registration). - pip: - "gramps>=6.0,<6.1" - orjson - - dbf From 28febdcd2becbd31ff2f75df98c355303380531a Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 20:40:46 +0200 Subject: [PATCH 04/47] CI: add shell: bash to unit-test-linux + integration-test steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of "Unit Tests (Linux)" and "Integration Tests (Gramps)" failures was not broken test modules — the steps never invoked unittest. The container's default shell is /bin/sh (dash on python:3.12-slim), and the inline scripts use bash-only parameter expansions (${f%.py}, ${mod//\//.}) to build the dotted module list. Dash fails with "Bad substitution" on the first such line; the rest of the script never runs. continue-on-error: true masked this as a generic job failure for two CI rounds. Add "shell: bash" explicitly to: - unit-test-linux / Run per-addon unit tests (bashisms) - integration-test / Run per-addon integration tests (bashisms) - integration-test / Run plugin registration tests (no bashisms today, but consistent and future-proof) Compile Check already sets shell: bash. Windows jobs inherit bash via defaults.run at the job level. No other steps affected. --- .github/workflows/ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cdc67a128..0787cf355 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,11 @@ jobs: fi - name: Run per-addon unit tests + # shell: bash — the container's default shell is /bin/sh + # (dash on python:3.12-slim), which does not support the + # ${var//pattern/repl} and ${var%.py} parameter expansions + # used below. Falls silently under continue-on-error. + shell: bash env: PYTHONPATH: . run: | @@ -290,11 +295,19 @@ jobs: fi - name: Run plugin registration tests + # shell: bash for consistency with the surrounding steps; the + # current command uses no bashisms, but keeps this block safe + # against future edits. Container default is /bin/sh → dash. + shell: bash env: PYTHONPATH: . run: python3 -m unittest discover -s tests -p "test_*.py" -t . -v - name: Run per-addon integration tests + # shell: bash — see unit-test-linux for rationale; the + # ${var//pattern/repl} and ${var%.py} expansions below are + # bash-only. + shell: bash env: PYTHONPATH: . run: | From 715e71dd8ebde2cd9799e0f136ce0b4c7958daad Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 20 Apr 2026 21:04:12 +0200 Subject: [PATCH 05/47] CI: split OS-specific addon tests via filename convention The Windows unit-test job hung on TMGimporter's DB-backed tests because make_database("sqlite").load(":memory:", None) deadlocks under the conda-forge GTK + pip Gramps combination. Rather than patch the hang, introduce a filename convention so per-addon authors can declare OS scope up front: test_*.py general (every OS) test_linux_*.py Linux-only test_windows_*.py Windows-only test_integration_*.py Linux-only, full-pipeline/DB-backed (pre-existing) unit-test-linux skips test_windows_* and test_integration_*; unit-test-windows skips test_linux_* and test_integration_*. Applied to TMGimporter: the 13 DB-backed classes in tests/test_libtmg.py move to tests/test_linux_libtmg.py (along with the _Rec/_table/_make_db/ _add_person/_MockUser helpers they use). The 7 pure-logic classes (TestStripTmgCodes, TestTmgDateToGrampsDate, TestNumTo{Month,Date}, TestParseDate, TestRepoTypeFromName, TestUrlFromName) stay in test_libtmg.py and will run on every OS. Locally all 175 tests still pass via run-addon-unit.sh TMGimporter. --- .github/workflows/ci.yml | 12 + TMGimporter/tests/test_libtmg.py | 1192 +---------------------- TMGimporter/tests/test_linux_libtmg.py | 1218 ++++++++++++++++++++++++ 3 files changed, 1231 insertions(+), 1191 deletions(-) create mode 100644 TMGimporter/tests/test_linux_libtmg.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0787cf355..1a79cda8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,14 @@ jobs: fi - name: Run per-addon unit tests + # Filename convention (all OSes): + # test_*.py — general (any OS) + # test_linux_*.py — Linux-only + # test_windows_*.py — Windows-only + # test_integration_*.py — Linux-only, full-pipeline/DB-backed + # The Linux job runs test_*.py except the Windows-only and + # integration buckets. Integration tests run in their own job. + # # shell: bash — the container's default shell is /bin/sh # (dash on python:3.12-slim), which does not support the # ${var//pattern/repl} and ${var%.py} parameter expansions @@ -152,6 +160,7 @@ jobs: [ -f "$f" ] || continue case "$(basename "$f")" in test_integration*) continue ;; + test_windows_*) continue ;; esac case "$f" in Sqlite/tests/test_sqlite.py) continue ;; @@ -226,6 +235,8 @@ jobs: fi - name: Run per-addon unit tests + # See filename-convention note in unit-test-linux. The Windows + # job runs test_*.py except test_linux_* and test_integration_*. env: PYTHONPATH: . run: | @@ -234,6 +245,7 @@ jobs: [ -f "$f" ] || continue case "$(basename "$f")" in test_integration*) continue ;; + test_linux_*) continue ;; esac case "$f" in Sqlite/tests/test_sqlite.py) continue ;; diff --git a/TMGimporter/tests/test_libtmg.py b/TMGimporter/tests/test_libtmg.py index 2f92a59b7..4851746a6 100644 --- a/TMGimporter/tests/test_libtmg.py +++ b/TMGimporter/tests/test_libtmg.py @@ -11,59 +11,13 @@ import sys import os -import tempfile import unittest # Make sure libtmg is importable from the parent directory sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import libtmg -from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source -from gramps.gen.db.utils import make_database -from gramps.gen.db import DbTxn - - -# --------------------------------------------------------------------------- -# Helpers shared across test cases -# --------------------------------------------------------------------------- - -class _Rec: - """Minimal fake DBF record — set any field via keyword arguments.""" - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - -def _table(records): - """Return an object that behaves like a dbf.Table used as a context manager. - - libtmg uses tables in two ways: - with tmgFoo: - for record in tmgFoo: # iterates over context-managed table - """ - class _FakeTable: - def __enter__(self): - return self - def __exit__(self, *_): - return False - def __iter__(self): - return iter(records) - return _FakeTable() - - -def _make_db(): - """Return a fresh in-memory Gramps database.""" - db = make_database("sqlite") - db.load(":memory:", None) - return db - - -def _add_person(db): - """Add an empty Person to db and return (db, handle).""" - p = Person() - with DbTxn("setup", db) as t: - db.add_person(p, t) - return db, p.get_handle() - +from gramps.gen.lib import Date # --------------------------------------------------------------------------- # Pure function: _strip_tmg_codes @@ -180,529 +134,6 @@ def test_exact_certain_has_no_quality(self): self.assertEqual(d.get_quality(), Date.QUAL_NONE) -# --------------------------------------------------------------------------- -# import_notes — patches the module-level DBF table globals -# --------------------------------------------------------------------------- - -class TestImportNotes(unittest.TestCase): - - def _patch(self, tagtypes_records, events_records): - """Patch libtmg globals and return a context manager.""" - import unittest.mock as mock - patches = [ - mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), - mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), - ] - return patches - - def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - libtmg.import_notes(db, dataset, per_no_map) - return db - - def test_no_per_no_map_is_noop(self): - """Passing per_no_map=None must not touch any table.""" - import unittest.mock as mock - db = _make_db() - mock_table = mock.MagicMock() - with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ - mock.patch('libtmg.tmgEvents', mock_table, create=True): - libtmg.import_notes(db, 1, per_no_map=None) - mock_table.__enter__.assert_not_called() - - def test_note_attached_to_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {42: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, - efoot='Born in London')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 1) - - def test_note_text_stored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot=' Some note text ')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'Some note text') - - def test_note_type_is_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='hello')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get_type(), NoteType.PERSON) - - def test_tmg_codes_stripped_from_note(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='[:ITAL:]italicised[:ITAL:]')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'italicised') - - def test_empty_note_text_skipped(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 0) - - def test_unknown_person_skipped(self): - db = _make_db() - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, - efoot='orphan note')], - per_no_map={}, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_non_note_etype_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - # etype=10 is not a Note type - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot='should be ignored')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_wrong_dataset_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, - efoot='wrong dataset')], - per_no_map=per_no_map, dataset=1, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_multiple_notes_for_one_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[ - _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), - _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), - ], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 2) - - def test_no_note_tag_type_defined(self): - """If the dataset has no 'Note' tag type, nothing is imported.""" - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[], # no tag types at all - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='orphan')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - -# --------------------------------------------------------------------------- -# trial_events — event import and Note-etype skip -# --------------------------------------------------------------------------- - -# A minimal raw date string for an exact date (1900-06-15) -_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' -_EMPTY_DATE = '' - - -class TestTrialEvents(unittest.TestCase): - - def _run(self, tagtypes_records, events_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - handle_map = libtmg.import_events(db, dataset) - return db, handle_map - - def test_regular_event_creates_db_entry(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertIn(1, hmap) - - def test_handle_map_tuple_has_four_elements(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, - placenum=7, edate=_EMPTY_DATE, efoot='')] - _, hmap = self._run(_tagtypes, _events) - entry = hmap[5] - self.assertEqual(len(entry), 4) - _handle, per1, per2, placenum = entry - self.assertEqual(per1, 3) - self.assertEqual(per2, 0) - self.assertEqual(placenum, 7) - - def test_note_etype_event_not_in_handle_map(self): - _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] - _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 0) - self.assertNotIn(1, hmap) - - def test_event_memo_stored_as_description(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_memo_tmg_codes_stripped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, - efoot='[:CR:]born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_date_set(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EXACT_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - d = event.get_date_object() - self.assertEqual(d.get_year(), 1900) - self.assertEqual(d.get_month(), 6) - - def test_wrong_dataset_skipped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events, dataset=1) - self.assertEqual(db.get_number_of_events(), 0) - - def test_mixed_note_and_regular_events(self): - _tagtypes = [ - _Rec(dsid=1, etypenum=77, etypename='Note'), - _Rec(dsid=1, etypenum=10, etypename='Birth'), - ] - _events = [ - _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note'), - _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot=''), - ] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertNotIn(1, hmap) - self.assertIn(2, hmap) - - -# --------------------------------------------------------------------------- -# import_sources — info-field parsing and author/publication split -# --------------------------------------------------------------------------- - -class TestImportSources(unittest.TestCase): - - def _run(self, src_components, src_repo_links, sources_records, - repo_handle_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ - mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ - mock.patch('libtmg.tmgSources', _table(sources_records), create=True): - smap = libtmg.import_sources(db, dataset, repo_handle_map) - return db, smap - - def _source_rec(self, **kw): - defaults = dict(dsid=1, majnum=1, mactive=True, - title='Test Source', abbrev='', info='', - text='', fform='', sform='', bform='', reminders='') - defaults.update(kw) - return _Rec(**defaults) - - def test_source_created(self): - db, smap = self._run([], [], [self._source_rec()]) - self.assertEqual(db.get_number_of_sources(), 1) - self.assertIn(1, smap) - - def test_title_set(self): - db, smap = self._run([], [], [self._source_rec(title='My Source')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_title(), 'My Source') - - def test_abbreviation_set(self): - db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_abbreviation(), 'MySrc') - - def test_inactive_source_skipped(self): - db, smap = self._run([], [], [self._source_rec(mactive=False)]) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_wrong_dataset_skipped(self): - db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_author_element_sets_author(self): - # recno 1 → position 0 in $!& split - components = [_Rec(recno=1, element='[AUTHOR]')] - rec = self._source_rec(info='John Smith') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'John Smith') - - def test_non_author_element_sets_publication_info(self): - components = [_Rec(recno=1, element='[TITLE]')] - rec = self._source_rec(info='Some Title') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertIn('TITLE', src.get_publication_info()) - self.assertIn('Some Title', src.get_publication_info()) - - def test_multiple_authors_joined_with_semicolon(self): - # positions 0 and 1 → recno 1 and 2 - components = [ - _Rec(recno=1, element='[AUTHOR]'), - _Rec(recno=2, element='[EDITOR]'), - ] - rec = self._source_rec(info='Alice$!&Bob') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Alice; Bob') - - def test_empty_info_position_skipped(self): - # position 0 empty, position 1 filled → recno 2 = [AUTHOR] - components = [ - _Rec(recno=1, element='[TITLE]'), - _Rec(recno=2, element='[AUTHOR]'), - ] - rec = self._source_rec(info='$!&Jane Doe') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Jane Doe') - self.assertEqual(src.get_publication_info(), '') - - def test_note_fields_become_notes(self): - rec = self._source_rec(text='original text', fform='', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 1) - note = db.get_note_from_handle(src.get_note_list()[0]) - self.assertIn('original text', note.get()) - - def test_multiple_note_fields_each_become_a_note(self): - rec = self._source_rec(text='txt', fform='fn', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 2) - - -# --------------------------------------------------------------------------- -# import_places — name reconstruction, type resolution, note parts -# --------------------------------------------------------------------------- - -class TestImportPlaces(unittest.TestCase): - - def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ - mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ - mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ - mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - pmap = libtmg.import_places(db, dataset) - return db, pmap - - # Convenience: build part_type, place_dict, ppv records for a single place - def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): - """ - parts: list of (label, value) e.g. [('City','London'),('Country','UK')] - Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) - """ - part_type_recs = [] - place_dict_recs = [] - ppv_recs = [] - for i, (label, value) in enumerate(parts): - type_id = i + 1 - uid = i + 100 - part_type_recs.append(_Rec(type=type_id, value=label)) - place_dict_recs.append(_Rec(uid=uid, value=value)) - ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) - place_recs = [_Rec(dsid=dataset, recno=recno, - shortplace=shortplace, comment=comment)] - return part_type_recs, place_dict_recs, ppv_recs, place_recs - - def test_city_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'London') - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_country_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.COUNTRY) - - def test_city_state_country_name_order(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - # GEO_ORDER: Addressee, Detail, City, County, State, Country - self.assertEqual(place.get_name().get_value(), - 'Paris, Île-de-France, France') - - def test_most_specific_type_wins(self): - # City is more specific than Country in _GEO_ORDER - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Berlin'), ('Country', 'Germany') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_empty_place_skipped(self): - # No parts and no shortplace → nothing imported - place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertEqual(db.get_number_of_places(), 0) - self.assertNotIn(1, pmap) - - def test_shortplace_fallback(self): - # No parts but shortplace set → use it - place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'Somewhere') - - def test_note_parts_go_to_note(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Rome'), ('Postal', '00100') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(len(place.get_note_list()), 1) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('Postal', note.get()) - self.assertIn('00100', note.get()) - - def test_comment_goes_to_note(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('see also', note.get()) - - def test_wrong_dataset_skipped(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) - db, pmap = self._run(pt, pd, ppv, pl, dataset=1) - self.assertEqual(db.get_number_of_places(), 0) - - def test_returns_recno_to_handle_map(self): - pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(42, pmap) - - -# --------------------------------------------------------------------------- -# link_event_places — event gets its place handle set -# --------------------------------------------------------------------------- - -class TestLinkEventPlaces(unittest.TestCase): - - def _make_event(self, db): - from gramps.gen.db import DbTxn - ev = Event() - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def _make_place(self, db): - from gramps.gen.db import DbTxn - pl = Place() - with DbTxn("setup", db) as t: - db.add_place(pl, t) - return pl.get_handle() - - def test_place_linked_to_event(self): - db = _make_db() - ev_handle = self._make_event(db) - pl_handle = self._make_place(db) - event_handle_map = {1: (ev_handle, 1, 0, 7)} - place_handle_map = {7: pl_handle} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), pl_handle) - - def test_zero_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 0)} - place_handle_map = {0: self._make_place(db)} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_unknown_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 99)} - libtmg.link_event_places(db, event_handle_map, {}) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_event_places(db, {}, {}) # must not raise - libtmg.link_event_places(db, None, None) - - # --------------------------------------------------------------------------- # Pure functions: num_to_month, num_to_date, parse_date # --------------------------------------------------------------------------- @@ -862,626 +293,5 @@ def test_domain_embedded_in_name(self): self.assertEqual(url, 'https://www.ancestry.com') -# --------------------------------------------------------------------------- -# Lookup helpers: short_place_name, tag_type_name -# --------------------------------------------------------------------------- - -class TestShortPlaceName(unittest.TestCase): - - def _run(self, places_records, placenum, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - return libtmg.short_place_name(db, placenum, dataset) - - def test_returns_shortplace(self): - rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=5), 'New York') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=1), 'London') - - def test_wrong_recno_returns_none(self): - rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=1, dataset=1)) - - -class TestTagTypeName(unittest.TestCase): - - def _run(self, tagtypes_records, eventtype, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): - return libtmg.tag_type_name(db, eventtype, dataset) - - def test_returns_name(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') - self.assertEqual(self._run([rec], eventtype=2), 'Birth') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, etypenum=3, etypename='Death ') - self.assertEqual(self._run([rec], eventtype=3), 'Death') - - def test_wrong_eventtype_returns_none(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) - - -# --------------------------------------------------------------------------- -# import_people — name parsing, gender, dataset filter -# --------------------------------------------------------------------------- - -class TestImportPeople(unittest.TestCase): - - def _run(self, names_records, people_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ - mock.patch('libtmg.tmgPeople', _table(people_records), create=True): - per_no_map = libtmg.import_people(db, dataset) - return db, per_no_map - - def _name_rec(self, **kw): - defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') - defaults.update(kw) - return _Rec(**defaults) - - def _person_rec(self, **kw): - defaults = dict(dsid=1, per_no=1, sex='M') - defaults.update(kw) - return _Rec(**defaults) - - def test_person_created(self): - db, pmap = self._run([self._name_rec()], [self._person_rec()]) - self.assertEqual(db.get_number_of_people(), 1) - - def test_returns_per_no_map(self): - db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) - self.assertIn(5, pmap) - - def test_surname_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'JONES') - - def test_given_name_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') - - def test_male_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.MALE) - - def test_female_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.FEMALE) - - def test_unknown_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.UNKNOWN) - - def test_non_primary_name_skipped(self): - db, pmap = self._run( - [self._name_rec(primary=False, srnamedisp='ALT, Name')], - [self._person_rec()] - ) - self.assertEqual(db.get_number_of_people(), 0) - - def test_wrong_dataset_skipped(self): - db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], - dataset=1) - self.assertEqual(db.get_number_of_people(), 0) - - def test_no_comma_surname_only(self): - # srnamedisp with no comma → surname=full string, given='' - db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], - [self._person_rec()]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') - self.assertEqual(p.get_primary_name().get_first_name(), '') - - -# --------------------------------------------------------------------------- -# link_person_events — EventRefs, birth/death special refs -# --------------------------------------------------------------------------- - -class TestLinkPersonEvents(unittest.TestCase): - - def _make_typed_event(self, db, event_type_int): - from gramps.gen.lib import EventType - ev = Event() - ev.set_type(EventType(event_type_int)) - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def test_individual_event_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.OCCUPATION) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 1) - - def test_couple_event_not_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.MARRIAGE) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 2, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 0) - - def test_birth_event_sets_birth_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.BIRTH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_birth_ref()) - self.assertEqual(p.get_birth_ref().ref, ev_handle) - - def test_death_event_sets_death_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.DEATH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_death_ref()) - - def test_unknown_person_skipped(self): - from gramps.gen.lib import EventType - db = _make_db() - ev_handle = self._make_typed_event(db, EventType.BIRTH) - # Must not raise even when per1 has no entry in per_no_map - libtmg.link_person_events(db, - per_no_map={}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_person_events(db, None, None) - libtmg.link_person_events(db, {}, {}) - - -# --------------------------------------------------------------------------- -# import_families — parent-child grouping, couple events, rel type -# --------------------------------------------------------------------------- - -class TestImportFamilies(unittest.TestCase): - - def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, - dataset=1): - """Create persons from per_no_by_gender={per_no: gender}, run import.""" - import unittest.mock as mock - db = _make_db() - pmap = {} - for per_no, gender in per_no_by_gender.items(): - p = Person() - p.set_gender(gender) - with DbTxn("setup", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): - libtmg.import_families(db, dataset, pmap, event_handle_map) - return db, pmap - - def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): - return _Rec(dsid=dsid, parent=parent, child=child, - ptype=ptype, primary=primary, pnote=pnote) - - def _father_type(self, num=1): - return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') - - def _mother_type(self, num=2): - return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') - - def test_father_child_creates_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - - def test_mother_child_creates_family(self): - db, pmap = self._run([self._mother_type()], - [self._pc(1, 2, ptype=2)], - {1: Person.FEMALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_mother_handle(), pmap[1]) - - def test_father_and_mother_same_family(self): - db, pmap = self._run( - [self._father_type(1), self._mother_type(2)], - [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], - {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, - ) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - self.assertEqual(fam.get_mother_handle(), pmap[2]) - - def test_child_added_to_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(len(fam.get_child_ref_list()), 1) - self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) - - def test_child_ref_type_biological(self): - from gramps.gen.lib import ChildRefType - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), - ChildRefType.BIRTH) - - def test_wrong_dataset_skipped(self): - db, _ = self._run([self._father_type()], - [self._pc(1, 2, ptype=1, dsid=2)], - {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) - self.assertEqual(db.get_number_of_families(), 0) - - def test_no_per_no_map_is_noop(self): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): - libtmg.import_families(db, 1, per_no_map=None) - self.assertEqual(db.get_number_of_families(), 0) - - def test_marriage_event_sets_rel_type_married(self): - from gramps.gen.lib import EventType, FamilyRelType - import unittest.mock as mock - - db = _make_db() - pmap = {} - for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): - p = Person() - p.set_gender(gender) - with DbTxn("s", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - - ev = Event() - ev.set_type(EventType(EventType.MARRIAGE)) - with DbTxn("s", db) as t: - db.add_event(ev, t) - - tagtypes = [self._father_type(1), self._mother_type(2)] - pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] - event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} - - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): - libtmg.import_families(db, 1, pmap, event_handle_map) - - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) - self.assertEqual(len(fam.get_event_ref_list()), 1) - - -# --------------------------------------------------------------------------- -# import_repositories — name, type inference, URL, notes -# --------------------------------------------------------------------------- - -class TestImportRepositories(unittest.TestCase): - - def _run(self, repo_records, per_no_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): - repo_map = libtmg.import_repositories(db, dataset, per_no_map) - return db, repo_map - - def _repo_rec(self, **kw): - defaults = dict(dsid=1, recno=1, name='City Library', - abbrev='', rnote='', rperno=0) - defaults.update(kw) - return _Rec(**defaults) - - def test_repository_created(self): - db, rmap = self._run([self._repo_rec()]) - self.assertEqual(db.get_number_of_repositories(), 1) - self.assertIn(1, rmap) - - def test_name_set(self): - db, rmap = self._run([self._repo_rec(name='National Archives')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'National Archives') - - def test_wrong_dataset_skipped(self): - db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_repositories(), 0) - - def test_type_inferred_from_name(self): - from gramps.gen.lib import RepositoryType - db, rmap = self._run([self._repo_rec(name='ancestry.com')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) - - def test_url_added_for_web_repo(self): - db, rmap = self._run([self._repo_rec(name='familysearch.org')]) - repo = db.get_repository_from_handle(rmap[1]) - urls = repo.get_url_list() - self.assertEqual(len(urls), 1) - self.assertIn('familysearch', urls[0].get_path()) - - def test_no_url_for_non_web_repo(self): - db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_url_list()), 0) - - def test_blank_name_falls_back_to_abbrev(self): - db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'TNA') - - def test_note_added_when_rnote_set(self): - db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_note_list()), 1) - note = db.get_note_from_handle(repo.get_note_list()[0]) - self.assertIn('Open Mon-Fri', note.get()) - - def test_returns_recno_to_handle_map(self): - db, rmap = self._run([self._repo_rec(recno=42)]) - self.assertIn(42, rmap) - - -# --------------------------------------------------------------------------- -# import_citations — creation and attachment to events / persons -# --------------------------------------------------------------------------- - -class TestImportCitations(unittest.TestCase): - - def _run(self, citation_records, names_records=None, pc_records=None, - source_handle_map=None, event_handle_map=None, per_no_map=None, - dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ - mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): - libtmg.import_citations(db, dataset, - source_handle_map=source_handle_map, - event_handle_map=event_handle_map, - per_no_map=per_no_map) - return db - - def _cit_rec(self, **kw): - defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, - exclude=False, subsource='', citref='', citmemo='', - sdsure='', snsure='', sssure='', spsure='', sfsure='') - defaults.update(kw) - return _Rec(**defaults) - - def test_no_source_map_is_noop(self): - db = self._run([self._cit_rec()]) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_created(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec()], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 1) - - def test_excluded_citation_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(exclude=True)], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_wrong_dataset_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(dsid=2)], - source_handle_map={1: src.get_handle()}, db=db, dataset=1) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_unknown_source_skipped(self): - db = _make_db() - db = self._run([self._cit_rec(majsource=99)], - source_handle_map={1: 'some_handle'}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_attached_to_event(self): - db = _make_db() - src = Source() - ev = Event() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_event(ev, t) - ev_handle = ev.get_handle() - - db = self._run([self._cit_rec(stype='E', refrec=7)], - source_handle_map={1: src.get_handle()}, - event_handle_map={7: (ev_handle, 1, 0, 0)}, - db=db) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(len(event.get_citation_list()), 1) - - def test_citation_attached_to_person_via_name(self): - db = _make_db() - src = Source() - p = Person() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_person(p, t) - phandle = p.get_handle() - - # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle - db = self._run( - [self._cit_rec(stype='N', refrec=3)], - names_records=[_Rec(dsid=1, recno=3, nper=5)], - source_handle_map={1: src.get_handle()}, - per_no_map={5: phandle}, - db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_citation_list()), 1) - - def test_subsource_becomes_page(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(subsource='p.42')], - source_handle_map={1: src.get_handle()}, db=db) - cit_handle = list(db.get_citation_handles())[0] - cit = db.get_citation_from_handle(cit_handle) - self.assertEqual(cit.get_page(), 'p.42') - - - -# ── TmgProject._read_pjc_config ────────────────────────────────────────── - -def _make_minimal_sqz(tmp_dir, pjc_content): - """Create a minimal .SQZ zip containing only a PJC file.""" - import zipfile as _zf - pjc_path = os.path.join(tmp_dir, 'test.pjc') - sqz_path = os.path.join(tmp_dir, 'test.sqz') - with open(pjc_path, 'w', encoding='latin-1') as f: - f.write(pjc_content) - with _zf.ZipFile(sqz_path, 'w') as zf: - zf.write(pjc_path, 'test.pjc') - return sqz_path - - -class _MockUser: - """Minimal stand-in for the Gramps user object used in importData.""" - def __init__(self): - self.error_shown = False - self.error_message = None - self.uistate = None - - def notify_error(self, title, message=''): - self.error_shown = True - self.error_message = message - - def begin_progress(self, *a, **kw): pass - def end_progress(self): pass - def step_progress(self): pass - - -class TestReadPjcConfig(unittest.TestCase): - """Tests for TmgProject._read_pjc_config PJC parsing.""" - - _MINIMAL_PJC = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Researcher]\n" - "Name=Test User\n" - ) - - def _make_project(self, pjc_content, tmp_path): - pjc_file = os.path.join(tmp_path, "test.pjc") - with open(pjc_file, 'w', encoding='latin-1') as f: - f.write(pjc_content) - return libtmg.TmgProject(pjc_file) - - def test_well_formed_pjc_returns_version(self): - """A clean PJC file parses successfully and version() returns a float.""" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(self._MINIMAL_PJC, tmp) - self.assertEqual(project.version(), 11.0) - - def test_malformed_section_header_does_not_raise(self): - """Lines like '[Exho' (no closing bracket) are silently dropped.""" - pjc = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Exho\n" # malformed — the crash trigger - "SomeGarbage\n" - "[Researcher]\n" - "Name=Test User\n" - ) - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - # Must not raise; must still find [Stamp] - self.assertEqual(project.version(), 11.0) - - def test_null_bytes_stripped(self): - """NUL bytes in the PJC file are stripped before parsing.""" - pjc = "[Stamp]\x00\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - self.assertEqual(project.version(), 11.0) - - def test_parse_error_returns_partial_config(self): - """If configparser still raises after filtering, a warning is logged - and a (possibly empty) config object is returned rather than crashing.""" - import logging - # Feed content that survives the filter but still breaks configparser: - # a key=value line before any section header is technically invalid. - pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: - cfg = project._read_pjc_config() - self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() - for m in cm.output)) - # Config object is returned (not None), even if incomplete - self.assertIsNotNone(cfg) - - def test_version_too_old_notifies_user(self): - """A PJC version < 11.0 calls user.notify_error and aborts import.""" - pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for old version") - self.assertIn('9.05', user.error_message or '', - "Error message should mention TMG 9.05") - - def test_missing_pjc_version_notifies_user(self): - """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" - pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for missing version") - if __name__ == '__main__': unittest.main() diff --git a/TMGimporter/tests/test_linux_libtmg.py b/TMGimporter/tests/test_linux_libtmg.py new file mode 100644 index 000000000..35bd11717 --- /dev/null +++ b/TMGimporter/tests/test_linux_libtmg.py @@ -0,0 +1,1218 @@ +"""Linux-only unit tests for libtmg.py + +Split off from test_libtmg.py: every class in this module creates an +in-memory Gramps SQLite database via + + make_database("sqlite").load(":memory:", None) + +which currently hangs on Windows under the conda-forge GTK + pip Gramps +combination used by the CI unit-test-windows job. Pure-logic tests that +do not touch the Gramps DB layer stay in test_libtmg.py and run on every +OS. + +Filename convention (see .github/workflows/ci.yml): + test_*.py general (every OS) + test_linux_*.py Linux-only + test_windows_*.py Windows-only + test_integration_*.py Linux-only, full-pipeline/DB-backed +""" + +import sys +import os +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import libtmg + +from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source +from gramps.gen.db.utils import make_database +from gramps.gen.db import DbTxn + + +# --------------------------------------------------------------------------- +# Helpers shared across test cases +# --------------------------------------------------------------------------- + +class _Rec: + """Minimal fake DBF record — set any field via keyword arguments.""" + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def _table(records): + """Return an object that behaves like a dbf.Table used as a context manager. + + libtmg uses tables in two ways: + with tmgFoo: + for record in tmgFoo: # iterates over context-managed table + """ + class _FakeTable: + def __enter__(self): + return self + def __exit__(self, *_): + return False + def __iter__(self): + return iter(records) + return _FakeTable() + + +def _make_db(): + """Return a fresh in-memory Gramps database.""" + db = make_database("sqlite") + db.load(":memory:", None) + return db + + +def _add_person(db): + """Add an empty Person to db and return (db, handle).""" + p = Person() + with DbTxn("setup", db) as t: + db.add_person(p, t) + return db, p.get_handle() + + +# --------------------------------------------------------------------------- +# import_notes — per-person note creation from tmg events +# --------------------------------------------------------------------------- + +class TestImportNotes(unittest.TestCase): + + def _patch(self, tagtypes_records, events_records): + """Patch libtmg globals and return a context manager.""" + import unittest.mock as mock + patches = [ + mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), + mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), + ] + return patches + + def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + libtmg.import_notes(db, dataset, per_no_map) + return db + + def test_no_per_no_map_is_noop(self): + """Passing per_no_map=None must not touch any table.""" + import unittest.mock as mock + db = _make_db() + mock_table = mock.MagicMock() + with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ + mock.patch('libtmg.tmgEvents', mock_table, create=True): + libtmg.import_notes(db, 1, per_no_map=None) + mock_table.__enter__.assert_not_called() + + def test_note_attached_to_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {42: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, + efoot='Born in London')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 1) + + def test_note_text_stored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot=' Some note text ')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'Some note text') + + def test_note_type_is_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='hello')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get_type(), NoteType.PERSON) + + def test_tmg_codes_stripped_from_note(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='[:ITAL:]italicised[:ITAL:]')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'italicised') + + def test_empty_note_text_skipped(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 0) + + def test_unknown_person_skipped(self): + db = _make_db() + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, + efoot='orphan note')], + per_no_map={}, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_non_note_etype_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + # etype=10 is not a Note type + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot='should be ignored')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_wrong_dataset_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, + efoot='wrong dataset')], + per_no_map=per_no_map, dataset=1, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_multiple_notes_for_one_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[ + _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), + _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), + ], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 2) + + def test_no_note_tag_type_defined(self): + """If the dataset has no 'Note' tag type, nothing is imported.""" + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[], # no tag types at all + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='orphan')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + +# --------------------------------------------------------------------------- +# trial_events — event import and Note-etype skip +# --------------------------------------------------------------------------- + +# A minimal raw date string for an exact date (1900-06-15) +_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' +_EMPTY_DATE = '' + + +class TestTrialEvents(unittest.TestCase): + + def _run(self, tagtypes_records, events_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + handle_map = libtmg.import_events(db, dataset) + return db, handle_map + + def test_regular_event_creates_db_entry(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertIn(1, hmap) + + def test_handle_map_tuple_has_four_elements(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, + placenum=7, edate=_EMPTY_DATE, efoot='')] + _, hmap = self._run(_tagtypes, _events) + entry = hmap[5] + self.assertEqual(len(entry), 4) + _handle, per1, per2, placenum = entry + self.assertEqual(per1, 3) + self.assertEqual(per2, 0) + self.assertEqual(placenum, 7) + + def test_note_etype_event_not_in_handle_map(self): + _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] + _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 0) + self.assertNotIn(1, hmap) + + def test_event_memo_stored_as_description(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_memo_tmg_codes_stripped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, + efoot='[:CR:]born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_date_set(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EXACT_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + d = event.get_date_object() + self.assertEqual(d.get_year(), 1900) + self.assertEqual(d.get_month(), 6) + + def test_wrong_dataset_skipped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events, dataset=1) + self.assertEqual(db.get_number_of_events(), 0) + + def test_mixed_note_and_regular_events(self): + _tagtypes = [ + _Rec(dsid=1, etypenum=77, etypename='Note'), + _Rec(dsid=1, etypenum=10, etypename='Birth'), + ] + _events = [ + _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note'), + _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot=''), + ] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertNotIn(1, hmap) + self.assertIn(2, hmap) + + +# --------------------------------------------------------------------------- +# import_sources — info-field parsing and author/publication split +# --------------------------------------------------------------------------- + +class TestImportSources(unittest.TestCase): + + def _run(self, src_components, src_repo_links, sources_records, + repo_handle_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ + mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ + mock.patch('libtmg.tmgSources', _table(sources_records), create=True): + smap = libtmg.import_sources(db, dataset, repo_handle_map) + return db, smap + + def _source_rec(self, **kw): + defaults = dict(dsid=1, majnum=1, mactive=True, + title='Test Source', abbrev='', info='', + text='', fform='', sform='', bform='', reminders='') + defaults.update(kw) + return _Rec(**defaults) + + def test_source_created(self): + db, smap = self._run([], [], [self._source_rec()]) + self.assertEqual(db.get_number_of_sources(), 1) + self.assertIn(1, smap) + + def test_title_set(self): + db, smap = self._run([], [], [self._source_rec(title='My Source')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_title(), 'My Source') + + def test_abbreviation_set(self): + db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_abbreviation(), 'MySrc') + + def test_inactive_source_skipped(self): + db, smap = self._run([], [], [self._source_rec(mactive=False)]) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_wrong_dataset_skipped(self): + db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_author_element_sets_author(self): + # recno 1 → position 0 in $!& split + components = [_Rec(recno=1, element='[AUTHOR]')] + rec = self._source_rec(info='John Smith') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'John Smith') + + def test_non_author_element_sets_publication_info(self): + components = [_Rec(recno=1, element='[TITLE]')] + rec = self._source_rec(info='Some Title') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertIn('TITLE', src.get_publication_info()) + self.assertIn('Some Title', src.get_publication_info()) + + def test_multiple_authors_joined_with_semicolon(self): + # positions 0 and 1 → recno 1 and 2 + components = [ + _Rec(recno=1, element='[AUTHOR]'), + _Rec(recno=2, element='[EDITOR]'), + ] + rec = self._source_rec(info='Alice$!&Bob') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Alice; Bob') + + def test_empty_info_position_skipped(self): + # position 0 empty, position 1 filled → recno 2 = [AUTHOR] + components = [ + _Rec(recno=1, element='[TITLE]'), + _Rec(recno=2, element='[AUTHOR]'), + ] + rec = self._source_rec(info='$!&Jane Doe') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Jane Doe') + self.assertEqual(src.get_publication_info(), '') + + def test_note_fields_become_notes(self): + rec = self._source_rec(text='original text', fform='', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 1) + note = db.get_note_from_handle(src.get_note_list()[0]) + self.assertIn('original text', note.get()) + + def test_multiple_note_fields_each_become_a_note(self): + rec = self._source_rec(text='txt', fform='fn', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 2) + + +# --------------------------------------------------------------------------- +# import_places — name reconstruction, type resolution, note parts +# --------------------------------------------------------------------------- + +class TestImportPlaces(unittest.TestCase): + + def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ + mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ + mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ + mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + pmap = libtmg.import_places(db, dataset) + return db, pmap + + # Convenience: build part_type, place_dict, ppv records for a single place + def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): + """ + parts: list of (label, value) e.g. [('City','London'),('Country','UK')] + Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) + """ + part_type_recs = [] + place_dict_recs = [] + ppv_recs = [] + for i, (label, value) in enumerate(parts): + type_id = i + 1 + uid = i + 100 + part_type_recs.append(_Rec(type=type_id, value=label)) + place_dict_recs.append(_Rec(uid=uid, value=value)) + ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) + place_recs = [_Rec(dsid=dataset, recno=recno, + shortplace=shortplace, comment=comment)] + return part_type_recs, place_dict_recs, ppv_recs, place_recs + + def test_city_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'London') + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_country_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.COUNTRY) + + def test_city_state_country_name_order(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + # GEO_ORDER: Addressee, Detail, City, County, State, Country + self.assertEqual(place.get_name().get_value(), + 'Paris, Île-de-France, France') + + def test_most_specific_type_wins(self): + # City is more specific than Country in _GEO_ORDER + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Berlin'), ('Country', 'Germany') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_empty_place_skipped(self): + # No parts and no shortplace → nothing imported + place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertEqual(db.get_number_of_places(), 0) + self.assertNotIn(1, pmap) + + def test_shortplace_fallback(self): + # No parts but shortplace set → use it + place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'Somewhere') + + def test_note_parts_go_to_note(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Rome'), ('Postal', '00100') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(len(place.get_note_list()), 1) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('Postal', note.get()) + self.assertIn('00100', note.get()) + + def test_comment_goes_to_note(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('see also', note.get()) + + def test_wrong_dataset_skipped(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) + db, pmap = self._run(pt, pd, ppv, pl, dataset=1) + self.assertEqual(db.get_number_of_places(), 0) + + def test_returns_recno_to_handle_map(self): + pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(42, pmap) + + +# --------------------------------------------------------------------------- +# link_event_places — event gets its place handle set +# --------------------------------------------------------------------------- + +class TestLinkEventPlaces(unittest.TestCase): + + def _make_event(self, db): + from gramps.gen.db import DbTxn + ev = Event() + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def _make_place(self, db): + from gramps.gen.db import DbTxn + pl = Place() + with DbTxn("setup", db) as t: + db.add_place(pl, t) + return pl.get_handle() + + def test_place_linked_to_event(self): + db = _make_db() + ev_handle = self._make_event(db) + pl_handle = self._make_place(db) + event_handle_map = {1: (ev_handle, 1, 0, 7)} + place_handle_map = {7: pl_handle} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), pl_handle) + + def test_zero_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 0)} + place_handle_map = {0: self._make_place(db)} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_unknown_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 99)} + libtmg.link_event_places(db, event_handle_map, {}) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_event_places(db, {}, {}) # must not raise + libtmg.link_event_places(db, None, None) + + +# --------------------------------------------------------------------------- +# Pure functions: num_to_month, num_to_date, parse_date +# --------------------------------------------------------------------------- +class TestShortPlaceName(unittest.TestCase): + + def _run(self, places_records, placenum, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + return libtmg.short_place_name(db, placenum, dataset) + + def test_returns_shortplace(self): + rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=5), 'New York') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=1), 'London') + + def test_wrong_recno_returns_none(self): + rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=1, dataset=1)) + + +class TestTagTypeName(unittest.TestCase): + + def _run(self, tagtypes_records, eventtype, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): + return libtmg.tag_type_name(db, eventtype, dataset) + + def test_returns_name(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') + self.assertEqual(self._run([rec], eventtype=2), 'Birth') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, etypenum=3, etypename='Death ') + self.assertEqual(self._run([rec], eventtype=3), 'Death') + + def test_wrong_eventtype_returns_none(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) + + +# --------------------------------------------------------------------------- +# import_people — name parsing, gender, dataset filter +# --------------------------------------------------------------------------- + +class TestImportPeople(unittest.TestCase): + + def _run(self, names_records, people_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ + mock.patch('libtmg.tmgPeople', _table(people_records), create=True): + per_no_map = libtmg.import_people(db, dataset) + return db, per_no_map + + def _name_rec(self, **kw): + defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') + defaults.update(kw) + return _Rec(**defaults) + + def _person_rec(self, **kw): + defaults = dict(dsid=1, per_no=1, sex='M') + defaults.update(kw) + return _Rec(**defaults) + + def test_person_created(self): + db, pmap = self._run([self._name_rec()], [self._person_rec()]) + self.assertEqual(db.get_number_of_people(), 1) + + def test_returns_per_no_map(self): + db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) + self.assertIn(5, pmap) + + def test_surname_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'JONES') + + def test_given_name_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') + + def test_male_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.MALE) + + def test_female_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.FEMALE) + + def test_unknown_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.UNKNOWN) + + def test_non_primary_name_skipped(self): + db, pmap = self._run( + [self._name_rec(primary=False, srnamedisp='ALT, Name')], + [self._person_rec()] + ) + self.assertEqual(db.get_number_of_people(), 0) + + def test_wrong_dataset_skipped(self): + db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], + dataset=1) + self.assertEqual(db.get_number_of_people(), 0) + + def test_no_comma_surname_only(self): + # srnamedisp with no comma → surname=full string, given='' + db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], + [self._person_rec()]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') + self.assertEqual(p.get_primary_name().get_first_name(), '') + + +# --------------------------------------------------------------------------- +# link_person_events — EventRefs, birth/death special refs +# --------------------------------------------------------------------------- + +class TestLinkPersonEvents(unittest.TestCase): + + def _make_typed_event(self, db, event_type_int): + from gramps.gen.lib import EventType + ev = Event() + ev.set_type(EventType(event_type_int)) + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def test_individual_event_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.OCCUPATION) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 1) + + def test_couple_event_not_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.MARRIAGE) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 2, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 0) + + def test_birth_event_sets_birth_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.BIRTH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_birth_ref()) + self.assertEqual(p.get_birth_ref().ref, ev_handle) + + def test_death_event_sets_death_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.DEATH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_death_ref()) + + def test_unknown_person_skipped(self): + from gramps.gen.lib import EventType + db = _make_db() + ev_handle = self._make_typed_event(db, EventType.BIRTH) + # Must not raise even when per1 has no entry in per_no_map + libtmg.link_person_events(db, + per_no_map={}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_person_events(db, None, None) + libtmg.link_person_events(db, {}, {}) + + +# --------------------------------------------------------------------------- +# import_families — parent-child grouping, couple events, rel type +# --------------------------------------------------------------------------- + +class TestImportFamilies(unittest.TestCase): + + def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, + dataset=1): + """Create persons from per_no_by_gender={per_no: gender}, run import.""" + import unittest.mock as mock + db = _make_db() + pmap = {} + for per_no, gender in per_no_by_gender.items(): + p = Person() + p.set_gender(gender) + with DbTxn("setup", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): + libtmg.import_families(db, dataset, pmap, event_handle_map) + return db, pmap + + def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): + return _Rec(dsid=dsid, parent=parent, child=child, + ptype=ptype, primary=primary, pnote=pnote) + + def _father_type(self, num=1): + return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') + + def _mother_type(self, num=2): + return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') + + def test_father_child_creates_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + + def test_mother_child_creates_family(self): + db, pmap = self._run([self._mother_type()], + [self._pc(1, 2, ptype=2)], + {1: Person.FEMALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_mother_handle(), pmap[1]) + + def test_father_and_mother_same_family(self): + db, pmap = self._run( + [self._father_type(1), self._mother_type(2)], + [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], + {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, + ) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + self.assertEqual(fam.get_mother_handle(), pmap[2]) + + def test_child_added_to_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(len(fam.get_child_ref_list()), 1) + self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) + + def test_child_ref_type_biological(self): + from gramps.gen.lib import ChildRefType + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), + ChildRefType.BIRTH) + + def test_wrong_dataset_skipped(self): + db, _ = self._run([self._father_type()], + [self._pc(1, 2, ptype=1, dsid=2)], + {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) + self.assertEqual(db.get_number_of_families(), 0) + + def test_no_per_no_map_is_noop(self): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): + libtmg.import_families(db, 1, per_no_map=None) + self.assertEqual(db.get_number_of_families(), 0) + + def test_marriage_event_sets_rel_type_married(self): + from gramps.gen.lib import EventType, FamilyRelType + import unittest.mock as mock + + db = _make_db() + pmap = {} + for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): + p = Person() + p.set_gender(gender) + with DbTxn("s", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + + ev = Event() + ev.set_type(EventType(EventType.MARRIAGE)) + with DbTxn("s", db) as t: + db.add_event(ev, t) + + tagtypes = [self._father_type(1), self._mother_type(2)] + pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] + event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} + + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): + libtmg.import_families(db, 1, pmap, event_handle_map) + + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) + self.assertEqual(len(fam.get_event_ref_list()), 1) + + +# --------------------------------------------------------------------------- +# import_repositories — name, type inference, URL, notes +# --------------------------------------------------------------------------- + +class TestImportRepositories(unittest.TestCase): + + def _run(self, repo_records, per_no_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): + repo_map = libtmg.import_repositories(db, dataset, per_no_map) + return db, repo_map + + def _repo_rec(self, **kw): + defaults = dict(dsid=1, recno=1, name='City Library', + abbrev='', rnote='', rperno=0) + defaults.update(kw) + return _Rec(**defaults) + + def test_repository_created(self): + db, rmap = self._run([self._repo_rec()]) + self.assertEqual(db.get_number_of_repositories(), 1) + self.assertIn(1, rmap) + + def test_name_set(self): + db, rmap = self._run([self._repo_rec(name='National Archives')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'National Archives') + + def test_wrong_dataset_skipped(self): + db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_repositories(), 0) + + def test_type_inferred_from_name(self): + from gramps.gen.lib import RepositoryType + db, rmap = self._run([self._repo_rec(name='ancestry.com')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) + + def test_url_added_for_web_repo(self): + db, rmap = self._run([self._repo_rec(name='familysearch.org')]) + repo = db.get_repository_from_handle(rmap[1]) + urls = repo.get_url_list() + self.assertEqual(len(urls), 1) + self.assertIn('familysearch', urls[0].get_path()) + + def test_no_url_for_non_web_repo(self): + db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_url_list()), 0) + + def test_blank_name_falls_back_to_abbrev(self): + db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'TNA') + + def test_note_added_when_rnote_set(self): + db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_note_list()), 1) + note = db.get_note_from_handle(repo.get_note_list()[0]) + self.assertIn('Open Mon-Fri', note.get()) + + def test_returns_recno_to_handle_map(self): + db, rmap = self._run([self._repo_rec(recno=42)]) + self.assertIn(42, rmap) + + +# --------------------------------------------------------------------------- +# import_citations — creation and attachment to events / persons +# --------------------------------------------------------------------------- + +class TestImportCitations(unittest.TestCase): + + def _run(self, citation_records, names_records=None, pc_records=None, + source_handle_map=None, event_handle_map=None, per_no_map=None, + dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ + mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): + libtmg.import_citations(db, dataset, + source_handle_map=source_handle_map, + event_handle_map=event_handle_map, + per_no_map=per_no_map) + return db + + def _cit_rec(self, **kw): + defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, + exclude=False, subsource='', citref='', citmemo='', + sdsure='', snsure='', sssure='', spsure='', sfsure='') + defaults.update(kw) + return _Rec(**defaults) + + def test_no_source_map_is_noop(self): + db = self._run([self._cit_rec()]) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_created(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec()], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 1) + + def test_excluded_citation_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(exclude=True)], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_wrong_dataset_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(dsid=2)], + source_handle_map={1: src.get_handle()}, db=db, dataset=1) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_unknown_source_skipped(self): + db = _make_db() + db = self._run([self._cit_rec(majsource=99)], + source_handle_map={1: 'some_handle'}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_attached_to_event(self): + db = _make_db() + src = Source() + ev = Event() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_event(ev, t) + ev_handle = ev.get_handle() + + db = self._run([self._cit_rec(stype='E', refrec=7)], + source_handle_map={1: src.get_handle()}, + event_handle_map={7: (ev_handle, 1, 0, 0)}, + db=db) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(len(event.get_citation_list()), 1) + + def test_citation_attached_to_person_via_name(self): + db = _make_db() + src = Source() + p = Person() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_person(p, t) + phandle = p.get_handle() + + # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle + db = self._run( + [self._cit_rec(stype='N', refrec=3)], + names_records=[_Rec(dsid=1, recno=3, nper=5)], + source_handle_map={1: src.get_handle()}, + per_no_map={5: phandle}, + db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_citation_list()), 1) + + def test_subsource_becomes_page(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(subsource='p.42')], + source_handle_map={1: src.get_handle()}, db=db) + cit_handle = list(db.get_citation_handles())[0] + cit = db.get_citation_from_handle(cit_handle) + self.assertEqual(cit.get_page(), 'p.42') + + + +# ── TmgProject._read_pjc_config ────────────────────────────────────────── +def _make_minimal_sqz(tmp_dir, pjc_content): + """Create a minimal .SQZ zip containing only a PJC file.""" + import zipfile as _zf + pjc_path = os.path.join(tmp_dir, 'test.pjc') + sqz_path = os.path.join(tmp_dir, 'test.sqz') + with open(pjc_path, 'w', encoding='latin-1') as f: + f.write(pjc_content) + with _zf.ZipFile(sqz_path, 'w') as zf: + zf.write(pjc_path, 'test.pjc') + return sqz_path + + +class _MockUser: + """Minimal stand-in for the Gramps user object used in importData.""" + def __init__(self): + self.error_shown = False + self.error_message = None + self.uistate = None + + def notify_error(self, title, message=''): + self.error_shown = True + self.error_message = message + + def begin_progress(self, *a, **kw): pass + def end_progress(self): pass + def step_progress(self): pass + + +class TestReadPjcConfig(unittest.TestCase): + """Tests for TmgProject._read_pjc_config PJC parsing.""" + + _MINIMAL_PJC = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Researcher]\n" + "Name=Test User\n" + ) + + def _make_project(self, pjc_content, tmp_path): + pjc_file = os.path.join(tmp_path, "test.pjc") + with open(pjc_file, 'w', encoding='latin-1') as f: + f.write(pjc_content) + return libtmg.TmgProject(pjc_file) + + def test_well_formed_pjc_returns_version(self): + """A clean PJC file parses successfully and version() returns a float.""" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(self._MINIMAL_PJC, tmp) + self.assertEqual(project.version(), 11.0) + + def test_malformed_section_header_does_not_raise(self): + """Lines like '[Exho' (no closing bracket) are silently dropped.""" + pjc = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Exho\n" # malformed — the crash trigger + "SomeGarbage\n" + "[Researcher]\n" + "Name=Test User\n" + ) + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + # Must not raise; must still find [Stamp] + self.assertEqual(project.version(), 11.0) + + def test_null_bytes_stripped(self): + """NUL bytes in the PJC file are stripped before parsing.""" + pjc = "[Stamp]\x00\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + self.assertEqual(project.version(), 11.0) + + def test_parse_error_returns_partial_config(self): + """If configparser still raises after filtering, a warning is logged + and a (possibly empty) config object is returned rather than crashing.""" + import logging + # Feed content that survives the filter but still breaks configparser: + # a key=value line before any section header is technically invalid. + pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: + cfg = project._read_pjc_config() + self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() + for m in cm.output)) + # Config object is returned (not None), even if incomplete + self.assertIsNotNone(cfg) + + def test_version_too_old_notifies_user(self): + """A PJC version < 11.0 calls user.notify_error and aborts import.""" + pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for old version") + self.assertIn('9.05', user.error_message or '', + "Error message should mention TMG 9.05") + + def test_missing_pjc_version_notifies_user(self): + """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" + pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for missing version") + +if __name__ == '__main__': + unittest.main() From dd0fd3867fdf07b50e28bb6f2f39e7914419684a Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 11 May 2026 17:41:08 +0200 Subject: [PATCH 06/47] CI image: add gir1.2-gexiv2-0.10 for EditExifMetadata + PhotoTagging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EditExifMetadata and PhotoTaggingGramplet both do `from gi.repository import GExiv2` at module load. The CI image installs several gir1.2-* typelibs (glib, gtk, pango, gdkpixbuf, atk) but not gexiv2, so plugin-registration smoke tests on those addons fail with: cannot import name GExiv2, introspection typelib not found Add the missing apt package. GObject Introspection typelibs come from system apt packages, not pip — they cannot be auto-derived from requires_gi the way requires_mod is. Companion to addons-source PRs #878 (EditExifMetadata declares requires_gi=[(GExiv2, 0.10)] alongside requires_mod=[Pillow]) and #880 (PhotoTaggingGramplet declares requires_gi=[(GExiv2, 0.10)]). Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/docker/gramps-ci/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index d5102ac99..4ff57cd58 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -26,6 +26,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gir1.2-pango-1.0 \ gir1.2-gdkpixbuf-2.0 \ gir1.2-atk-1.0 \ + gir1.2-gexiv2-0.10 \ gcc \ pkg-config \ python3-dev \ From 205b21ca96be3fe279db3fdc0bb35023bf6721c7 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 11 May 2026 17:57:17 +0200 Subject: [PATCH 07/47] =?UTF-8?q?CI:=20lint=20trailing-whitespace=20check?= =?UTF-8?q?=20=E2=80=94=20switch=20BRE=20[=20\t]=20to=20PCRE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `git grep '[ \t]$'` in BRE/ERE mode interprets `[ \t]` as the character class { space, backslash, 't' } — i.e. the bracket contains a literal 't', not a tab. The check therefore matches every line that ends in 't', '\', '[', ']' or a space, fires on ~3,000 false-positive lines across the tree, and would block any PR touching such a file even when no real trailing whitespace exists. Switch to PCRE via `-P` so `\t` means tab. Also tighten to `[ \t]+$` to make intent obvious (no behavioural change for the match decision). Verified on the live tree: - BRE: matches 3182 lines across ~430 files (mostly false positives). - PCRE: matches 597 lines across 21 files — real trailing whitespace. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a79cda8b..e714b6bfc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,10 @@ jobs: - name: Check trailing whitespace in Python files run: | - if git --no-pager grep --color -n --full-name '[ \t]$' -- '*.py'; then + # Use PCRE (-P): in BRE/ERE the bracket expression [ \t] is the + # set { space, backslash, 't' } — git grep matches anything ending + # in 't', not just whitespace. -P makes \t a tab. + if git --no-pager grep --color -n --full-name -P '[ \t]+$' -- '*.py'; then echo "::error::Trailing whitespace found in Python files" exit 1 fi From f5907afc9cc09a66a587d209c9266e0589e249b3 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Fri, 15 May 2026 10:31:14 +0200 Subject: [PATCH 08/47] ci: skip addons whose .gpr.py declares include_in_listing=False MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per Gary Griffin's request on PR #820: addons whose every register() call sets include_in_listing=False are not built or released by make.py, so the CI does not gate on them. Across the current addon tree this skips 11 directories (CheckPlaceTitles, DetId, FaceDetection, HtmlView, MongoDB, PhpGedView, Query, RebuildTypes, SourceIndex, SourceReferences, WordleGramplet) — the ones Gary called out in his comment, modulo case differences and a couple of names that no longer exist as directories. The check is inlined at each iteration site rather than centralised in a helper script: a small shell function (is_active) per relevant job step, plus an include_in_listing filter in the four addon-iterating tests in tests/test_plugin_registration.py. Repeating ~7 lines of bash across six job steps was preferred to introducing a separate .github/scripts/ helper file, mirroring the pattern the workflow already uses for the requires_mod auto-derive step. To re-enable CI gating for an addon, set include_in_listing=True on at least one register() call in its descriptor (or remove the field entirely — Gramps' default is True). TMGimporter, for example, stays gated because its main register() carries True even though three conditional ones use False. Affected steps: - lint (ruff): builds --exclude args from the skipped list - addon-structure: skips the po/template.pot check - compile-check (py_compile): skips files under skipped dirs - unit-test-linux / unit-test-windows: skips test modules - integration-test (per-addon): skips test modules The integration-test plugin-registration smoke step inherits the filter via the test file changes. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 99 +++++++++++++++++++++++++++++-- tests/test_plugin_registration.py | 31 ++++++---- 2 files changed, 115 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e714b6bfc..244bb13cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,32 @@ jobs: - uses: actions/checkout@v4 - name: Run ruff (syntax and import errors only) - run: ruff check --select=E9,F63,F7,F82 --no-fix --exclude='*.gpr.py' . + # Skip addon directories whose every register() in .gpr.py sets + # include_in_listing=False — those addons are not built or released + # by make.py, so CI does not gate on their lint state (per Gary + # Griffin's request on PR #820). To re-enable lint gating for an + # addon, set include_in_listing=True on at least one register() + # call in its descriptor (or remove the field — Gramps' default + # is True). Repeated inline rather than centralised so each job + # step stays self-contained. + shell: bash + run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } + excludes="" + for d in */; do + d="${d%/}" + ls "$d"/*.gpr.py >/dev/null 2>&1 || continue + is_active "$d" || excludes="$excludes --exclude=$d" + done + ruff check --select=E9,F63,F7,F82 --no-fix --exclude='*.gpr.py' $excludes . - name: Check trailing whitespace in Python files run: | @@ -49,11 +74,23 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Check all addons have po/template.pot + - name: Check all listed addons have po/template.pot + # Skip include_in_listing=False addons (see lint job for rationale). + shell: bash run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } failed=0 for gpr in */*.gpr.py; do addon_dir="$(dirname "$gpr")" + is_active "$addon_dir" || continue if [ ! -d "$addon_dir/po" ]; then echo "::error::$addon_dir is missing po/ directory" failed=1 @@ -63,7 +100,7 @@ jobs: fi done if [ "$failed" -eq 0 ]; then - echo "All addons have po/template.pot" + echo "All listed addons have po/template.pot" fi exit $failed @@ -78,11 +115,32 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Compile all Python files (excluding .gpr.py) + - name: Compile all Python files in listed addons (excluding .gpr.py) + # Skip include_in_listing=False addons (see lint job for rationale). shell: bash run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } + skipped="" + for d in */; do + d="${d%/}" + ls "$d"/*.gpr.py >/dev/null 2>&1 || continue + is_active "$d" || skipped="$skipped $d" + done failed=0 while IFS= read -r f; do + skip=0 + for s in $skipped; do + case "$f" in ./$s/*) skip=1; break;; esac + done + [ "$skip" = 1 ] && continue if ! python3 -m py_compile "$f" 2>&1; then failed=1 fi @@ -158,9 +216,20 @@ jobs: env: PYTHONPATH: . run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } modules="" for f in */tests/test_*.py; do [ -f "$f" ] || continue + addon="${f%%/*}" + is_active "$addon" || continue case "$(basename "$f")" in test_integration*) continue ;; test_windows_*) continue ;; @@ -243,9 +312,20 @@ jobs: env: PYTHONPATH: . run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } modules="" for f in */tests/test_*.py; do [ -f "$f" ] || continue + addon="${f%%/*}" + is_active "$addon" || continue case "$(basename "$f")" in test_integration*) continue ;; test_linux_*) continue ;; @@ -326,9 +406,20 @@ jobs: env: PYTHONPATH: . run: | + is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 + } modules="" for f in */tests/test_integration*.py; do [ -f "$f" ] || continue + addon="${f%%/*}" + is_active "$addon" || continue mod="${f%.py}" mod="${mod//\//.}" modules="$modules $mod" diff --git a/tests/test_plugin_registration.py b/tests/test_plugin_registration.py index f242fcf35..dae8d6f87 100644 --- a/tests/test_plugin_registration.py +++ b/tests/test_plugin_registration.py @@ -54,16 +54,26 @@ LOG = logging.getLogger(__name__) -def _get_addon_plugins(registry: Any) -> list[Any]: +def _get_addon_plugins(registry: Any, include_unlisted: bool = False) -> list[Any]: """Return all :class:`PluginData` objects whose ``fpath`` is inside the addons tree. + By default, plugins whose ``.gpr.py`` declares ``include_in_listing=False`` + are filtered out: those addons are not built or released by ``make.py``, + so this CI does not gate on their state (per Gary Griffin's discussion on + PR #820). Pass ``include_unlisted=True`` to inspect them anyway. + :param registry: A :class:`PluginRegister` instance. + :param include_unlisted: If ``True``, also return plugins whose + ``include_in_listing`` field is ``False``. + :type include_unlisted: bool :returns: List of :class:`PluginData` entries belonging to this repository. """ return [ pdata for pdata in registry._PluginRegister__plugindata - if pdata.fpath and ADDONS_ROOT in pdata.fpath + if pdata.fpath + and ADDONS_ROOT in pdata.fpath + and (include_unlisted or pdata.include_in_listing) ] @@ -118,12 +128,11 @@ def test_all_plugins_have_valid_metadata(self) -> None: self.assertTrue(pdata.version, f"Plugin {pdata.id} missing version") def test_target_version_is_6_0(self) -> None: - """All addons on this branch should target Gramps 6.0.""" + """All listed addons on this branch should target Gramps 6.0.""" issues: list[str] = [] - for pdata in self.plugin_registry._PluginRegister__plugindata: - if pdata.fpath and ADDONS_ROOT in pdata.fpath: - if not pdata.gramps_target_version.startswith("6.0"): - issues.append(f"{pdata.id}: targets {pdata.gramps_target_version}") + for pdata in _get_addon_plugins(self.plugin_registry): + if not pdata.gramps_target_version.startswith("6.0"): + issues.append(f"{pdata.id}: targets {pdata.gramps_target_version}") if issues: self.fail("Addons not targeting Gramps 6.0:\n" + "\n".join(issues)) @@ -218,11 +227,11 @@ class TestImportPluginSmoke(GrampsTestCase): """Verify import plugins have a callable ``import_function`` attribute.""" def test_import_plugins_have_callable(self) -> None: - """Each IMPORT plugin must reference a callable import function.""" + """Each listed IMPORT plugin must reference a callable import function.""" import_plugins = [ p for p in self.plugin_registry.type_plugins(IMPORT) - if p.fpath and ADDONS_ROOT in p.fpath + if p.fpath and ADDONS_ROOT in p.fpath and p.include_in_listing ] issues: list[str] = [] for pdata in import_plugins: @@ -250,11 +259,11 @@ class TestExportPluginSmoke(GrampsTestCase): """Verify export plugins have a callable ``export_function`` attribute.""" def test_export_plugins_have_callable(self) -> None: - """Each EXPORT plugin must reference a callable export function.""" + """Each listed EXPORT plugin must reference a callable export function.""" export_plugins = [ p for p in self.plugin_registry.type_plugins(EXPORT) - if p.fpath and ADDONS_ROOT in p.fpath + if p.fpath and ADDONS_ROOT in p.fpath and p.include_in_listing ] issues: list[str] = [] for pdata in export_plugins: From ff215ac6bffa03bd0543db012f3853d17923892c Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sat, 16 May 2026 01:03:47 +0200 Subject: [PATCH 09/47] CI: validate requires_mod names against find_spec (Pillow/PIL trap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The auto-derive step parses requires_mod from every .gpr.py and pip- installs each name, which validates the install-side of the contract. But Gramps' end-user dep gate (gen/utils/requirements.py:check_mod) calls find_spec() — the importable name, not the PyPI name. For most addons the two coincide; for Pillow they don't, so requires_mod= ["Pillow"] pip-installs cleanly, CI stays green, and the Addon Manager still rejects the addon on every end-user install. Adds a follow-up step after each "Install addon runtime deps" block (one per job: unit-test-linux, unit-test-windows, integration-test). For each declared requires_mod name: if pip-show confirms the package is installed but find_spec returns None, the declaration is wrong. Pip-install failures are skipped, so missing system deps (graphviz-dev for pygraphviz, libpq-dev for psycopg2) don't cause false positives. Verified locally: catches requires_mod=["Pillow"] with exit 1 and a GitHub annotation; current addons-source tree (all 9 declared mods) passes with no false positives. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 144 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 244bb13cc..531f2eedb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,6 +199,58 @@ jobs: echo "no requires_mod declarations found" fi + - name: Validate requires_mod names against Gramps' dep gate + # Cross-check: every requires_mod entry that pip successfully + # installed in the previous step must also resolve via + # find_spec(), since that is what Gramps' Addon Manager calls + # (gramps/gen/utils/requirements.py:check_mod). A name that + # pip-installs but does not import is a declaration bug — e.g. + # requires_mod=["Pillow"] when the importable name is "PIL". + # Pip-install failures upstream are skipped: those are + # system-dep / image gaps, not PR-caused. + shell: bash + run: | + python3 - <<'PY' + import ast, glob, re, subprocess, sys + from importlib.util import find_spec + + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + names = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + names.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + + bad = [] + for name in sorted(names): + installed = subprocess.run( + [sys.executable, "-m", "pip", "show", name], + capture_output=True, + ).returncode == 0 + if not installed: + print(f"~ {name} (pip-install failed earlier, skipping)") + continue + if find_spec(name) is None: + bad.append(name) + print(f"x {name} (pip-installed but find_spec returned None)") + else: + print(f"ok {name}") + + if bad: + print() + print(f"::error::Wrong requires_mod names: {bad}") + print("These pip-install but are not importable. requires_mod is") + print("consumed by gramps' check_mod() via find_spec(), so the") + print("importable module name is required (e.g. 'PIL', not 'Pillow').") + sys.exit(1) + PY + - name: Run per-addon unit tests # Filename convention (all OSes): # test_*.py — general (any OS) @@ -306,6 +358,51 @@ jobs: echo "no requires_mod declarations found" fi + - name: Validate requires_mod names against Gramps' dep gate + # See unit-test-linux for rationale. Uses `python` (conda-forge + # env) to match the surrounding Windows job style. + run: | + python - <<'PY' + import ast, glob, re, subprocess, sys + from importlib.util import find_spec + + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + names = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + names.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + + bad = [] + for name in sorted(names): + installed = subprocess.run( + [sys.executable, "-m", "pip", "show", name], + capture_output=True, + ).returncode == 0 + if not installed: + print(f"~ {name} (pip-install failed earlier, skipping)") + continue + if find_spec(name) is None: + bad.append(name) + print(f"x {name} (pip-installed but find_spec returned None)") + else: + print(f"ok {name}") + + if bad: + print() + print(f"::error::Wrong requires_mod names: {bad}") + print("These pip-install but are not importable. requires_mod is") + print("consumed by gramps' check_mod() via find_spec(), so the") + print("importable module name is required (e.g. 'PIL', not 'Pillow').") + sys.exit(1) + PY + - name: Run per-addon unit tests # See filename-convention note in unit-test-linux. The Windows # job runs test_*.py except test_linux_* and test_integration_*. @@ -389,6 +486,53 @@ jobs: echo "no requires_mod declarations found" fi + - name: Validate requires_mod names against Gramps' dep gate + # See unit-test-linux for rationale. This is the blocking copy: + # integration-test does not set continue-on-error, so a wrong + # requires_mod name surfaces as a CI failure here. + shell: bash + run: | + python3 - <<'PY' + import ast, glob, re, subprocess, sys + from importlib.util import find_spec + + pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + names = set() + for f in glob.glob("*/*.gpr.py"): + try: + text = open(f, encoding="utf-8").read() + except OSError: + continue + for m in pat.finditer(text): + try: + names.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + + bad = [] + for name in sorted(names): + installed = subprocess.run( + [sys.executable, "-m", "pip", "show", name], + capture_output=True, + ).returncode == 0 + if not installed: + print(f"~ {name} (pip-install failed earlier, skipping)") + continue + if find_spec(name) is None: + bad.append(name) + print(f"x {name} (pip-installed but find_spec returned None)") + else: + print(f"ok {name}") + + if bad: + print() + print(f"::error::Wrong requires_mod names: {bad}") + print("These pip-install but are not importable. requires_mod is") + print("consumed by gramps' check_mod() via find_spec(), so the") + print("importable module name is required (e.g. 'PIL', not 'Pillow').") + sys.exit(1) + PY + - name: Run plugin registration tests # shell: bash for consistency with the surrounding steps; the # current command uses no bashisms, but keeps this block safe From d2656125e8534f0c9e7ec8e29f49c6d8ba47d2bf Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 19 May 2026 17:52:17 +0200 Subject: [PATCH 10/47] =?UTF-8?q?CI:=20make=20lint=20job=20blocking=20?= =?UTF-8?q?=E2=80=94=20ruff=20backlog=20cleared=20on=20maintenance/gramps6?= =?UTF-8?q?0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops the lint job's continue-on-error: true (and its TODO comment), making ruff E9/F63/F7/F82 a blocking gate per the comment's own instruction. The ruff backlog the comment guarded against was cleared by 24 PRs (#843 + #847-#869) merged into maintenance/gramps60 between 2026-05-12 and 2026-05-18. Verified locally: pipx run ruff check --select=E9,F63,F7,F82 \ --no-fix --exclude='*.gpr.py' . on `gramps-project/addons-source:maintenance/gramps60` reports "All checks passed!". Other continue-on-error gates in this workflow (addon-structure, compile-check, integration) each guard their own backlogs and stay non-blocking until those clear separately. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 531f2eedb..76d44b4d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,6 @@ jobs: lint: name: Lint runs-on: ubuntu-latest - # Non-blocking until the existing ruff E9/F63/F7/F82 errors across the - # addon set are cleaned up in a follow-up PR. Flip this off in that PR. - continue-on-error: true container: image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 steps: From 0dd3f1b2a6e59cfe18ddf84324bb81d9444db7ed Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 19 May 2026 18:04:37 +0200 Subject: [PATCH 11/47] =?UTF-8?q?CI:=20make=20unit-test-linux=20and=20unit?= =?UTF-8?q?-test-windows=20blocking=20=E2=80=94=20backlog=20cleared?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops continue-on-error: true (and the TODO comments) from both unit-test jobs. They were guarding against "currently-broken addon unit modules (import failures, stale API usage)" that have since been fixed by the merged plugin-registration / dep-declaration round (#875 WordleGramplet, #876 SourceReferences, #878 EditExifMetadata, #879 MongoDB, #880 PhotoTaggingGramplet, #869 SurnameMappingGramplet, gramps#2299 ClipboardGramplet) plus the lint backlog (#843, #847-#869). Verified by the most recent fork CI run on eduralph/addons-source:maintenance/gramps60 (sha 8aabdd4db, 2026-05-18): both Unit Tests (Linux) and Unit Tests (Windows) reported success. Also clears two stale comments that referenced the removed flag: - unit-test-linux's "Run per-addon unit tests" step: drop the trailing "Falls silently under continue-on-error." sentence. - integration's requires_mod-validate step: collapse the now-moot "this is the blocking copy" rationale to a plain pointer at unit-test-linux. The lone remaining continue-on-error is on addon-structure (line 70), gated by the po/template.pot backlog. PRs #838-#841 were closed without merge; per Eduard's call the po/template.pot check stays non-blocking for now. Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76d44b4d5..ea85d0f98 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -150,9 +150,6 @@ jobs: unit-test-linux: name: Unit Tests (Linux) runs-on: ubuntu-latest - # Non-blocking until the currently-broken addon unit modules (import - # failures, stale API usage) are sorted out in follow-up PRs. - continue-on-error: true container: image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 steps: @@ -260,7 +257,7 @@ jobs: # shell: bash — the container's default shell is /bin/sh # (dash on python:3.12-slim), which does not support the # ${var//pattern/repl} and ${var%.py} parameter expansions - # used below. Falls silently under continue-on-error. + # used below. shell: bash env: PYTHONPATH: . @@ -303,8 +300,6 @@ jobs: unit-test-windows: name: Unit Tests (Windows) runs-on: windows-latest - # Non-blocking for the same reason as unit-test-linux. - continue-on-error: true defaults: run: shell: bash -el {0} @@ -484,9 +479,7 @@ jobs: fi - name: Validate requires_mod names against Gramps' dep gate - # See unit-test-linux for rationale. This is the blocking copy: - # integration-test does not set continue-on-error, so a wrong - # requires_mod name surfaces as a CI failure here. + # See unit-test-linux for rationale. shell: bash run: | python3 - <<'PY' From 9a91d89873d212106beb2098d084d0e095df909f Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 21:05:07 +0200 Subject: [PATCH 12/47] CI: derive image tag and make.py argument from the branch ref Replaces the seven hardcoded "gramps60" strings in ci.yml with values computed by a new setup job that strips "maintenance/" off the ref and validates the remainder matches grampsNN. Container jobs now pull gramps-ci: and the build step calls make.py with the matching suffix, so the same workflow runs unchanged on maintenance/gramps60, maintenance/gramps61, and any future maintenance/grampsNN branch. A non-matching ref (master, topic branch, gramps100) fails fast in the setup job rather than racing through with a malformed image tag. Verified: - YAML parses (python3 -c "import yaml; yaml.safe_load(...)") - bash dry-run of the case statement accepts gramps60/61/62/42 and rejects master, feature/foo, maintenance/gramps60-test, gramps100 - container jobs (lint, compile-check, unit-test-linux, integration-test, build) all carry needs: setup; non-container jobs (addon-structure, unit-test-windows) do not Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 50 ++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ea85d0f98..28a9213d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,22 +2,45 @@ name: CI on: push: - branches: [maintenance/gramps60] + branches: [maintenance/gramps**] pull_request: - branches: [maintenance/gramps60] - -env: - CI_IMAGE: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + branches: [maintenance/gramps**] jobs: + # ----------------------------------------------------------------- + # Setup — derive the branch suffix (gramps60 / gramps61 / …) from + # the ref. On push events github.ref_name is the branch being + # pushed; on pull_request events github.base_ref is the target + # branch. Either way the suffix is what follows "maintenance/". + # ----------------------------------------------------------------- + setup: + name: Setup + runs-on: ubuntu-latest + outputs: + branch_suffix: ${{ steps.compute.outputs.branch_suffix }} + ci_image: ${{ steps.compute.outputs.ci_image }} + steps: + - id: compute + shell: bash + run: | + ref="${{ github.base_ref || github.ref_name }}" + suffix="${ref#maintenance/}" + case "$suffix" in + gramps[0-9][0-9]) ;; + *) echo "::error::unexpected ref '$ref' (suffix '$suffix')"; exit 1 ;; + esac + echo "branch_suffix=$suffix" >> "$GITHUB_OUTPUT" + echo "ci_image=ghcr.io/${{ github.repository }}/gramps-ci:$suffix" >> "$GITHUB_OUTPUT" + # ----------------------------------------------------------------- # Lint (ci container) # ----------------------------------------------------------------- lint: name: Lint + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -106,9 +129,10 @@ jobs: # ----------------------------------------------------------------- compile-check: name: Compile Check + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -149,9 +173,10 @@ jobs: # ----------------------------------------------------------------- unit-test-linux: name: Unit Tests (Linux) + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -439,9 +464,9 @@ jobs: integration-test: name: Integration Tests (Gramps) runs-on: ubuntu-latest - needs: [unit-test-linux] + needs: [setup, unit-test-linux] container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} options: --init steps: - uses: actions/checkout@v4 @@ -570,9 +595,10 @@ jobs: # ----------------------------------------------------------------- build: name: Build + needs: setup runs-on: ubuntu-latest container: - image: ghcr.io/${{ github.repository }}/gramps-ci:gramps60 + image: ${{ needs.setup.outputs.ci_image }} steps: - uses: actions/checkout@v4 @@ -587,4 +613,4 @@ jobs: GRAMPSPATH: ${{ steps.gramps-path.outputs.path }} run: | mkdir -p ../download - python3 make.py gramps60 build all + python3 make.py "${{ needs.setup.outputs.branch_suffix }}" build all From abefdbe7b7fc9c9ea88c43075a373132600f7583 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 21:06:00 +0200 Subject: [PATCH 13/47] docker-build: derive image tag and Gramps series from the branch ref MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the three hardcoded "gramps60" strings in docker-build.yml with values computed from github.ref_name in a new "Compute branch parameters" step. The image is tagged gramps-ci: and built with GRAMPS_SERIES= (e.g. gramps60→6.0, gramps61→6.1), so the same workflow produces the correct image on every maintenance branch. Also drops the paths: filter from the push trigger. With the filter the workflow only fired when .github/docker/** changed, which means newly-created maintenance branches (inheriting the Dockerfile from their parent unchanged) would never produce their gramps-ci: image, and ci.yml jobs would fail pulling a non-existent tag. Every push to a maintenance branch now runs docker-build; buildx layer cache turns the steady-state case into a ~20-30 s no-op rebuild. Verified: - YAML parses (python3 -c "import yaml; yaml.safe_load(...)") - bash dry-run of the case/series logic produces 6.0, 6.1, 6.2, 7.0, 4.2 for the corresponding maintenance/grampsNN refs and rejects master Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/docker-build.yml | 32 +++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9289949b2..b466bc142 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -2,9 +2,11 @@ name: Build Docker Images on: push: - branches: [maintenance/gramps60] - paths: - - '.github/docker/**' + # No paths filter on purpose. Buildx layer cache makes the rebuild + # a ~20-30 s no-op when nothing under .github/docker/ has changed, + # in exchange for guaranteeing gramps-ci: exists on the + # first push to any newly-created maintenance branch. + branches: [maintenance/gramps**] workflow_dispatch: env: @@ -32,14 +34,32 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Compute branch parameters + # Derive the image-tag suffix and Gramps minor series from the + # branch ref. Same validation as ci.yml's setup job: anything + # outside maintenance/grampsNN fails fast. + id: params + shell: bash + run: | + ref="${{ github.ref_name }}" + suffix="${ref#maintenance/}" + case "$suffix" in + gramps[0-9][0-9]) ;; + *) echo "::error::unexpected ref '$ref' (suffix '$suffix')"; exit 1 ;; + esac + # gramps60 → 6.0, gramps61 → 6.1, gramps62 → 6.2, … + series="${suffix:6:1}.${suffix:7}" + echo "suffix=$suffix" >> "$GITHUB_OUTPUT" + echo "series=$series" >> "$GITHUB_OUTPUT" + - name: Docker metadata id: meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.REPO }}/gramps-ci tags: | - type=raw,value=gramps60 - type=sha,prefix=gramps60- + type=raw,value=${{ steps.params.outputs.suffix }} + type=sha,prefix=${{ steps.params.outputs.suffix }}- - name: Build and push gramps-ci uses: docker/build-push-action@v6 @@ -48,5 +68,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + build-args: | + GRAMPS_SERIES=${{ steps.params.outputs.series }} cache-from: type=gha cache-to: type=gha,mode=max From 99276264a931bdab47d9a27a8b3ec16a97aca19f Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 21:07:17 +0200 Subject: [PATCH 14/47] Dockerfile: parameterise Gramps series via GRAMPS_SERIES build arg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hardcoded "gramps>=6.0,<6.1" pip pin with "gramps==\${GRAMPS_SERIES}.*", and pulls GRAMPS_SERIES from a build arg with no default. Same Dockerfile now builds gramps-ci:gramps60 when invoked with GRAMPS_SERIES=6.0, gramps-ci:gramps61 with 6.1, and so on. docker-build.yml derives the value from the branch ref. No default on GRAMPS_SERIES on purpose: a wrong default would silently produce an image for the wrong Gramps series that "looks fine" but is mismatched against the branch's addon code. A guard line fails the build loudly if the arg is missing. Verified: - docker build --check --build-arg GRAMPS_SERIES=6.0 → no warnings - a minimal repro Dockerfile invoking the guard without GRAMPS_SERIES exits 1 with "GRAMPS_SERIES is required (e.g. 6.0)" Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/docker/gramps-ci/Dockerfile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index 4ff57cd58..d979c15f6 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -1,6 +1,9 @@ # .github/docker/gramps-ci/Dockerfile # -# Unified Gramps 6.0 CI image. Includes everything jobs need: +# Gramps CI image. The Gramps minor series (6.0, 6.1, …) is picked at +# build time via the GRAMPS_SERIES build arg, so the same Dockerfile +# produces gramps-ci:gramps60, gramps-ci:gramps61, … without per-branch +# edits. Includes everything jobs need: # - Python + pip-installed Gramps, PyGObject, pycairo # - GTK typelibs (so addon modules that `from gi.repository import Gtk` # at module load time are importable — widgets still need xvfb to render) @@ -13,11 +16,21 @@ # `--init` (or use a container runtime that injects tini) because xvfb-run # hangs if it inherits PID 1. # +# Local build: +# docker build --build-arg GRAMPS_SERIES=6.0 .github/docker/gramps-ci +# ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim +# Gramps minor series to install (e.g. 6.0, 6.1). No default — must be +# passed explicitly so a wrong default cannot silently produce an image +# for the wrong Gramps series. docker-build.yml derives this from the +# branch ref. +ARG GRAMPS_SERIES +RUN [ -n "$GRAMPS_SERIES" ] || { echo "GRAMPS_SERIES is required (e.g. 6.0)"; exit 1; } + LABEL org.opencontainers.image.source="https://github.com/gramps-project/addons-source" -LABEL org.opencontainers.image.description="Unified Gramps 6.0 CI image (Python, Gramps, GTK typelibs, xvfb)" +LABEL org.opencontainers.image.description="Gramps ${GRAMPS_SERIES} CI image (Python, Gramps, GTK typelibs, xvfb)" RUN apt-get update && apt-get install -y --no-install-recommends \ libgirepository-2.0-dev \ @@ -46,7 +59,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN pip install --no-cache-dir \ PyGObject \ pycairo \ - "gramps>=6.0,<6.1" \ + "gramps==${GRAMPS_SERIES}.*" \ orjson \ ruff From 894e4843a24f3dabed3f8c26215d08d09f5c5247 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 23:21:19 +0200 Subject: [PATCH 15/47] CI image: hybrid PyPI / git-clone install for unreleased maintenance branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR 820's "pip install gramps==${SERIES}.*" only works on maintenance branches whose Gramps release has been published to PyPI. The maintenance/gramps61 branch carries the in-development 6.1 series and has no PyPI release yet, so docker-build fails with "No matching distribution found for gramps==6.1.*" the moment the branch-neutral pipeline is exercised against it. This commit adds a hybrid install path to the Dockerfile and the matching wiring in docker-build.yml: - Dockerfile attempts pip install first. On "No matching distribution found for gramps==..." it falls back to a SHA-pinned git clone of gramps-project/gramps@maintenance/gramps${SERIES_NODOT} and "pip install ." from that working tree. Any other pip failure (network, 503, etc.) is fatal so a transient PyPI hiccup cannot silently flip a normally-released branch into "test against moving tip" mode. - docker-build.yml's params step captures the upstream branch's current HEAD SHA via git ls-remote and passes it in as GRAMPS_FALLBACK_SHA. The SHA participates in the buildx cache key so a moved upstream tip actually re-runs the install layer (without that, gramps61 CI would stay frozen on whichever revision was first baked into the image). Trade-off — and worth flagging to addon contributors: a green CI on an unreleased branch means "addons work with the current upstream tip," not "addons work with X.Y.0." The ::notice:: (PyPI path) vs ::warning:: (fallback path) log distinction in the Dockerfile makes the active path visible in every docker-build run. Verified locally: - docker build --check --build-arg GRAMPS_SERIES=6.0 → clean - docker build with GRAMPS_SERIES=6.0 + valid fallback SHA → ::notice::installed gramps==6.0.* from PyPI; image reports "Gramps 6.0.8" (released path unchanged) - docker build with GRAMPS_SERIES=6.1 + upstream HEAD SHA → fallback fires; image reports "Gramps 6.1.0-beta1" (only obtainable from git clone, since no 6.1.* exists on PyPI) - bash unit test of the failure-mode triage: missing SHA when PyPI lacks the version exits 1 with ::error::no gramps==... and GRAMPS_FALLBACK_SHA is unset; non-version pip error exits 1 with ::error::pip install gramps failed (non-version reason) and dumps the captured stderr Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/docker/gramps-ci/Dockerfile | 59 +++++++++++++++++++++++++---- .github/workflows/docker-build.yml | 23 ++++++++--- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index d979c15f6..38a4e9a28 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -16,9 +16,27 @@ # `--init` (or use a container runtime that injects tini) because xvfb-run # hangs if it inherits PID 1. # -# Local build: +# Gramps install path. PyPI is tried first; when no gramps==${SERIES}.* +# release exists (e.g. while 6.1 is still in development) the build +# falls back to a SHA-pinned git clone of +# gramps-project/gramps@maintenance/gramps${SERIES_NODOT}. Other pip +# failures (network, etc.) are NOT silently retried so a transient +# PyPI outage cannot flip a normally-released branch to "test against +# moving tip" mode. docker-build.yml captures the upstream SHA via +# git ls-remote and passes it in as GRAMPS_FALLBACK_SHA; the SHA +# becomes part of the buildx cache key so a moved upstream tip +# actually re-runs the install layer. +# +# Local build (released series): # docker build --build-arg GRAMPS_SERIES=6.0 .github/docker/gramps-ci # +# Local build (unreleased series, e.g. 6.1): +# sha=$(git ls-remote https://github.com/gramps-project/gramps.git \ +# refs/heads/maintenance/gramps61 | awk '{print $1}') +# docker build --build-arg GRAMPS_SERIES=6.1 \ +# --build-arg GRAMPS_FALLBACK_SHA=$sha \ +# .github/docker/gramps-ci +# ARG PYTHON_VERSION=3.12 FROM python:${PYTHON_VERSION}-slim @@ -27,6 +45,11 @@ FROM python:${PYTHON_VERSION}-slim # for the wrong Gramps series. docker-build.yml derives this from the # branch ref. ARG GRAMPS_SERIES +# Commit SHA on gramps-project/gramps@maintenance/grampsNN. Only used +# when no gramps==${GRAMPS_SERIES}.* release exists on PyPI. Ignored +# otherwise. Empty default is intentional — on released branches the +# fallback never fires. +ARG GRAMPS_FALLBACK_SHA="" RUN [ -n "$GRAMPS_SERIES" ] || { echo "GRAMPS_SERIES is required (e.g. 6.0)"; exit 1; } LABEL org.opencontainers.image.source="https://github.com/gramps-project/addons-source" @@ -56,12 +79,34 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # requires_mod)" step pip-installs them at CI runtime from every # .gpr.py's requires_mod list, matching what Gramps' Addon Manager does # for an end user. Keeps .gpr.py the single source of truth. -RUN pip install --no-cache-dir \ - PyGObject \ - pycairo \ - "gramps==${GRAMPS_SERIES}.*" \ - orjson \ - ruff +RUN pip install --no-cache-dir PyGObject pycairo orjson ruff + +# Install gramps: PyPI first, SHA-pinned git clone as fallback. +RUN /bin/bash -eo pipefail <<'BASH' +suffix_nodot="${GRAMPS_SERIES//./}" +if pip install --no-cache-dir "gramps==${GRAMPS_SERIES}.*" 2>/tmp/pip.err; then + echo "::notice::installed gramps==${GRAMPS_SERIES}.* from PyPI" +elif grep -q "No matching distribution found for gramps==" /tmp/pip.err; then + if [ -z "${GRAMPS_FALLBACK_SHA}" ]; then + echo "::error::no gramps==${GRAMPS_SERIES}.* on PyPI and GRAMPS_FALLBACK_SHA is unset" + exit 1 + fi + echo "::warning::no gramps==${GRAMPS_SERIES}.* on PyPI; installing from gramps-project/gramps@maintenance/gramps${suffix_nodot} at ${GRAMPS_FALLBACK_SHA}" + mkdir -p /tmp/gramps + cd /tmp/gramps + git init -q + git remote add origin https://github.com/gramps-project/gramps.git + git fetch --depth 1 origin "${GRAMPS_FALLBACK_SHA}" + git checkout FETCH_HEAD + pip install --no-cache-dir . + cd / + rm -rf /tmp/gramps +else + echo "::error::pip install gramps failed (non-version reason); aborting rather than silently switching to git tip" + cat /tmp/pip.err + exit 1 +fi +BASH RUN apt-get purge -y gcc python3-dev pkg-config && apt-get autoremove -y diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index b466bc142..6ebd19ed5 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -35,9 +35,16 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Compute branch parameters - # Derive the image-tag suffix and Gramps minor series from the - # branch ref. Same validation as ci.yml's setup job: anything - # outside maintenance/grampsNN fails fast. + # Derive the image-tag suffix, Gramps minor series, and upstream + # fallback SHA from the branch ref. Same validation as ci.yml's + # setup job: anything outside maintenance/grampsNN fails fast. + # The fallback SHA is the current tip of gramps-project/gramps + # at the matching maintenance branch; the Dockerfile only uses + # it when no gramps==${series}.* release exists on PyPI. The + # SHA is part of the buildx cache key so a moved upstream tip + # actually re-runs the install layer (otherwise gramps61 CI + # would stay on the same stale gramps revision build after + # build). id: params shell: bash run: | @@ -49,8 +56,13 @@ jobs: esac # gramps60 → 6.0, gramps61 → 6.1, gramps62 → 6.2, … series="${suffix:6:1}.${suffix:7}" - echo "suffix=$suffix" >> "$GITHUB_OUTPUT" - echo "series=$series" >> "$GITHUB_OUTPUT" + fallback_sha=$(git ls-remote https://github.com/gramps-project/gramps.git "refs/heads/maintenance/${suffix}" | awk '{print $1}') + if [ -z "$fallback_sha" ]; then + echo "::warning::upstream gramps-project/gramps has no maintenance/${suffix} branch; fallback path will fail if PyPI lacks gramps==${series}.*" + fi + echo "suffix=$suffix" >> "$GITHUB_OUTPUT" + echo "series=$series" >> "$GITHUB_OUTPUT" + echo "fallback_sha=$fallback_sha" >> "$GITHUB_OUTPUT" - name: Docker metadata id: meta @@ -70,5 +82,6 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GRAMPS_SERIES=${{ steps.params.outputs.series }} + GRAMPS_FALLBACK_SHA=${{ steps.params.outputs.fallback_sha }} cache-from: type=gha cache-to: type=gha,mode=max From 73d75edf65a757c8f616f55b9856b96488b73d0c Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 18 May 2026 23:58:24 +0200 Subject: [PATCH 16/47] docs: note the first-push race and unreleased-branch CI semantics Two doc-only additions covering practical implications of the branch-neutral CI from the preceding commits: - .github/workflows/ci.yml header: short NOTE for maintainers explaining that the first push to a newly-created maintenance branch will see container jobs fail at "Initialize containers" because the gramps-ci: image is still being built by the companion docker-build.yml workflow on the same push. Solution is a one-time re-run after docker-build finishes. This is inherent to running both workflows on the same push event and is not a problem on subsequent pushes (the image already exists). - CONTRIBUTING.md "Work Towards a Merge": short paragraph for contributors explaining what a green CI check means on an unreleased maintenance branch. When PyPI lacks a release for the matching Gramps series, the CI image is built from a SHA-pinned snapshot of gramps-project/gramps@maintenance/grampsNN, so green means "addons work with the upstream branch tip at ", not "addons work with the released X.Y.0". The exact SHA is logged as a ::warning:: line in the Build Docker Images output. Verified: - YAML still parses Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/workflows/ci.yml | 15 +++++++++++++++ CONTRIBUTING.md | 12 ++++++++++++ 2 files changed, 27 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28a9213d5..4ebd61d3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,20 @@ name: CI +# Branch-neutral workflow: the image tag, make.py argument, and Gramps +# series pin are all derived from the branch ref at runtime, so the same +# file runs unchanged on maintenance/gramps60, maintenance/gramps61, and +# any future maintenance/grampsNN branch. +# +# NOTE for maintainers — first push to a new maintenance branch: the +# corresponding gramps-ci: image does not exist in GHCR yet, so +# the container jobs in this workflow will fail at "Initialize containers" +# on the very first run. The companion docker-build.yml workflow fires +# on the same push and builds/pushes the image (~5 min cold). After it +# finishes, re-run the failed CI jobs (Actions tab → this run → "Re-run +# failed jobs") and they will pull the now-existing image. This race +# happens only on initial branch creation — every subsequent push to +# that branch finds the image already in GHCR. + on: push: branches: [maintenance/gramps**] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dda23171b..96a0c7c77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1094,6 +1094,18 @@ With the PR created, it's now a matter of working with the ```addons-source``` maintainers. Suggestions and corrections will be made and it may be mecessary to modify the original submission to get the addon accepted. +Your PR will also run automated CI checks (lint, compile, unit and +integration tests, plus an addon build); results appear on the PR page. +**Note:** on ```maintenance/grampsNN``` branches whose corresponding +Gramps release is not yet on PyPI (e.g. ```maintenance/gramps61``` while +6.1 is in development), the CI image is built from a SHA-pinned snapshot +of upstream ```gramps-project/gramps@maintenance/grampsNN``` rather than +a tagged release — a green check on such a PR means the addon works +against that *branch tip*, not against a released version. The exact +SHA used is logged as a ```::warning::``` line in the ```Build Docker +Images``` workflow output. + + The key thing is to monitor progress and comments. Your PR will have an ID number -- 1234, for example -- so you can always go to the github web page for it: From 3b2a947be34c3d9ad1b8b526d1fb122e1fc5173e Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 19 May 2026 00:07:26 +0200 Subject: [PATCH 17/47] docs: add CI maintainer runbook (.github/CI-MAINTAINER.md) Operational document for the gramps-project/addons-source maintainer covering the new steps introduced by PR 820 + the branch-neutral follow-up: - One-time setup: make the gramps-ci GHCR package public (so fork PR contributors can pull the image), and expect the first-push race on maintenance/gramps60 immediately after merge. - Creating a new maintenance branch: a plain `git branch && git push` plus a one-time re-run of the failed CI jobs after the image is built. - When a Gramps minor release lands on PyPI: nothing to do (the hybrid Dockerfile auto-detects); optional workflow_dispatch to rebuild immediately. - Diagnostic log markers: ::notice:: / ::warning:: / ::error:: annotations emitted from docker-build.yml and the Dockerfile, with their respective root causes. - Future-proofing knobs: upstream repo URL and GHCR tag retention, only mentioned in case they ever become relevant. The ci.yml header NOTE about the first-push race now points to this runbook for the full picture (GHCR visibility, log markers, etc.). Verified: - YAML parses - All TOC anchors in the new file resolve to existing headers - ../MAINTAINERS.md and ../CONTRIBUTING.md (relative links in the new file) both exist - CONTRIBUTING.md#work-towards-a-merge anchor exists Co-Authored-By: Claude Opus 4.7 (1M context) --- .github/CI-MAINTAINER.md | 145 +++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 3 + 2 files changed, 148 insertions(+) create mode 100644 .github/CI-MAINTAINER.md diff --git a/.github/CI-MAINTAINER.md b/.github/CI-MAINTAINER.md new file mode 100644 index 000000000..69324a203 --- /dev/null +++ b/.github/CI-MAINTAINER.md @@ -0,0 +1,145 @@ +# CI Maintainer Notes + +Operational steps that a `gramps-project/addons-source` maintainer +needs to handle once PR 820 (and the branch-neutral follow-up) lands. +The pipeline is otherwise self-driving — this document is only about +the rough edges. + +For day-to-day addon-release maintenance see [MAINTAINERS.md](../MAINTAINERS.md); +for the contributor-facing summary of what a green CI check means on +an unreleased branch see [CONTRIBUTING.md](../CONTRIBUTING.md#work-towards-a-merge). + +## Contents + +1. [One-time setup when the PR first merges](#one-time-setup-when-the-pr-first-merges) +2. [Creating a new maintenance branch](#creating-a-new-maintenance-branch) +3. [When a Gramps minor release lands on PyPI](#when-a-gramps-minor-release-lands-on-pypi) +4. [Diagnostic log markers](#diagnostic-log-markers) +5. [Optional future-proofing knobs](#optional-future-proofing-knobs) + +## One-time setup when the PR first merges + +### 1. Make the `gramps-ci` GHCR package public + +`docker-build.yml` pushes images to +`ghcr.io/gramps-project/addons-source/gramps-ci:` using the +workflow's `GITHUB_TOKEN`. GHCR creates the package as **private** the +first time. Same-repo CI keeps working (token covers own packages), +but **fork PRs cannot pull the image** because their `GITHUB_TOKEN` +has no read access to private packages in `gramps-project`. Fork-PR +container jobs would fail at "Initialize containers" with an +authentication error. + +Fix once, immediately after the first `Build Docker Images` run +finishes: + +1. Go to +2. Under "Danger Zone" → "Change visibility" → set to **Public** + +Every existing and future `gramps-ci:` tag inherits public +visibility from this single setting. + +### 2. Expect the first-push race on `maintenance/gramps60` + +The first push event after merge fires both workflows in parallel: + +- `Build Docker Images` builds and pushes `gramps-ci:gramps60` + (~5 min cold). +- `CI` runs `setup` (~2 s), then its container jobs try to pull the + image. + +Because both start on the same push, the CI container jobs race the +image push and may fail at "Initialize containers" the first time. +This race only happens once per branch: + +1. Wait for `Build Docker Images` to complete. +2. Open the failed CI run → click "Re-run failed jobs". +3. Subsequent pushes find the image already in GHCR — no race. + +This is also explained in the header comment of `ci.yml`. + +## Creating a new maintenance branch + +When a new Gramps minor series goes into development and addons need +a corresponding branch (e.g. `maintenance/gramps62` once 6.2 opens): + +``` +git branch maintenance/gramps62 maintenance/gramps61 +git push origin maintenance/gramps62 +``` + +No workflow edits required. The workflows derive everything from +`github.ref_name`. The first push fires the same race described +above — re-run the failed CI jobs once `Build Docker Images` +finishes. + +The setup job's regex (`gramps[0-9][0-9]`) requires a two-digit +suffix. When Gramps 10.0 opens this regex needs updating in two +places (`ci.yml` setup job and `docker-build.yml` params step). + +## When a Gramps minor release lands on PyPI + +The hybrid Dockerfile auto-detects PyPI availability: + +- `pip install "gramps==X.Y.*"` succeeds → image installs the tagged + PyPI release (`::notice::` log line). +- pip reports "No matching distribution found" → image falls back to a + SHA-pinned `git clone` of `gramps-project/gramps@maintenance/grampsNN` + at the SHA captured by `docker-build.yml`'s params step + (`::warning::` log line). + +When `gramps==6.1.0` is finally published to PyPI, the next image +rebuild on `maintenance/gramps61` silently switches from "git tip" to +"PyPI release" — no maintainer action needed. + +To **immediately** rebuild against the new PyPI release without +waiting for the next push: + +1. Open the `Build Docker Images` workflow in the Actions tab. +2. "Run workflow" → select `maintenance/grampsNN` → Run. + +The `::notice::` line in the build log confirms the switch. + +## Diagnostic log markers + +When investigating a CI failure, the install-step output in +`Build Docker Images` carries these annotations: + +| Annotation | Meaning | +| --- | --- | +| `::notice::installed gramps==X.Y.* from PyPI` | Released-path install. CI is testing against a tagged release. | +| `::warning::no gramps==X.Y.* on PyPI; installing from gramps-project/gramps@maintenance/grampsNN at ` | Unreleased-branch fallback. CI is testing against the upstream branch tip at ``. Visible to contributors via the CONTRIBUTING.md note. | +| `::error::no gramps==X.Y.* on PyPI and GRAMPS_FALLBACK_SHA is unset` | `git ls-remote` in `docker-build.yml`'s params step returned no SHA for `maintenance/grampsNN` on `gramps-project/gramps`. The matching upstream branch is missing, or the addons-source branch is misnamed. | +| `::error::pip install gramps failed (non-version reason)` | pip failed for a network/registry reason, not because the version is missing. Captured stderr is dumped after the line. The build does **not** fall back to git in this case — by design, so a transient PyPI hiccup cannot silently flip a released branch into "git tip" mode. | + +Other useful entry points: + +- `ci.yml` setup job log shows the derived `branch_suffix` and + `ci_image`. A failure here means the branch name doesn't match + `maintenance/gramps[0-9][0-9]`. +- `docker-build.yml` params step log shows the captured + `fallback_sha`. An empty value means upstream `gramps-project/gramps` + has no matching maintenance branch (warning issued; image build will + fail iff the fallback is needed). + +## Optional future-proofing knobs + +Not needed today; record here in case they ever come up. + +### Upstream gramps repo URL + +The Dockerfile hardcodes `https://github.com/gramps-project/gramps.git` +as the fallback source. If the project ever reorgs or renames, this +URL must change in one place (`.github/docker/gramps-ci/Dockerfile`). +It could be parameterised as a build arg (`ARG GRAMPS_UPSTREAM_REPO`) +with the current URL as default, but a hardcoded value keeps the +Dockerfile simpler and a rename is a sufficiently large event that +editing one string is not the bottleneck. + +### GHCR tag retention + +`docker-build.yml` pushes both a moving `gramps-ci:` tag (e.g. +`gramps60`) and a per-commit `gramps-ci:-` tag. +The moving tag is always overwritten; the SHA tags accumulate over +time. Set a retention policy on the GHCR package settings page if +the count becomes inconvenient — the moving tags are what CI consumes. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ebd61d3b..8ad018e0d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,9 @@ name: CI # failed jobs") and they will pull the now-existing image. This race # happens only on initial branch creation — every subsequent push to # that branch finds the image already in GHCR. +# +# Full operational runbook (GHCR visibility, PyPI-release transitions, +# diagnostic log markers, etc.): .github/CI-MAINTAINER.md on: push: From ad2815cee357543f5f546bf1f39ce88515de3b68 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sun, 24 May 2026 12:21:57 +0200 Subject: [PATCH 18/47] tests: detect addons that import a sibling without depends_on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds tests/test_addon_dependencies.py, an independent detector for the class of bug behind Mantis 13707 (WebConnect packs importing libwebconnect without declaring depends_on=["libwebconnect"], so install fails when libwebconnect is absent). Picked up automatically by the integration-test job's `python3 -m unittest discover -s tests -p "test_*.py"` — no ci.yml change. Design - Independent of Gramps' loader. Does NOT import gramps.gen.plug — no PluginRegister, no BasePluginManager, no PluginData. Using Gramps' loader would test addons through the very dependency resolver whose leniency let 13707 ship, and would tie this test to Gramps' internal, unstable API. - Reads every *.gpr.py via an exec-shim of its own: a permissive globals dict (LOAD_GLOBAL → __missing__ → sentinel, so plugin-type constants like GRAMPLET / REPORT / TOOL never NameError) plus a fake register() that records each call's kwargs. Builds the addon-provided-module set and an id → {modules, depends_on, requires_mod, dir} map. - Isolated-load each plugin's registered module in a fresh subprocess with sys.path = [target_dir] + dep_dirs (declared depends_on ids resolved → their directories via the index). Uses `python3 -I`, cwd=/tmp, PYTHONPATH stripped, and strips "" from sys.path defensively — without those, PEP 420 implicit namespace packages let sibling addons import as empty packages and silently defeat the isolation. Gramps remains reachable from system site-packages, as intended: the isolation is addon-from-addon, not addon-from-Gramps. - Pins GI namespace versions Gramps itself pins (Gtk 3.0, PangoCairo 1.0, etc.) before importing the addon. Not Gramps' loader — just matching the runtime conditions a plugin is loaded under, so addons whose top-level `from gi.repository import X` is version-sensitive don't generate false (c) failures from ambiguous GI defaults. - Classifies failures: (a) missing name is an addon-provided module not in this addon's depends_on — FINDING, fails the test; (b) missing name is in this addon's requires_mod — environment concern (PR 820's auto-derive owns it), ignored; (c) anything else — logged separately, not a finding. A given stderr may name multiple missing modules; (a) wins over (b) wins over (c) so the highest-signal finding is reported. Limitation, stated in the module docstring: catches undeclared dependencies that manifest at LOAD time (top-level imports). MISSES lazily-imported deps — a sibling addon imported inside a function not called at module load. No false positives, not exhaustive. Verified - Synthetic positive/negative: with depends_on=["libwebconnect"] stripped from a USWebConnectPack copy, detector flags bucket (a) `libwebconnect`; with the declaration left in, rc=0 clean load. (Both checks live under /tmp/, not committed; they validate the classifier and isolation end-to-end.) - Full tree run in gramps-ci-local:gramps60-test (PR 820's CI image, Gramps 6.0.8): 187 plugins indexed, 159 pass, 0 (a), 7 (b) ignored, 21 (c) logged. Every WebConnect pack on this branch already declares depends_on=["libwebconnect"] — the 13707 declaration is in effect on gramps60, so the test is green here. On gramps61 the situation may differ and the remediation round will be informed by re-running this detector there. This commit is the detector + the regression check. It does NOT apply any depends_on fixes — the WebConnect remediation is a separate later round on maintenance/gramps61. Issue #13707. Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_addon_dependencies.py | 421 +++++++++++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 tests/test_addon_dependencies.py diff --git a/tests/test_addon_dependencies.py b/tests/test_addon_dependencies.py new file mode 100644 index 000000000..6baf2a876 --- /dev/null +++ b/tests/test_addon_dependencies.py @@ -0,0 +1,421 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Detect addons that USE another addon without declaring it in ``depends_on``. + +This is the bug class behind Mantis 13707: the WebConnect packs imported +``libwebconnect`` at module load without declaring it, so installing a +pack without libwebconnect already present failed. + +The detector is INDEPENDENT of Gramps' plugin loader by design: + +* it reads ``.gpr.py`` with an exec-shim of its own (no + ``gramps.gen.plug._pluginreg`` import, no PluginManager, no + PluginRegister), because using Gramps' loader would test addons + through the very dependency resolver whose leniency lets the bug + ship — and would tie the test to Gramps' internal, unstable API; +* it loads each registered module in a fresh subprocess with + ``sys.path`` scoped to that addon's directory plus the directories + of its declared ``depends_on`` (so a real missing dep blows up the + way it would on a clean install), and parses the resulting + exception to classify it. + +The Gramps runtime is allowed to be importable from the subprocess — +addons do ``from gi.repository import Gtk`` and ``from gramps.gen.lib +import X`` at load time. The isolation being enforced is addon-from- +addon, not addon-from-Gramps. + +LIMITATION (important): +This catches undeclared addon dependencies that manifest at LOAD time +(top-level imports). It MISSES lazily-imported deps — e.g. a sibling +addon imported inside a function that is not called at module load. +No false positives, but not exhaustive — do not let it be mistaken +for one. + +Failures are bucketed: + +a. ``undeclared_addon_dep`` — the import error names a module that + another addon in this tree provides AND that this addon does not + declare in ``depends_on``. This is a FINDING and fails the test. +b. ``missing_requires_mod`` — the import error names a module the + addon declares in ``requires_mod`` (e.g. ``litellm``). That is an + environment concern, not a dependency-declaration bug; ignored. +c. ``other`` — any other isolated-load failure. Logged so the + information is not lost, but NOT a finding and NOT a test failure. + Examples: GI namespace mismatches, host-environment library issues, + addon import-time side effects requiring full GUI state. +""" + +# ------------------------ +# Python modules +# ------------------------ +import logging +import os +import re +import subprocess +import sys +import textwrap +import unittest +from typing import Any + +LOG = logging.getLogger(__name__) + +ADDONS_ROOT: str = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# ------------------------ +# .gpr.py exec shim +# ------------------------ +class _GprSentinel: + """Stands in for any unresolved plugin-type/category constant. + + The shim does not need real enum values — only kwargs captured by + a fake ``register()`` call. Attribute access, calls, and arithmetic + on the sentinel all return the sentinel so common patterns inside + ``.gpr.py`` files do not raise. + """ + + def __repr__(self) -> str: + return "" + + def __getattr__(self, name: str) -> "_GprSentinel": + return self + + def __call__(self, *args: Any, **kwargs: Any) -> "_GprSentinel": + return self + + +_SENT = _GprSentinel() + + +class _PermissiveGlobals(dict): + """Globals dict that returns a sentinel for any unknown plain name. + + CPython's ``LOAD_GLOBAL`` opcode calls ``__getitem__`` (and hence + ``__missing__``) on ``dict`` subclasses, so unresolved plugin-type + constants (``GRAMPLET``, ``REPORT``, ``CATEGORY_TEXT``, …) do not + raise ``NameError`` inside the exec. Dunder names still miss so + Python's own machinery behaves normally. + """ + + def __missing__(self, key: str) -> Any: + if key.startswith("__"): + raise KeyError(key) + return _SENT + + +def _exec_gpr(gpr_path: str) -> list[dict]: + """Exec one ``.gpr.py`` with a fake ``register()`` and return its kwargs. + + :param gpr_path: Absolute path to a ``*.gpr.py`` file. + :returns: One dict per ``register()`` call, with the kwargs verbatim + plus ``_ptype`` for the positional plugin type. + """ + plugins: list[dict] = [] + + def register(ptype: Any, **kwargs: Any) -> None: + kwargs["_ptype"] = ptype + plugins.append(kwargs) + + env = _PermissiveGlobals( + { + "__builtins__": __builtins__, + "__file__": gpr_path, + "__name__": "_gpr_shim", + "register": register, + "_": lambda s, *a, **k: s, + } + ) + with open(gpr_path, "r", encoding="utf-8") as f: + src = f.read() + exec(compile(src, gpr_path, "exec"), env) + return plugins + + +def _index_addons( + addons_root: str, +) -> tuple[dict[str, dict], set[str], list[tuple[str, str]]]: + """Walk ``addons_root`` and build the addon metadata index. + + :returns: ``(id_to_addon, all_modules, skipped)`` where + ``id_to_addon[plugin_id]`` is + ``{directory, modules, depends_on, requires_mod, gpr_files}``, + ``all_modules`` is the set of registered module names across + the whole tree, and ``skipped`` is a list of + ``(gpr_path, error)`` tuples for files whose exec raised. + """ + id_to_addon: dict[str, dict] = {} + all_modules: set[str] = set() + skipped: list[tuple[str, str]] = [] + + for dirname in sorted(os.listdir(addons_root)): + addon_dir = os.path.join(addons_root, dirname) + if not os.path.isdir(addon_dir): + continue + if dirname.startswith("."): + continue + gpr_files = sorted( + os.path.join(addon_dir, f) + for f in os.listdir(addon_dir) + if f.endswith(".gpr.py") + ) + if not gpr_files: + continue + for gpr in gpr_files: + try: + plugins = _exec_gpr(gpr) + except BaseException as exc: # noqa: BLE001 + skipped.append((gpr, f"{type(exc).__name__}: {exc}")) + continue + for plugin in plugins: + pid = plugin.get("id") + fname = plugin.get("fname") + if not isinstance(pid, str) or not isinstance(fname, str): + continue + module = re.sub(r"\.py$", "", fname) + rec = id_to_addon.setdefault( + pid, + { + "directory": addon_dir, + "modules": [], + "depends_on": [], + "requires_mod": [], + "gpr_files": [], + }, + ) + if module not in rec["modules"]: + rec["modules"].append(module) + for dep in plugin.get("depends_on") or []: + if isinstance(dep, str) and dep not in rec["depends_on"]: + rec["depends_on"].append(dep) + for req in plugin.get("requires_mod") or []: + if isinstance(req, str) and req not in rec["requires_mod"]: + rec["requires_mod"].append(req) + if gpr not in rec["gpr_files"]: + rec["gpr_files"].append(gpr) + all_modules.add(module) + return id_to_addon, all_modules, skipped + + +# ------------------------ +# Isolated-load subprocess +# ------------------------ +_LOADER = textwrap.dedent( + """ + import sys, importlib, traceback + # Strip the implicit "" CWD entry. Without this, if the subprocess + # is run from a directory that contains addon subdirectories, PEP + # 420 implicit namespace packages let sibling addons import as + # empty packages — which silently defeats the isolation we are + # trying to enforce. The caller also chdirs to a neutral directory, + # but stripping "" makes the isolation independent of CWD. + sys.path[:] = [p for p in sys.path if p not in ("", ".")] + # Pin the GI namespace versions Gramps itself pins before loading + # any plugin, so addons whose top-level `from gi.repository import X` + # is version-sensitive do not generate false (c) failures from + # ambiguous GI defaults. This is NOT Gramps' loader; it is matching + # the runtime conditions an addon is loaded under. + try: + import gi + for ns, ver in ( + ("Gtk", "3.0"), + ("PangoCairo", "1.0"), + ("OsmGpsMap", "1.0"), + ("GExiv2", "0.10"), + ("Gspell", "1"), + ("GeocodeGlib", "1.0"), + ): + try: + gi.require_version(ns, ver) + except (ValueError, AttributeError): + pass + except ImportError: + pass + target_dir = {target_dir!r} + dep_dirs = {dep_dirs!r} + sys.path[:0] = [target_dir] + list(dep_dirs) + try: + importlib.import_module({module!r}) + except BaseException: + traceback.print_exc() + sys.exit(2) + sys.exit(0) + """ +) + + +def _isolated_load( + target_dir: str, dep_dirs: list[str], module: str, timeout: int = 30 +) -> tuple[int, str]: + """Spawn a subprocess that tries to import ``module`` in isolation. + + The subprocess is run from a neutral CWD (the system temp dir) and + with ``PYTHONPATH`` stripped from the environment, so neither the + parent's working directory nor a stray ``PYTHONPATH`` can leak + sibling-addon paths into the child's ``sys.path``. + + :returns: ``(returncode, stderr)``. Returncode ``-1`` indicates + the subprocess timed out. + """ + code = _LOADER.format(target_dir=target_dir, dep_dirs=dep_dirs, module=module) + env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} + try: + proc = subprocess.run( + [sys.executable, "-I", "-c", code], + capture_output=True, + text=True, + timeout=timeout, + cwd="/tmp", + env=env, + ) + except subprocess.TimeoutExpired: + return -1, "TIMEOUT" + return proc.returncode, proc.stderr + + +_MISSING_NAME_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") + + +def _classify( + stderr: str, + declared_dep_modules: set[str], + requires_mod: set[str], + all_addon_modules: set[str], +) -> tuple[str, str]: + """Bucket a failed isolated-load. + + A given failure may name multiple missing modules. Bucket (a) wins + over (b) wins over (c), so the highest-signal finding is reported. + """ + missing = [m.split(".")[0] for m in _MISSING_NAME_RE.findall(stderr)] + for name in missing: + if name in all_addon_modules and name not in declared_dep_modules: + return ("a_undeclared_addon_dep", name) + for name in missing: + if name in requires_mod: + return ("b_requires_mod", name) + last_lines = stderr.strip().split("\n")[-5:] + return ("c_other", last_lines[-1] if last_lines else "") + + +# ------------------------------------------------------------ +# +# TestAddonDependencies +# +# ------------------------------------------------------------ +class TestAddonDependencies(unittest.TestCase): + """Fail when any addon imports a sibling addon it does not declare.""" + + id_to_addon: dict[str, dict] = {} + all_modules: set[str] = set() + skipped_gpr: list[tuple[str, str]] = [] + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.id_to_addon, cls.all_modules, cls.skipped_gpr = _index_addons(ADDONS_ROOT) + + def test_no_undeclared_addon_dependencies(self) -> None: + """Every addon's registered modules must import in isolation, given + only the directories of its declared ``depends_on``. + + A failure that names a sibling-addon module not listed in + ``depends_on`` is a finding (the #13707 class). A failure that + names a declared ``requires_mod`` is the environment, not the + declaration. Anything else is logged separately. + """ + self.assertGreater( + len(self.id_to_addon), 0, "No addons found — index is empty" + ) + + findings_a: list[str] = [] + findings_c: list[str] = [] + counts = {"pass": 0, "a": 0, "b": 0, "c": 0} + + for pid in sorted(self.id_to_addon): + rec = self.id_to_addon[pid] + target_dir = rec["directory"] + dep_dirs: list[str] = [] + declared_modules: set[str] = set() + for dep_id in rec["depends_on"]: + dep_rec = self.id_to_addon.get(dep_id) + if dep_rec is None: + continue + if dep_rec["directory"] not in dep_dirs: + dep_dirs.append(dep_rec["directory"]) + declared_modules.update(dep_rec["modules"]) + requires_mod_set = set(rec["requires_mod"]) + + for module in rec["modules"]: + rc, err = _isolated_load(target_dir, dep_dirs, module) + if rc == 0: + counts["pass"] += 1 + continue + bucket, detail = _classify( + err, declared_modules, requires_mod_set, self.all_modules + ) + if bucket == "a_undeclared_addon_dep": + counts["a"] += 1 + findings_a.append( + f" {pid} (module {module}) — undeclared addon dep: " + f"{detail}" + ) + elif bucket == "b_requires_mod": + counts["b"] += 1 + else: + counts["c"] += 1 + findings_c.append(f" {pid} (module {module}) — {detail}") + + LOG.info( + "Indexed %d plugins; load pass=%d, bucket a=%d, b=%d, c=%d; " + "gpr exec skipped=%d", + len(self.id_to_addon), + counts["pass"], + counts["a"], + counts["b"], + counts["c"], + len(self.skipped_gpr), + ) + if self.skipped_gpr: + LOG.warning( + "%d .gpr.py file(s) failed exec-shim and were skipped:\n%s", + len(self.skipped_gpr), + "\n".join(f" {p}: {e}" for p, e in self.skipped_gpr), + ) + if findings_c: + LOG.warning( + "%d addon(s) failed isolated load for non-dependency reasons " + "(NOT a finding — environment / GUI-state / import-time side " + "effects):\n%s", + len(findings_c), + "\n".join(findings_c), + ) + + if findings_a: + self.fail( + "Found %d addon(s) that import a sibling addon without " + "declaring it in depends_on:\n%s" + % (len(findings_a), "\n".join(findings_a)) + ) + + +if __name__ == "__main__": + unittest.main() From bc8df21637c5b2394b023cbb358cb8a7778d0f2f Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sun, 24 May 2026 12:51:02 +0200 Subject: [PATCH 19/47] tests: derive expected addon target version from Gramps install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `test_target_version_is_6_0` hardcoded "6.0" in the assertion and its name, so the same harness copied to maintenance/gramps61 (via the bootstrap commit 458ebd0d5) flagged every 6.1-targeted addon as mistargeted. Switch the prefix to ``f"{VERSION_TUPLE[0]}.{VERSION_TUPLE[1]}"``, read from the loaded ``gramps.version``, so the same test works on every maintenance branch — the CI image's Gramps install is always the series that branch's addons should target, which is exactly what we want to assert. Rename the method to ``test_target_version_matches_gramps_install`` to match the new semantics, and update the failure message to include the expected prefix instead of a fixed "6.0". Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/test_plugin_registration.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_plugin_registration.py b/tests/test_plugin_registration.py index dae8d6f87..b2f2f4dde 100644 --- a/tests/test_plugin_registration.py +++ b/tests/test_plugin_registration.py @@ -45,6 +45,7 @@ # Gramps modules # ------------------------ from gramps.gen.plug._pluginreg import EXPORT, GRAMPLET, IMPORT, REPORT, TOOL +from gramps.version import VERSION_TUPLE # ------------------------ # Gramps specific @@ -127,14 +128,24 @@ def test_all_plugins_have_valid_metadata(self) -> None: self.assertTrue(pdata.name, f"Plugin {pdata.id} missing name") self.assertTrue(pdata.version, f"Plugin {pdata.id} missing version") - def test_target_version_is_6_0(self) -> None: - """All listed addons on this branch should target Gramps 6.0.""" + def test_target_version_matches_gramps_install(self) -> None: + """All listed addons must target the Gramps series they're running against. + + The expected prefix is derived from the installed Gramps' version + (``gramps.version.VERSION_TUPLE``), so the same assertion works on + every maintenance branch — gramps60 expects "6.0", gramps61 expects + "6.1", etc. + """ + expected_prefix = f"{VERSION_TUPLE[0]}.{VERSION_TUPLE[1]}" issues: list[str] = [] for pdata in _get_addon_plugins(self.plugin_registry): - if not pdata.gramps_target_version.startswith("6.0"): + if not pdata.gramps_target_version.startswith(expected_prefix): issues.append(f"{pdata.id}: targets {pdata.gramps_target_version}") if issues: - self.fail("Addons not targeting Gramps 6.0:\n" + "\n".join(issues)) + self.fail( + f"Addons not targeting Gramps {expected_prefix}:\n" + + "\n".join(issues) + ) # ------------------------------------------------------------ From 7f8a83918616d27761916f80f03163745c173750 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Fri, 29 May 2026 20:13:04 +0200 Subject: [PATCH 20/47] ci: install addon system deps, run under xvfb, pin GI, fail on silent skips MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI ran per-addon tests but quietly no-op'd for any GUI/graphviz addon: the image shipped no goocanvas/osm-gps-map/graphviz, tests were never run under xvfb, nothing pinned the GTK version before gramps.gui imports, and an all-skipped module still exited 0. Close all four gaps. System deps (requires_gi / requires_exe): - .github/scripts/addon_system_deps.py is the single source mapping each declared GI typelib / executable to its per-platform package (apt and conda), and scans the .gpr.py files. The Linux jobs derive the apt set at runtime and install it (the container runs as root; the image build context excludes addons-source so it cannot bake them). A drift-guard step fails if an addon declares a dep with no map entry — the system-dep analogue of the existing requires_mod find_spec gate. - The GTK 3 addon libs (goocanvas, osm-gps-map, gexiv2) are not on conda-forge, so the map records conda=None for them; the Windows lane installs only the available subset (graphviz) and those addons skip there by platform necessity. Display + GI bootstrap: - Per-addon test runs are wrapped in xvfb-run (addons that build a Gtk style context at import need a display, else a hard Gtk-ERROR). - .github/scripts/run_addon_tests.py and a gi_bootstrap sitecustomize pin Pango/PangoCairo/Gtk the way gramps/gui/grampsgui.py does, so a test importing a gramps.gui.* module loads GTK 3 instead of warning / risking GTK 4. The sitecustomize covers the subprocess-loading plugin registration test via PYTHONPATH. Honest skip accounting: - run_addon_tests.py replaces bare `unittest` for the per-addon runs and FAILS a wholly-skipped module, UNLESS the addon's declared system deps are unavailable on the platform (e.g. goocanvas on conda), in which case the skip is expected and tolerated. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/addon_system_deps.py | 209 +++++++++++++++++ .github/scripts/gi_bootstrap/sitecustomize.py | 29 +++ .github/scripts/run_addon_tests.py | 210 ++++++++++++++++++ .github/workflows/ci.yml | 91 +++++++- 4 files changed, 534 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/addon_system_deps.py create mode 100644 .github/scripts/gi_bootstrap/sitecustomize.py create mode 100644 .github/scripts/run_addon_tests.py diff --git a/.github/scripts/addon_system_deps.py b/.github/scripts/addon_system_deps.py new file mode 100644 index 000000000..6e3cef514 --- /dev/null +++ b/.github/scripts/addon_system_deps.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Single source of truth for addon *system* dependencies in CI. + +Addons declare three dependency kinds in their ``.gpr.py``: + +* ``requires_mod`` — importable Python modules. pip-installable; ci.yml already + auto-derives these from the ``.gpr.py`` files. Nothing to do here. +* ``requires_gi`` — GObject-introspection typelibs (e.g. ``GooCanvas``). +* ``requires_exe`` — system executables (e.g. ``dot`` from graphviz). + +The latter two are *system* packages: not pip-installable, named differently per +platform, and Gramps' own ``Requirements`` only *checks* them (never installs). +This module maps each declared ``requires_gi`` namespace / ``requires_exe`` name +to its package on each CI platform and scans the addons for what they declare, +so ci.yml derives the install list from one place instead of a hand-kept list. + +Platform availability is asymmetric and encoded here: the GTK 3 addon libs +(goocanvas, osm-gps-map, gexiv2) exist on Debian/apt but **not on conda-forge**, +so the conda (Windows) lane cannot install them — addons needing them skip there +by necessity. A ``conda`` value of ``None`` records that. + +Pure stdlib so it runs anywhere in CI without bootstrapping. + +CLI:: + + addon_system_deps.py --platform apt # space-separated install list + addon_system_deps.py --platform conda # (only packages available there) + addon_system_deps.py --unmapped . # declared deps with no map entry; exit 1 if any +""" + +# ------------------------ +# Python modules +# ------------------------ +from __future__ import annotations + +import argparse +import ast +import glob +import os +import re +import sys + +# --------------------------------------------------------------------------- +# The map. Keys are what addons declare; values give the package per platform. +# A None value means "no package provides this on that platform" (so it is not +# installed there and an addon needing it is expected to skip). +# --------------------------------------------------------------------------- + +# requires_gi namespace -> package providing the typelib, per platform. +GI_PACKAGES: dict[str, dict[str, str | None]] = { + "GExiv2": {"apt": "gir1.2-gexiv2-0.10", "conda": None}, + "GooCanvas": {"apt": "gir1.2-goocanvas-2.0", "conda": None}, + "OsmGpsMap": {"apt": "gir1.2-osmgpsmap-1.0", "conda": None}, + # PlaceCoordinateGramplet declares GeocodeGlib 1.0, but modern distros ship + # only the 2.0 typelib and conda-forge ships none; the addon has no tests. + # Recorded so the drift-guard recognises the namespace; not installed. + "GeocodeGlib": {"apt": None, "conda": None}, +} + +# requires_exe executable -> package providing it, per platform. +EXE_PACKAGES: dict[str, dict[str, str | None]] = { + "dot": {"apt": "graphviz", "conda": "graphviz"}, +} + +PLATFORMS = ("apt", "conda") + + +# ------------------------------------------------------------ +# +# scanning +# +# ------------------------------------------------------------ +_GI_RE = re.compile(r"requires_gi\s*=\s*(\[[^\]]*\])") +_EXE_RE = re.compile(r"requires_exe\s*=\s*(\[[^\]]*\])") + + +def _gpr_files(root: str) -> list[str]: + return sorted(glob.glob(os.path.join(root, "*", "*.gpr.py"))) + + +def _literal(src: str): + try: + return ast.literal_eval(src) + except (ValueError, SyntaxError): + return [] + + +def _scan(root: str, pattern: re.Pattern, first_of_tuple: bool) -> set[str]: + found: set[str] = set() + for path in _gpr_files(root): + try: + text = open(path, encoding="utf-8").read() + except OSError: + continue + for match in pattern.finditer(text): + for entry in _literal(match.group(1)): + if first_of_tuple and isinstance(entry, (tuple, list)): + entry = entry[0] if entry else None + if entry: + found.add(entry) + return found + + +def scan_gi_namespaces(root: str) -> set[str]: + return _scan(root, _GI_RE, first_of_tuple=True) + + +def scan_executables(root: str) -> set[str]: + return _scan(root, _EXE_RE, first_of_tuple=False) + + +def addon_requirements(addon_dir: str) -> tuple[set[str], set[str]]: + """Return (gi_namespaces, executables) declared by a single addon dir.""" + gi: set[str] = set() + exe: set[str] = set() + for path in sorted(glob.glob(os.path.join(addon_dir, "*.gpr.py"))): + try: + text = open(path, encoding="utf-8").read() + except OSError: + continue + for match in _GI_RE.finditer(text): + for entry in _literal(match.group(1)): + ns = entry[0] if isinstance(entry, (tuple, list)) else entry + if ns: + gi.add(ns) + for match in _EXE_RE.finditer(text): + for entry in _literal(match.group(1)): + if entry: + exe.add(entry) + return gi, exe + + +# ------------------------------------------------------------ +# +# derivation +# +# ------------------------------------------------------------ +def packages(platform: str) -> list[str]: + """All install-by-name packages available for a platform (full mapped set).""" + pkgs: list[str] = [] + for table in (GI_PACKAGES, EXE_PACKAGES): + for entry in table.values(): + pkg = entry.get(platform) + if pkg: + pkgs.append(pkg) + return sorted(set(pkgs)) + + +def unmapped(root: str) -> tuple[set[str], set[str]]: + """Declared deps with no entry in the maps at all (drift).""" + return ( + scan_gi_namespaces(root) - set(GI_PACKAGES), + scan_executables(root) - set(EXE_PACKAGES), + ) + + +def addon_satisfiable_on(addon_dir: str, platform: str) -> bool: + """ + True if every system dep the addon declares has a package on this platform. + + Used by the test runner to tell an *expected* platform skip (a declared dep + that simply is not packaged here, e.g. goocanvas on conda) from a suspicious + all-skip that should fail. + """ + gi, exe = addon_requirements(addon_dir) + for ns in gi: + entry = GI_PACKAGES.get(ns) + if entry is None or entry.get(platform) is None: + return False + for name in exe: + entry = EXE_PACKAGES.get(name) + if entry is None or entry.get(platform) is None: + return False + return True + + +# ------------------------------------------------------------ +# +# CLI +# +# ------------------------------------------------------------ +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--platform", choices=PLATFORMS) + parser.add_argument( + "--unmapped", + metavar="ROOT", + help="print declared GI/exe deps with no map entry; exit 1 if any", + ) + args = parser.parse_args(argv) + + if args.unmapped is not None: + gi, exe = unmapped(args.unmapped) + for ns in sorted(gi): + print(f"gi:{ns}") + for name in sorted(exe): + print(f"exe:{name}") + return 1 if (gi or exe) else 0 + + if args.platform: + print(" ".join(packages(args.platform))) + return 0 + + parser.error("nothing to do: pass --platform or --unmapped") + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/gi_bootstrap/sitecustomize.py b/.github/scripts/gi_bootstrap/sitecustomize.py new file mode 100644 index 000000000..9f5cb090d --- /dev/null +++ b/.github/scripts/gi_bootstrap/sitecustomize.py @@ -0,0 +1,29 @@ +"""Pin the GObject-introspection versions, like the Gramps GUI launcher. + +Put this directory on ``PYTHONPATH`` for a test step and the interpreter imports +this ``sitecustomize`` at startup — before any test (or subprocess it spawns) +imports a ``gramps.gui`` module. + +Why: gramps pins its GI versions in the GUI launcher (``gramps/gui/grampsgui.py`` +calls ``gi.require_version`` for Pango/PangoCairo/Gtk at import). A test that +imports a ``gramps.gui.*`` module directly never runs that launcher, so Gtk/Pango +get imported with no version pinned first — emitting a ``PyGIWarning`` and, on a +host where GTK 4 is the default, risking the wrong stack. This shim performs the +same bootstrap, so tests run under the supported GTK 3 stack. + +Used for the discover-based / subprocess-loading steps (e.g. plugin +registration), where the bootstrap must be inherited via ``PYTHONPATH`` by every +spawned interpreter. The in-process unit/integration runner +(``run_addon_tests.py``) does the same ``require_version`` itself. +""" + +try: + import gi + + for _ns, _ver in (("Pango", "1.0"), ("PangoCairo", "1.0"), ("Gtk", "3.0")): + try: + gi.require_version(_ns, _ver) + except (ValueError, AttributeError): + pass +except ImportError: + pass diff --git a/.github/scripts/run_addon_tests.py b/.github/scripts/run_addon_tests.py new file mode 100644 index 000000000..6491aa0e5 --- /dev/null +++ b/.github/scripts/run_addon_tests.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Run per-addon unit tests with a GI bootstrap, a timeout, and honest skips. + +Replaces a bare ``python -m unittest `` in CI. It does three things +plain unittest does not: + +1. **GI version bootstrap.** Before any test imports a ``gramps.gui`` module, it + calls ``gi.require_version`` for Pango/PangoCairo/Gtk — the set the Gramps + GUI launcher (``gramps/gui/grampsgui.py``) pins at startup. A direct test + import never runs that launcher, so without this the first + ``from gi.repository import Gtk`` (in gramps core) warns and risks the wrong + GTK on a host where GTK 4 is the default. + +2. **A per-module timeout.** Each module runs in its own subprocess with a wall + clock. A test that hangs (e.g. a DB import that blocks on a platform) would + otherwise hang the whole CI job indefinitely — neither plain unittest nor + xmlrunner has a timeout. A module that exceeds the limit is killed and + reported as a FAILURE, so the job stays bounded and names the culprit. + +3. **Honest skip accounting.** unittest exits 0 when every test SKIPS, so a + wholly-skipped module reads as a pass. This runner FAILS such a module — + UNLESS the addon's declared system deps are unavailable on this platform + (e.g. goocanvas/osm-gps-map are not on conda-forge), in which case the skip + is expected and tolerated (the map lives in ``addon_system_deps.py``). + +Usage:: + + run_addon_tests.py --platform apt Addon.tests.test_x Other.tests.test_y + run_addon_tests.py --platform conda Addon.tests.test_x + +Exit code is non-zero if any module is a hard failure (test failure/error, +timeout, or an unexpected all-skip on a platform where the addon's deps are +available). +""" + +# ------------------------ +# Python modules +# ------------------------ +from __future__ import annotations + +import argparse +import os +import subprocess +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import addon_system_deps as deps # noqa: E402 + +# Per-module wall clock. Generous enough for a legitimate DB-backed suite, small +# enough that a hung test is caught promptly instead of running to the job cap. +# Overridable via env for tuning/testing. +MODULE_TIMEOUT_S = int(os.environ.get("RUN_ADDON_TESTS_TIMEOUT", "300")) + +# Markers the worker prints so the parent can read the outcome without needing +# xmlrunner (820's env has only stdlib unittest). +_OK = "__RESULT__ ok" +_LOADERROR = "__RESULT__ loaderror" + + +def _bootstrap_gi() -> None: + """Pin the GI versions the Gramps GUI launcher pins, before tests import.""" + try: + import gi + except ImportError: + return + for namespace, version in (("Pango", "1.0"), ("PangoCairo", "1.0"), ("Gtk", "3.0")): + try: + gi.require_version(namespace, version) + except (ValueError, AttributeError): + pass + + +# ------------------------------------------------------------ +# +# worker: runs ONE module in this (sub)process +# +# ------------------------------------------------------------ +def _run_worker(modname: str) -> int: + """Run a single module; print a machine-readable result line; exit 0. + + The parent classifies pass/fail from the printed counts and its own platform + knowledge, so the worker always exits 0 (a non-zero exit would be + indistinguishable from an interpreter crash). + """ + _bootstrap_gi() + try: + suite = unittest.defaultTestLoader.loadTestsFromName(modname) + except Exception as exc: # import-time failure + print(f"{_LOADERROR} {exc!r}", flush=True) + return 0 + result = unittest.TextTestRunner(verbosity=2).run(suite) + broke = len(result.failures) + len(result.errors) + print( + f"{_OK} tests={result.testsRun} skipped={len(result.skipped)} broke={broke}", + flush=True, + ) + return 0 + + +# ------------------------------------------------------------ +# +# parent: spawns a timed worker per module and classifies the outcome +# +# ------------------------------------------------------------ +def _classify(modname: str, platform: str, root: str) -> tuple[bool, str]: + """Run one module in a timed subprocess. Return (is_hard_failure, summary).""" + addon = modname.split(".", 1)[0] + satisfiable = deps.addon_satisfiable_on(os.path.join(root, addon), platform) + + proc = subprocess.Popen( + [sys.executable, os.path.abspath(__file__), "--worker", modname], + stdout=subprocess.PIPE, + stderr=None, # stream the test output straight to the CI log + text=True, + ) + try: + # communicate() enforces the wall clock and reaps the process. + stdout, _ = proc.communicate(timeout=MODULE_TIMEOUT_S) + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + return True, f" FAIL {modname} — timed out after {MODULE_TIMEOUT_S}s (hung)" + + out_lines = stdout.splitlines() + for line in out_lines: # echo the worker's result marker into the log + print(line) + + result_line = next( + (ln for ln in reversed(out_lines) if ln.startswith("__RESULT__")), "" + ) + + if result_line.startswith(_LOADERROR): + if satisfiable: + return True, f" FAIL {modname} — load error" + return False, ( + f" skip {modname} — not loadable on {platform} " + f"(addon system deps unavailable here)" + ) + + if not result_line.startswith(_OK): + return True, f" FAIL {modname} — no result (worker crashed)" + + fields = dict(tok.split("=", 1) for tok in result_line.split()[2:] if "=" in tok) + ran = int(fields.get("tests", 0)) + skipped = int(fields.get("skipped", 0)) + broke = int(fields.get("broke", 0)) + + if broke: + return True, f" FAIL {modname} — {broke} failed/errored" + if ran > 0 and skipped == ran: + if satisfiable: + return True, ( + f" FAIL {modname} — all {ran} tests skipped " + f"(degraded coverage; deps ARE available on {platform})" + ) + return False, ( + f" skip {modname} — all {ran} skipped, expected " + f"(addon system deps unavailable on {platform})" + ) + if skipped: + return False, f" ok {modname} — {ran} tests, {skipped} skipped" + return False, f" ok {modname} — {ran} tests" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--platform", choices=deps.PLATFORMS) + parser.add_argument( + "--root", + default=".", + help="addons-source root holding the / dirs (default: cwd)", + ) + parser.add_argument( + "--worker", + metavar="MODULE", + help="internal: run this single module and print its result line", + ) + parser.add_argument("modules", nargs="*", help="dotted test modules to run") + args = parser.parse_args(argv) + + if args.worker: + return _run_worker(args.worker) + + if not args.platform: + parser.error("--platform is required in parent mode") + if not args.modules: + print("No per-addon unit test modules found") + return 0 + + hard_failures: list[str] = [] + summary: list[str] = [] + for modname in args.modules: + failed, line = _classify(modname, args.platform, args.root) + summary.append(line) + if failed: + hard_failures.append(modname) + + print("\n=== addon test summary ===") + for line in summary: + print(line) + + if hard_failures: + print(f"\n{len(hard_failures)} module(s) failed: {hard_failures}") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ad018e0d..97edbea6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,6 +198,35 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install addon system deps (derived from requires_gi / requires_exe) + # System deps (GI typelibs, executables) are not pip-installable and + # gramps only *checks* them, so the image cannot bake them generically + # (its build context excludes addons-source). Derive the apt set from + # every .gpr.py via the single-source map and install it — the container + # runs as root. Mirrors the requires_mod derivation below. + shell: bash + run: | + pkgs=$(python3 .github/scripts/addon_system_deps.py --platform apt) + if [ -n "$pkgs" ]; then + echo "→ addon system deps (apt): $pkgs" + apt-get update + apt-get install -y --no-install-recommends $pkgs + else + echo "no requires_gi / requires_exe declarations found" + fi + + - name: Validate addon system deps are mapped + # Every requires_gi / requires_exe an addon declares must have an entry + # in addon_system_deps.py, so the install list never silently drifts + # from what addons declare (the system-dep analogue of the requires_mod + # find_spec gate below). + shell: bash + run: | + python3 .github/scripts/addon_system_deps.py --unmapped . || { + echo "::error::Addon(s) declare requires_gi/requires_exe with no entry in .github/scripts/addon_system_deps.py — add a mapping row." + exit 1 + } + - name: Install addon runtime deps (derived from requires_mod) # Auto-derive the union of requires_mod across every .gpr.py in # the repo. Mirrors Gramps' Addon Manager install path @@ -332,7 +361,14 @@ jobs: done if [ -n "$modules" ]; then echo "Running unit tests:$modules" - python3 -m unittest -v $modules + # xvfb-run: some addons create a Gtk style context at import and + # need a display (else a hard Gtk-ERROR abort, not a clean skip). + # run_addon_tests.py: pins the GI versions like gramps' launcher + # (so gramps.gui imports load GTK 3, no PyGIWarning) and fails a + # wholly-skipped module unless the addon's deps are unavailable on + # this platform. + xvfb-run -a --server-args="-screen 0 1920x1080x24" \ + python3 .github/scripts/run_addon_tests.py --platform apt --root . $modules else echo "No per-addon unit test modules found" fi @@ -363,6 +399,21 @@ jobs: mamba list | head -30 python -c "import gramps, gi; print('deps OK')" + - name: Install addon system deps (derived, conda-available subset) + # Only the conda-forge-available subset (e.g. graphviz). The GTK 3 addon + # GI libs (goocanvas/osm-gps-map/gexiv2) are NOT on conda-forge, so the + # map returns None for them on conda and they are not installed; addons + # needing them skip on Windows by necessity, which run_addon_tests + # tolerates (--platform conda). + run: | + pkgs=$(python .github/scripts/addon_system_deps.py --platform conda) + if [ -n "$pkgs" ]; then + echo "→ addon system deps (conda): $pkgs" + mamba install -y -c conda-forge $pkgs + else + echo "no conda-available addon system deps to install" + fi + - name: Install addon runtime deps (derived from requires_mod) # See unit-test-linux for rationale. Uses `python` (conda-forge # env) to match the surrounding Windows job style. @@ -471,7 +522,10 @@ jobs: done if [ -n "$modules" ]; then echo "Running unit tests:$modules" - python -m unittest -v $modules + # No xvfb on Windows (GTK renders natively). run_addon_tests pins + # the GI versions and tolerates addons whose GI deps are not on + # conda-forge (they skip here by platform necessity). + python .github/scripts/run_addon_tests.py --platform conda --root . $modules else echo "No per-addon unit test modules found" fi @@ -489,6 +543,21 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Install addon system deps (derived from requires_gi / requires_exe) + # Same as unit-test-linux: derive the apt set from the single-source + # map and install it (container runs as root). The plugin registration + # test subprocess-loads each addon module, so the GI typelibs those + # modules import must be present here too. (Mapping is drift-guarded in + # unit-test-linux, which this job needs:, so no duplicate gate here.) + shell: bash + run: | + pkgs=$(python3 .github/scripts/addon_system_deps.py --platform apt) + if [ -n "$pkgs" ]; then + echo "→ addon system deps (apt): $pkgs" + apt-get update + apt-get install -y --no-install-recommends $pkgs + fi + - name: Install addon runtime deps (derived from requires_mod) # See unit-test-linux for rationale. The plugin registration test # subprocess-loads each addon's module, which imports its @@ -570,10 +639,21 @@ jobs: # shell: bash for consistency with the surrounding steps; the # current command uses no bashisms, but keeps this block safe # against future edits. Container default is /bin/sh → dash. + # + # gi_bootstrap on PYTHONPATH pins the GI versions (like the gramps GUI + # launcher) for this process AND the addon-module subprocesses this test + # spawns, so gramps.gui imports load GTK 3 without a PyGIWarning. + # + # NOT run under xvfb: this test only *loads* (imports) addon modules in + # subprocesses and tolerates load failures; it does not render. Giving it + # a display made an addon load hang on the (absent) AT-SPI accessibility + # bus until the per-load timeout. Imports that build a Gtk style context + # are exercised under xvfb in the unit/integration test runs instead. shell: bash env: - PYTHONPATH: . - run: python3 -m unittest discover -s tests -p "test_*.py" -t . -v + PYTHONPATH: .github/scripts/gi_bootstrap:. + run: | + python3 -m unittest discover -s tests -p "test_*.py" -t . -v - name: Run per-addon integration tests # shell: bash — see unit-test-linux for rationale; the @@ -603,7 +683,8 @@ jobs: done if [ -n "$modules" ]; then echo "Running per-addon integration tests:$modules" - python3 -m unittest -v $modules + xvfb-run -a --server-args="-screen 0 1920x1080x24" \ + python3 .github/scripts/run_addon_tests.py --platform apt --root . $modules else echo "No per-addon integration test modules found" fi From 1466491abf5d6c0c886f26f3d1166a1ed9a4a1a2 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sat, 30 May 2026 00:22:46 +0200 Subject: [PATCH 21/47] ci(windows): document gramps-vs-branch series caveat conda-forge has no gramps 6.1 yet, so environment.yml's "gramps<6.1" pin resolves to 6.0.x on every branch. On maintenance/gramps61 the Windows lane therefore validates addons against gramps 6.0.x, not the branch's series. Git-building the matching gramps in the conda env (as the Linux CI image does) is not viable: gramps' own Windows build targets MSYS2 UCRT64, not conda, and the wheel build fails in build_intl when `msgfmt --xml` cannot locate the shared-mime-info/appstream ITS rules absent from the conda env. Add a non-failing "Report gramps-vs-branch series" step to the Windows job that surfaces the caveat in the log on every gramps61+ run. Addon tests needing series-exact gramps behaviour skip themselves on Windows (e.g. TMGimporter's real-DB import tests) and run on the Linux lane, which git-builds the branch's exact gramps. When 6.1 reaches conda-forge the pin picks it up and the caveat disappears. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/environment.yml | 7 +++++++ .github/workflows/ci.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/.github/environment.yml b/.github/environment.yml index dd9e85603..f1c7c9ae7 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -10,6 +10,13 @@ dependencies: # installed at CI runtime by ci.yml's auto-derive step from .gpr.py # requires_mod — single source of truth. Keep only the stable base # here (Gramps + orjson for plugin registration). + # + # conda-forge has no gramps 6.1 yet, so on a maintenance/gramps61 (or later) + # branch this resolves to 6.0.x — i.e. the Windows lane validates addons + # against conda-forge's newest in-range gramps, not the branch's exact series + # (the Linux CI image git-builds the exact series; conda-Windows cannot — see + # ci.yml's "Report gramps-vs-branch series" step). When 6.1 reaches + # conda-forge the pin picks it up automatically. - pip: - "gramps>=6.0,<6.1" - orjson diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97edbea6e..d606be7b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -378,6 +378,7 @@ jobs: # ----------------------------------------------------------------- unit-test-windows: name: Unit Tests (Windows) + needs: setup runs-on: windows-latest defaults: run: @@ -399,6 +400,32 @@ jobs: mamba list | head -30 python -c "import gramps, gi; print('deps OK')" + - name: Report gramps-vs-branch series (Windows lane caveat) + # The Linux lane git-builds the branch's exact gramps in its CI image + # (.github/docker/gramps-ci/Dockerfile, PyPI-first/git-fallback). The + # conda-forge Windows lane CANNOT match that: conda-forge has no gramps + # 6.1 yet, and gramps' own Windows build targets MSYS2 UCRT64, not conda + # — building 6.1 from git here fails in gramps' build hook (build_intl's + # `msgfmt --xml` cannot locate the shared-mime-info/appstream ITS rules, + # which are absent in the conda env). So on a maintenance/gramps61 (or + # later) branch this lane validates addons against conda-forge's newest + # in-range gramps (6.0.x today) rather than the branch's series. This + # step surfaces that honestly; it does NOT fail. Addon tests that depend + # on series-exact gramps behaviour skip themselves on Windows (e.g. + # TMGimporter's real-DB import tests) and run on the Linux lane instead. + # When gramps 6.1 reaches conda-forge, environment.yml's pin picks it up + # and this caveat disappears on its own. + run: | + suffix="${{ needs.setup.outputs.branch_suffix }}" # e.g. gramps61 + digits="${suffix#gramps}" # e.g. 61 + want="${digits:0:1}.${digits:1}" # e.g. 6.1 + have="$(python -c 'from gramps.version import major_version; print(major_version)')" + if [ "$want" = "$have" ]; then + echo "conda-forge gramps $have matches branch series $want — addons tested against the branch's gramps" + else + echo "::warning::Windows lane: branch targets gramps $want but conda-forge ships $have; addons here are validated against $have. Full $want coverage is on the Linux lane (its CI image git-builds $want). See step comment for why conda-Windows cannot build $want." + fi + - name: Install addon system deps (derived, conda-available subset) # Only the conda-forge-available subset (e.g. graphviz). The GTK 3 addon # GI libs (goocanvas/osm-gps-map/gexiv2) are NOT on conda-forge, so the From 06b95bcd04778e57e88b35a62fee2ab3369c2374 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Sat, 13 Jun 2026 16:07:28 +0200 Subject: [PATCH 22/47] Strip the dependency detector and TMG test split out of the CI PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the maintainer's earlier review ("too many completely different changes in a single commit"), this PR should carry only the CI infrastructure (workflows, image, the tests/ harness, run_addon_tests.py / addon_system_deps.py / gi_bootstrap). Two standalone pieces that had re-grown here move to their own follow-up PRs: - tests/test_addon_dependencies.py — the undeclared-depends_on dependency detector (Mantis-13707 class), a standalone feature; ships in its own PR with its per-module isolated-load loop parallelised. - the TMGimporter test rename (test_libtmg.py -> test_linux_libtmg.py) — a per-addon test edit; ships in its own PR as the test__* convention demonstrator. This commit removes both from #820, leaving only the CI infrastructure. Co-Authored-By: Claude Opus 4.8 (1M context) --- TMGimporter/tests/test_libtmg.py | 1192 ++++++++++++++++++++++- TMGimporter/tests/test_linux_libtmg.py | 1218 ------------------------ tests/test_addon_dependencies.py | 421 -------- 3 files changed, 1191 insertions(+), 1640 deletions(-) delete mode 100644 TMGimporter/tests/test_linux_libtmg.py delete mode 100644 tests/test_addon_dependencies.py diff --git a/TMGimporter/tests/test_libtmg.py b/TMGimporter/tests/test_libtmg.py index 4851746a6..2f92a59b7 100644 --- a/TMGimporter/tests/test_libtmg.py +++ b/TMGimporter/tests/test_libtmg.py @@ -11,13 +11,59 @@ import sys import os +import tempfile import unittest # Make sure libtmg is importable from the parent directory sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import libtmg -from gramps.gen.lib import Date +from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source +from gramps.gen.db.utils import make_database +from gramps.gen.db import DbTxn + + +# --------------------------------------------------------------------------- +# Helpers shared across test cases +# --------------------------------------------------------------------------- + +class _Rec: + """Minimal fake DBF record — set any field via keyword arguments.""" + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + +def _table(records): + """Return an object that behaves like a dbf.Table used as a context manager. + + libtmg uses tables in two ways: + with tmgFoo: + for record in tmgFoo: # iterates over context-managed table + """ + class _FakeTable: + def __enter__(self): + return self + def __exit__(self, *_): + return False + def __iter__(self): + return iter(records) + return _FakeTable() + + +def _make_db(): + """Return a fresh in-memory Gramps database.""" + db = make_database("sqlite") + db.load(":memory:", None) + return db + + +def _add_person(db): + """Add an empty Person to db and return (db, handle).""" + p = Person() + with DbTxn("setup", db) as t: + db.add_person(p, t) + return db, p.get_handle() + # --------------------------------------------------------------------------- # Pure function: _strip_tmg_codes @@ -134,6 +180,529 @@ def test_exact_certain_has_no_quality(self): self.assertEqual(d.get_quality(), Date.QUAL_NONE) +# --------------------------------------------------------------------------- +# import_notes — patches the module-level DBF table globals +# --------------------------------------------------------------------------- + +class TestImportNotes(unittest.TestCase): + + def _patch(self, tagtypes_records, events_records): + """Patch libtmg globals and return a context manager.""" + import unittest.mock as mock + patches = [ + mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), + mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), + ] + return patches + + def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + libtmg.import_notes(db, dataset, per_no_map) + return db + + def test_no_per_no_map_is_noop(self): + """Passing per_no_map=None must not touch any table.""" + import unittest.mock as mock + db = _make_db() + mock_table = mock.MagicMock() + with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ + mock.patch('libtmg.tmgEvents', mock_table, create=True): + libtmg.import_notes(db, 1, per_no_map=None) + mock_table.__enter__.assert_not_called() + + def test_note_attached_to_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {42: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, + efoot='Born in London')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 1) + + def test_note_text_stored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot=' Some note text ')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'Some note text') + + def test_note_type_is_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='hello')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get_type(), NoteType.PERSON) + + def test_tmg_codes_stripped_from_note(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='[:ITAL:]italicised[:ITAL:]')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + note = db.get_note_from_handle(person.get_note_list()[0]) + self.assertEqual(note.get(), 'italicised') + + def test_empty_note_text_skipped(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='')], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 0) + + def test_unknown_person_skipped(self): + db = _make_db() + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, + efoot='orphan note')], + per_no_map={}, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_non_note_etype_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + # etype=10 is not a Note type + events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, + efoot='should be ignored')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_wrong_dataset_ignored(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, + efoot='wrong dataset')], + per_no_map=per_no_map, dataset=1, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + def test_multiple_notes_for_one_person(self): + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], + events_records=[ + _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), + _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), + ], + per_no_map=per_no_map, db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_note_list()), 2) + + def test_no_note_tag_type_defined(self): + """If the dataset has no 'Note' tag type, nothing is imported.""" + db, phandle = _add_person(_make_db()) + per_no_map = {1: phandle} + self._run( + tagtypes_records=[], # no tag types at all + events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, + efoot='orphan')], + per_no_map=per_no_map, db=db, + ) + self.assertEqual(db.get_number_of_notes(), 0) + + +# --------------------------------------------------------------------------- +# trial_events — event import and Note-etype skip +# --------------------------------------------------------------------------- + +# A minimal raw date string for an exact date (1900-06-15) +_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' +_EMPTY_DATE = '' + + +class TestTrialEvents(unittest.TestCase): + + def _run(self, tagtypes_records, events_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ + mock.patch('libtmg.tmgEvents', _table(events_records), create=True): + handle_map = libtmg.import_events(db, dataset) + return db, handle_map + + def test_regular_event_creates_db_entry(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertIn(1, hmap) + + def test_handle_map_tuple_has_four_elements(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, + placenum=7, edate=_EMPTY_DATE, efoot='')] + _, hmap = self._run(_tagtypes, _events) + entry = hmap[5] + self.assertEqual(len(entry), 4) + _handle, per1, per2, placenum = entry + self.assertEqual(per1, 3) + self.assertEqual(per2, 0) + self.assertEqual(placenum, 7) + + def test_note_etype_event_not_in_handle_map(self): + _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] + _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note')] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 0) + self.assertNotIn(1, hmap) + + def test_event_memo_stored_as_description(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_memo_tmg_codes_stripped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, + efoot='[:CR:]born here')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + self.assertEqual(event.get_description(), 'born here') + + def test_event_date_set(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EXACT_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events) + event = db.get_event_from_handle(hmap[1][0]) + d = event.get_date_object() + self.assertEqual(d.get_year(), 1900) + self.assertEqual(d.get_month(), 6) + + def test_wrong_dataset_skipped(self): + _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] + _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='')] + db, hmap = self._run(_tagtypes, _events, dataset=1) + self.assertEqual(db.get_number_of_events(), 0) + + def test_mixed_note_and_regular_events(self): + _tagtypes = [ + _Rec(dsid=1, etypenum=77, etypename='Note'), + _Rec(dsid=1, etypenum=10, etypename='Birth'), + ] + _events = [ + _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot='a note'), + _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, + placenum=0, edate=_EMPTY_DATE, efoot=''), + ] + db, hmap = self._run(_tagtypes, _events) + self.assertEqual(db.get_number_of_events(), 1) + self.assertNotIn(1, hmap) + self.assertIn(2, hmap) + + +# --------------------------------------------------------------------------- +# import_sources — info-field parsing and author/publication split +# --------------------------------------------------------------------------- + +class TestImportSources(unittest.TestCase): + + def _run(self, src_components, src_repo_links, sources_records, + repo_handle_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ + mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ + mock.patch('libtmg.tmgSources', _table(sources_records), create=True): + smap = libtmg.import_sources(db, dataset, repo_handle_map) + return db, smap + + def _source_rec(self, **kw): + defaults = dict(dsid=1, majnum=1, mactive=True, + title='Test Source', abbrev='', info='', + text='', fform='', sform='', bform='', reminders='') + defaults.update(kw) + return _Rec(**defaults) + + def test_source_created(self): + db, smap = self._run([], [], [self._source_rec()]) + self.assertEqual(db.get_number_of_sources(), 1) + self.assertIn(1, smap) + + def test_title_set(self): + db, smap = self._run([], [], [self._source_rec(title='My Source')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_title(), 'My Source') + + def test_abbreviation_set(self): + db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_abbreviation(), 'MySrc') + + def test_inactive_source_skipped(self): + db, smap = self._run([], [], [self._source_rec(mactive=False)]) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_wrong_dataset_skipped(self): + db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_sources(), 0) + + def test_author_element_sets_author(self): + # recno 1 → position 0 in $!& split + components = [_Rec(recno=1, element='[AUTHOR]')] + rec = self._source_rec(info='John Smith') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'John Smith') + + def test_non_author_element_sets_publication_info(self): + components = [_Rec(recno=1, element='[TITLE]')] + rec = self._source_rec(info='Some Title') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertIn('TITLE', src.get_publication_info()) + self.assertIn('Some Title', src.get_publication_info()) + + def test_multiple_authors_joined_with_semicolon(self): + # positions 0 and 1 → recno 1 and 2 + components = [ + _Rec(recno=1, element='[AUTHOR]'), + _Rec(recno=2, element='[EDITOR]'), + ] + rec = self._source_rec(info='Alice$!&Bob') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Alice; Bob') + + def test_empty_info_position_skipped(self): + # position 0 empty, position 1 filled → recno 2 = [AUTHOR] + components = [ + _Rec(recno=1, element='[TITLE]'), + _Rec(recno=2, element='[AUTHOR]'), + ] + rec = self._source_rec(info='$!&Jane Doe') + db, smap = self._run(components, [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(src.get_author(), 'Jane Doe') + self.assertEqual(src.get_publication_info(), '') + + def test_note_fields_become_notes(self): + rec = self._source_rec(text='original text', fform='', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 1) + note = db.get_note_from_handle(src.get_note_list()[0]) + self.assertIn('original text', note.get()) + + def test_multiple_note_fields_each_become_a_note(self): + rec = self._source_rec(text='txt', fform='fn', sform='', bform='') + db, smap = self._run([], [], [rec]) + src = db.get_source_from_handle(smap[1]) + self.assertEqual(len(src.get_note_list()), 2) + + +# --------------------------------------------------------------------------- +# import_places — name reconstruction, type resolution, note parts +# --------------------------------------------------------------------------- + +class TestImportPlaces(unittest.TestCase): + + def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ + mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ + mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ + mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + pmap = libtmg.import_places(db, dataset) + return db, pmap + + # Convenience: build part_type, place_dict, ppv records for a single place + def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): + """ + parts: list of (label, value) e.g. [('City','London'),('Country','UK')] + Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) + """ + part_type_recs = [] + place_dict_recs = [] + ppv_recs = [] + for i, (label, value) in enumerate(parts): + type_id = i + 1 + uid = i + 100 + part_type_recs.append(_Rec(type=type_id, value=label)) + place_dict_recs.append(_Rec(uid=uid, value=value)) + ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) + place_recs = [_Rec(dsid=dataset, recno=recno, + shortplace=shortplace, comment=comment)] + return part_type_recs, place_dict_recs, ppv_recs, place_recs + + def test_city_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'London') + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_country_only_name_and_type(self): + pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.COUNTRY) + + def test_city_state_country_name_order(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + # GEO_ORDER: Addressee, Detail, City, County, State, Country + self.assertEqual(place.get_name().get_value(), + 'Paris, Île-de-France, France') + + def test_most_specific_type_wins(self): + # City is more specific than Country in _GEO_ORDER + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Berlin'), ('Country', 'Germany') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + from gramps.gen.lib import PlaceType + self.assertEqual(place.get_type().value, PlaceType.CITY) + + def test_empty_place_skipped(self): + # No parts and no shortplace → nothing imported + place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertEqual(db.get_number_of_places(), 0) + self.assertNotIn(1, pmap) + + def test_shortplace_fallback(self): + # No parts but shortplace set → use it + place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] + db, pmap = self._run([], [], [], place_recs) + self.assertIn(1, pmap) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(place.get_name().get_value(), 'Somewhere') + + def test_note_parts_go_to_note(self): + pt, pd, ppv, pl = self._setup(1, [ + ('City', 'Rome'), ('Postal', '00100') + ]) + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + self.assertEqual(len(place.get_note_list()), 1) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('Postal', note.get()) + self.assertIn('00100', note.get()) + + def test_comment_goes_to_note(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') + db, pmap = self._run(pt, pd, ppv, pl) + place = db.get_place_from_handle(pmap[1]) + note = db.get_note_from_handle(place.get_note_list()[0]) + self.assertIn('see also', note.get()) + + def test_wrong_dataset_skipped(self): + pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) + db, pmap = self._run(pt, pd, ppv, pl, dataset=1) + self.assertEqual(db.get_number_of_places(), 0) + + def test_returns_recno_to_handle_map(self): + pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) + db, pmap = self._run(pt, pd, ppv, pl) + self.assertIn(42, pmap) + + +# --------------------------------------------------------------------------- +# link_event_places — event gets its place handle set +# --------------------------------------------------------------------------- + +class TestLinkEventPlaces(unittest.TestCase): + + def _make_event(self, db): + from gramps.gen.db import DbTxn + ev = Event() + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def _make_place(self, db): + from gramps.gen.db import DbTxn + pl = Place() + with DbTxn("setup", db) as t: + db.add_place(pl, t) + return pl.get_handle() + + def test_place_linked_to_event(self): + db = _make_db() + ev_handle = self._make_event(db) + pl_handle = self._make_place(db) + event_handle_map = {1: (ev_handle, 1, 0, 7)} + place_handle_map = {7: pl_handle} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), pl_handle) + + def test_zero_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 0)} + place_handle_map = {0: self._make_place(db)} + libtmg.link_event_places(db, event_handle_map, place_handle_map) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_unknown_placenum_skipped(self): + db = _make_db() + ev_handle = self._make_event(db) + event_handle_map = {1: (ev_handle, 1, 0, 99)} + libtmg.link_event_places(db, event_handle_map, {}) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(event.get_place_handle(), '') + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_event_places(db, {}, {}) # must not raise + libtmg.link_event_places(db, None, None) + + # --------------------------------------------------------------------------- # Pure functions: num_to_month, num_to_date, parse_date # --------------------------------------------------------------------------- @@ -293,5 +862,626 @@ def test_domain_embedded_in_name(self): self.assertEqual(url, 'https://www.ancestry.com') +# --------------------------------------------------------------------------- +# Lookup helpers: short_place_name, tag_type_name +# --------------------------------------------------------------------------- + +class TestShortPlaceName(unittest.TestCase): + + def _run(self, places_records, placenum, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): + return libtmg.short_place_name(db, placenum, dataset) + + def test_returns_shortplace(self): + rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=5), 'New York') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') + self.assertEqual(self._run([rec], placenum=1), 'London') + + def test_wrong_recno_returns_none(self): + rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') + self.assertIsNone(self._run([rec], placenum=1, dataset=1)) + + +class TestTagTypeName(unittest.TestCase): + + def _run(self, tagtypes_records, eventtype, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): + return libtmg.tag_type_name(db, eventtype, dataset) + + def test_returns_name(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') + self.assertEqual(self._run([rec], eventtype=2), 'Birth') + + def test_trailing_whitespace_stripped(self): + rec = _Rec(dsid=1, etypenum=3, etypename='Death ') + self.assertEqual(self._run([rec], eventtype=3), 'Death') + + def test_wrong_eventtype_returns_none(self): + rec = _Rec(dsid=1, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=99)) + + def test_wrong_dataset_returns_none(self): + rec = _Rec(dsid=2, etypenum=2, etypename='Birth') + self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) + + +# --------------------------------------------------------------------------- +# import_people — name parsing, gender, dataset filter +# --------------------------------------------------------------------------- + +class TestImportPeople(unittest.TestCase): + + def _run(self, names_records, people_records, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ + mock.patch('libtmg.tmgPeople', _table(people_records), create=True): + per_no_map = libtmg.import_people(db, dataset) + return db, per_no_map + + def _name_rec(self, **kw): + defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') + defaults.update(kw) + return _Rec(**defaults) + + def _person_rec(self, **kw): + defaults = dict(dsid=1, per_no=1, sex='M') + defaults.update(kw) + return _Rec(**defaults) + + def test_person_created(self): + db, pmap = self._run([self._name_rec()], [self._person_rec()]) + self.assertEqual(db.get_number_of_people(), 1) + + def test_returns_per_no_map(self): + db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) + self.assertIn(5, pmap) + + def test_surname_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'JONES') + + def test_given_name_parsed(self): + db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], + [self._person_rec(per_no=1)]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') + + def test_male_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.MALE) + + def test_female_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.FEMALE) + + def test_unknown_gender(self): + db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_gender(), Person.UNKNOWN) + + def test_non_primary_name_skipped(self): + db, pmap = self._run( + [self._name_rec(primary=False, srnamedisp='ALT, Name')], + [self._person_rec()] + ) + self.assertEqual(db.get_number_of_people(), 0) + + def test_wrong_dataset_skipped(self): + db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], + dataset=1) + self.assertEqual(db.get_number_of_people(), 0) + + def test_no_comma_surname_only(self): + # srnamedisp with no comma → surname=full string, given='' + db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], + [self._person_rec()]) + p = db.get_person_from_handle(pmap[1]) + self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') + self.assertEqual(p.get_primary_name().get_first_name(), '') + + +# --------------------------------------------------------------------------- +# link_person_events — EventRefs, birth/death special refs +# --------------------------------------------------------------------------- + +class TestLinkPersonEvents(unittest.TestCase): + + def _make_typed_event(self, db, event_type_int): + from gramps.gen.lib import EventType + ev = Event() + ev.set_type(EventType(event_type_int)) + with DbTxn("setup", db) as t: + db.add_event(ev, t) + return ev.get_handle() + + def test_individual_event_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.OCCUPATION) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 1) + + def test_couple_event_not_linked_to_person(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.MARRIAGE) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 2, 0)}) + p = db.get_person_from_handle(phandle) + self.assertEqual(len(p.get_event_ref_list()), 0) + + def test_birth_event_sets_birth_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.BIRTH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_birth_ref()) + self.assertEqual(p.get_birth_ref().ref, ev_handle) + + def test_death_event_sets_death_ref(self): + from gramps.gen.lib import EventType + db, phandle = _add_person(_make_db()) + ev_handle = self._make_typed_event(db, EventType.DEATH) + libtmg.link_person_events(db, + per_no_map={1: phandle}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + p = db.get_person_from_handle(phandle) + self.assertIsNotNone(p.get_death_ref()) + + def test_unknown_person_skipped(self): + from gramps.gen.lib import EventType + db = _make_db() + ev_handle = self._make_typed_event(db, EventType.BIRTH) + # Must not raise even when per1 has no entry in per_no_map + libtmg.link_person_events(db, + per_no_map={}, + event_handle_map={1: (ev_handle, 1, 0, 0)}) + + def test_empty_maps_noop(self): + db = _make_db() + libtmg.link_person_events(db, None, None) + libtmg.link_person_events(db, {}, {}) + + +# --------------------------------------------------------------------------- +# import_families — parent-child grouping, couple events, rel type +# --------------------------------------------------------------------------- + +class TestImportFamilies(unittest.TestCase): + + def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, + dataset=1): + """Create persons from per_no_by_gender={per_no: gender}, run import.""" + import unittest.mock as mock + db = _make_db() + pmap = {} + for per_no, gender in per_no_by_gender.items(): + p = Person() + p.set_gender(gender) + with DbTxn("setup", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): + libtmg.import_families(db, dataset, pmap, event_handle_map) + return db, pmap + + def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): + return _Rec(dsid=dsid, parent=parent, child=child, + ptype=ptype, primary=primary, pnote=pnote) + + def _father_type(self, num=1): + return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') + + def _mother_type(self, num=2): + return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') + + def test_father_child_creates_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + + def test_mother_child_creates_family(self): + db, pmap = self._run([self._mother_type()], + [self._pc(1, 2, ptype=2)], + {1: Person.FEMALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_mother_handle(), pmap[1]) + + def test_father_and_mother_same_family(self): + db, pmap = self._run( + [self._father_type(1), self._mother_type(2)], + [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], + {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, + ) + self.assertEqual(db.get_number_of_families(), 1) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_father_handle(), pmap[1]) + self.assertEqual(fam.get_mother_handle(), pmap[2]) + + def test_child_added_to_family(self): + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(len(fam.get_child_ref_list()), 1) + self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) + + def test_child_ref_type_biological(self): + from gramps.gen.lib import ChildRefType + db, pmap = self._run([self._father_type()], + [self._pc(1, 2, ptype=1)], + {1: Person.MALE, 2: Person.UNKNOWN}) + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), + ChildRefType.BIRTH) + + def test_wrong_dataset_skipped(self): + db, _ = self._run([self._father_type()], + [self._pc(1, 2, ptype=1, dsid=2)], + {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) + self.assertEqual(db.get_number_of_families(), 0) + + def test_no_per_no_map_is_noop(self): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): + libtmg.import_families(db, 1, per_no_map=None) + self.assertEqual(db.get_number_of_families(), 0) + + def test_marriage_event_sets_rel_type_married(self): + from gramps.gen.lib import EventType, FamilyRelType + import unittest.mock as mock + + db = _make_db() + pmap = {} + for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): + p = Person() + p.set_gender(gender) + with DbTxn("s", db) as t: + db.add_person(p, t) + pmap[per_no] = p.get_handle() + + ev = Event() + ev.set_type(EventType(EventType.MARRIAGE)) + with DbTxn("s", db) as t: + db.add_event(ev, t) + + tagtypes = [self._father_type(1), self._mother_type(2)] + pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] + event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} + + with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): + libtmg.import_families(db, 1, pmap, event_handle_map) + + fam = db.get_family_from_handle(list(db.get_family_handles())[0]) + self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) + self.assertEqual(len(fam.get_event_ref_list()), 1) + + +# --------------------------------------------------------------------------- +# import_repositories — name, type inference, URL, notes +# --------------------------------------------------------------------------- + +class TestImportRepositories(unittest.TestCase): + + def _run(self, repo_records, per_no_map=None, dataset=1): + import unittest.mock as mock + db = _make_db() + with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): + repo_map = libtmg.import_repositories(db, dataset, per_no_map) + return db, repo_map + + def _repo_rec(self, **kw): + defaults = dict(dsid=1, recno=1, name='City Library', + abbrev='', rnote='', rperno=0) + defaults.update(kw) + return _Rec(**defaults) + + def test_repository_created(self): + db, rmap = self._run([self._repo_rec()]) + self.assertEqual(db.get_number_of_repositories(), 1) + self.assertIn(1, rmap) + + def test_name_set(self): + db, rmap = self._run([self._repo_rec(name='National Archives')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'National Archives') + + def test_wrong_dataset_skipped(self): + db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) + self.assertEqual(db.get_number_of_repositories(), 0) + + def test_type_inferred_from_name(self): + from gramps.gen.lib import RepositoryType + db, rmap = self._run([self._repo_rec(name='ancestry.com')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) + + def test_url_added_for_web_repo(self): + db, rmap = self._run([self._repo_rec(name='familysearch.org')]) + repo = db.get_repository_from_handle(rmap[1]) + urls = repo.get_url_list() + self.assertEqual(len(urls), 1) + self.assertIn('familysearch', urls[0].get_path()) + + def test_no_url_for_non_web_repo(self): + db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_url_list()), 0) + + def test_blank_name_falls_back_to_abbrev(self): + db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(repo.get_name(), 'TNA') + + def test_note_added_when_rnote_set(self): + db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) + repo = db.get_repository_from_handle(rmap[1]) + self.assertEqual(len(repo.get_note_list()), 1) + note = db.get_note_from_handle(repo.get_note_list()[0]) + self.assertIn('Open Mon-Fri', note.get()) + + def test_returns_recno_to_handle_map(self): + db, rmap = self._run([self._repo_rec(recno=42)]) + self.assertIn(42, rmap) + + +# --------------------------------------------------------------------------- +# import_citations — creation and attachment to events / persons +# --------------------------------------------------------------------------- + +class TestImportCitations(unittest.TestCase): + + def _run(self, citation_records, names_records=None, pc_records=None, + source_handle_map=None, event_handle_map=None, per_no_map=None, + dataset=1, db=None): + import unittest.mock as mock + if db is None: + db = _make_db() + with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ + mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ + mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): + libtmg.import_citations(db, dataset, + source_handle_map=source_handle_map, + event_handle_map=event_handle_map, + per_no_map=per_no_map) + return db + + def _cit_rec(self, **kw): + defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, + exclude=False, subsource='', citref='', citmemo='', + sdsure='', snsure='', sssure='', spsure='', sfsure='') + defaults.update(kw) + return _Rec(**defaults) + + def test_no_source_map_is_noop(self): + db = self._run([self._cit_rec()]) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_created(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec()], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 1) + + def test_excluded_citation_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(exclude=True)], + source_handle_map={1: src.get_handle()}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_wrong_dataset_skipped(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(dsid=2)], + source_handle_map={1: src.get_handle()}, db=db, dataset=1) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_unknown_source_skipped(self): + db = _make_db() + db = self._run([self._cit_rec(majsource=99)], + source_handle_map={1: 'some_handle'}, db=db) + self.assertEqual(db.get_number_of_citations(), 0) + + def test_citation_attached_to_event(self): + db = _make_db() + src = Source() + ev = Event() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_event(ev, t) + ev_handle = ev.get_handle() + + db = self._run([self._cit_rec(stype='E', refrec=7)], + source_handle_map={1: src.get_handle()}, + event_handle_map={7: (ev_handle, 1, 0, 0)}, + db=db) + event = db.get_event_from_handle(ev_handle) + self.assertEqual(len(event.get_citation_list()), 1) + + def test_citation_attached_to_person_via_name(self): + db = _make_db() + src = Source() + p = Person() + with DbTxn("s", db) as t: + db.add_source(src, t) + db.add_person(p, t) + phandle = p.get_handle() + + # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle + db = self._run( + [self._cit_rec(stype='N', refrec=3)], + names_records=[_Rec(dsid=1, recno=3, nper=5)], + source_handle_map={1: src.get_handle()}, + per_no_map={5: phandle}, + db=db, + ) + person = db.get_person_from_handle(phandle) + self.assertEqual(len(person.get_citation_list()), 1) + + def test_subsource_becomes_page(self): + db = _make_db() + src = Source() + with DbTxn("s", db) as t: + db.add_source(src, t) + db = self._run([self._cit_rec(subsource='p.42')], + source_handle_map={1: src.get_handle()}, db=db) + cit_handle = list(db.get_citation_handles())[0] + cit = db.get_citation_from_handle(cit_handle) + self.assertEqual(cit.get_page(), 'p.42') + + + +# ── TmgProject._read_pjc_config ────────────────────────────────────────── + +def _make_minimal_sqz(tmp_dir, pjc_content): + """Create a minimal .SQZ zip containing only a PJC file.""" + import zipfile as _zf + pjc_path = os.path.join(tmp_dir, 'test.pjc') + sqz_path = os.path.join(tmp_dir, 'test.sqz') + with open(pjc_path, 'w', encoding='latin-1') as f: + f.write(pjc_content) + with _zf.ZipFile(sqz_path, 'w') as zf: + zf.write(pjc_path, 'test.pjc') + return sqz_path + + +class _MockUser: + """Minimal stand-in for the Gramps user object used in importData.""" + def __init__(self): + self.error_shown = False + self.error_message = None + self.uistate = None + + def notify_error(self, title, message=''): + self.error_shown = True + self.error_message = message + + def begin_progress(self, *a, **kw): pass + def end_progress(self): pass + def step_progress(self): pass + + +class TestReadPjcConfig(unittest.TestCase): + """Tests for TmgProject._read_pjc_config PJC parsing.""" + + _MINIMAL_PJC = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Researcher]\n" + "Name=Test User\n" + ) + + def _make_project(self, pjc_content, tmp_path): + pjc_file = os.path.join(tmp_path, "test.pjc") + with open(pjc_file, 'w', encoding='latin-1') as f: + f.write(pjc_content) + return libtmg.TmgProject(pjc_file) + + def test_well_formed_pjc_returns_version(self): + """A clean PJC file parses successfully and version() returns a float.""" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(self._MINIMAL_PJC, tmp) + self.assertEqual(project.version(), 11.0) + + def test_malformed_section_header_does_not_raise(self): + """Lines like '[Exho' (no closing bracket) are silently dropped.""" + pjc = ( + "[Stamp]\n" + "PjcVersion=11.0\n" + "[Exho\n" # malformed — the crash trigger + "SomeGarbage\n" + "[Researcher]\n" + "Name=Test User\n" + ) + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + # Must not raise; must still find [Stamp] + self.assertEqual(project.version(), 11.0) + + def test_null_bytes_stripped(self): + """NUL bytes in the PJC file are stripped before parsing.""" + pjc = "[Stamp]\x00\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + self.assertEqual(project.version(), 11.0) + + def test_parse_error_returns_partial_config(self): + """If configparser still raises after filtering, a warning is logged + and a (possibly empty) config object is returned rather than crashing.""" + import logging + # Feed content that survives the filter but still breaks configparser: + # a key=value line before any section header is technically invalid. + pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" + with tempfile.TemporaryDirectory() as tmp: + project = self._make_project(pjc, tmp) + with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: + cfg = project._read_pjc_config() + self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() + for m in cm.output)) + # Config object is returned (not None), even if incomplete + self.assertIsNotNone(cfg) + + def test_version_too_old_notifies_user(self): + """A PJC version < 11.0 calls user.notify_error and aborts import.""" + pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for old version") + self.assertIn('9.05', user.error_message or '', + "Error message should mention TMG 9.05") + + def test_missing_pjc_version_notifies_user(self): + """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" + pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all + with tempfile.TemporaryDirectory() as tmp: + sqz = _make_minimal_sqz(tmp, pjc) + db = _make_db() + user = _MockUser() + libtmg.importData(db, sqz, user) + self.assertTrue(user.error_shown, + "notify_error should have been called for missing version") + if __name__ == '__main__': unittest.main() diff --git a/TMGimporter/tests/test_linux_libtmg.py b/TMGimporter/tests/test_linux_libtmg.py deleted file mode 100644 index 35bd11717..000000000 --- a/TMGimporter/tests/test_linux_libtmg.py +++ /dev/null @@ -1,1218 +0,0 @@ -"""Linux-only unit tests for libtmg.py - -Split off from test_libtmg.py: every class in this module creates an -in-memory Gramps SQLite database via - - make_database("sqlite").load(":memory:", None) - -which currently hangs on Windows under the conda-forge GTK + pip Gramps -combination used by the CI unit-test-windows job. Pure-logic tests that -do not touch the Gramps DB layer stay in test_libtmg.py and run on every -OS. - -Filename convention (see .github/workflows/ci.yml): - test_*.py general (every OS) - test_linux_*.py Linux-only - test_windows_*.py Windows-only - test_integration_*.py Linux-only, full-pipeline/DB-backed -""" - -import sys -import os -import tempfile -import unittest - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import libtmg - -from gramps.gen.lib import Date, Event, NoteType, Person, Place, Source -from gramps.gen.db.utils import make_database -from gramps.gen.db import DbTxn - - -# --------------------------------------------------------------------------- -# Helpers shared across test cases -# --------------------------------------------------------------------------- - -class _Rec: - """Minimal fake DBF record — set any field via keyword arguments.""" - def __init__(self, **kwargs): - self.__dict__.update(kwargs) - - -def _table(records): - """Return an object that behaves like a dbf.Table used as a context manager. - - libtmg uses tables in two ways: - with tmgFoo: - for record in tmgFoo: # iterates over context-managed table - """ - class _FakeTable: - def __enter__(self): - return self - def __exit__(self, *_): - return False - def __iter__(self): - return iter(records) - return _FakeTable() - - -def _make_db(): - """Return a fresh in-memory Gramps database.""" - db = make_database("sqlite") - db.load(":memory:", None) - return db - - -def _add_person(db): - """Add an empty Person to db and return (db, handle).""" - p = Person() - with DbTxn("setup", db) as t: - db.add_person(p, t) - return db, p.get_handle() - - -# --------------------------------------------------------------------------- -# import_notes — per-person note creation from tmg events -# --------------------------------------------------------------------------- - -class TestImportNotes(unittest.TestCase): - - def _patch(self, tagtypes_records, events_records): - """Patch libtmg globals and return a context manager.""" - import unittest.mock as mock - patches = [ - mock.patch.object(libtmg, 'tmgTagTypes', _table(tagtypes_records)), - mock.patch.object(libtmg, 'tmgEvents', _table(events_records)), - ] - return patches - - def _run(self, tagtypes_records, events_records, per_no_map, dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - libtmg.import_notes(db, dataset, per_no_map) - return db - - def test_no_per_no_map_is_noop(self): - """Passing per_no_map=None must not touch any table.""" - import unittest.mock as mock - db = _make_db() - mock_table = mock.MagicMock() - with mock.patch('libtmg.tmgTagTypes', mock_table, create=True), \ - mock.patch('libtmg.tmgEvents', mock_table, create=True): - libtmg.import_notes(db, 1, per_no_map=None) - mock_table.__enter__.assert_not_called() - - def test_note_attached_to_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {42: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=42, recno=1, - efoot='Born in London')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 1) - - def test_note_text_stored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=10, etypename='Note')], - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot=' Some note text ')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'Some note text') - - def test_note_type_is_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='hello')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get_type(), NoteType.PERSON) - - def test_tmg_codes_stripped_from_note(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='[:ITAL:]italicised[:ITAL:]')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - note = db.get_note_from_handle(person.get_note_list()[0]) - self.assertEqual(note.get(), 'italicised') - - def test_empty_note_text_skipped(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='')], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 0) - - def test_unknown_person_skipped(self): - db = _make_db() - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=1, etype=77, per1=99, recno=1, - efoot='orphan note')], - per_no_map={}, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_non_note_etype_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - # etype=10 is not a Note type - events_records=[_Rec(dsid=1, etype=10, per1=1, recno=1, - efoot='should be ignored')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_wrong_dataset_ignored(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[_Rec(dsid=2, etype=77, per1=1, recno=1, - efoot='wrong dataset')], - per_no_map=per_no_map, dataset=1, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - def test_multiple_notes_for_one_person(self): - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[_Rec(dsid=1, etypenum=77, etypename='Note')], - events_records=[ - _Rec(dsid=1, etype=77, per1=1, recno=1, efoot='first'), - _Rec(dsid=1, etype=77, per1=1, recno=2, efoot='second'), - ], - per_no_map=per_no_map, db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_note_list()), 2) - - def test_no_note_tag_type_defined(self): - """If the dataset has no 'Note' tag type, nothing is imported.""" - db, phandle = _add_person(_make_db()) - per_no_map = {1: phandle} - self._run( - tagtypes_records=[], # no tag types at all - events_records=[_Rec(dsid=1, etype=77, per1=1, recno=1, - efoot='orphan')], - per_no_map=per_no_map, db=db, - ) - self.assertEqual(db.get_number_of_notes(), 0) - - -# --------------------------------------------------------------------------- -# trial_events — event import and Note-etype skip -# --------------------------------------------------------------------------- - -# A minimal raw date string for an exact date (1900-06-15) -_EXACT_DATE = '1' + '19000615' + '0' + '3' + '00000000' + '0' + '0' -_EMPTY_DATE = '' - - -class TestTrialEvents(unittest.TestCase): - - def _run(self, tagtypes_records, events_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True), \ - mock.patch('libtmg.tmgEvents', _table(events_records), create=True): - handle_map = libtmg.import_events(db, dataset) - return db, handle_map - - def test_regular_event_creates_db_entry(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertIn(1, hmap) - - def test_handle_map_tuple_has_four_elements(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=5, etype=10, per1=3, per2=0, - placenum=7, edate=_EMPTY_DATE, efoot='')] - _, hmap = self._run(_tagtypes, _events) - entry = hmap[5] - self.assertEqual(len(entry), 4) - _handle, per1, per2, placenum = entry - self.assertEqual(per1, 3) - self.assertEqual(per2, 0) - self.assertEqual(placenum, 7) - - def test_note_etype_event_not_in_handle_map(self): - _tagtypes = [_Rec(dsid=1, etypenum=77, etypename='Note')] - _events = [_Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note')] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 0) - self.assertNotIn(1, hmap) - - def test_event_memo_stored_as_description(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_memo_tmg_codes_stripped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, - efoot='[:CR:]born here')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - self.assertEqual(event.get_description(), 'born here') - - def test_event_date_set(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=1, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EXACT_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events) - event = db.get_event_from_handle(hmap[1][0]) - d = event.get_date_object() - self.assertEqual(d.get_year(), 1900) - self.assertEqual(d.get_month(), 6) - - def test_wrong_dataset_skipped(self): - _tagtypes = [_Rec(dsid=1, etypenum=10, etypename='Birth')] - _events = [_Rec(dsid=2, recno=1, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='')] - db, hmap = self._run(_tagtypes, _events, dataset=1) - self.assertEqual(db.get_number_of_events(), 0) - - def test_mixed_note_and_regular_events(self): - _tagtypes = [ - _Rec(dsid=1, etypenum=77, etypename='Note'), - _Rec(dsid=1, etypenum=10, etypename='Birth'), - ] - _events = [ - _Rec(dsid=1, recno=1, etype=77, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot='a note'), - _Rec(dsid=1, recno=2, etype=10, per1=1, per2=0, - placenum=0, edate=_EMPTY_DATE, efoot=''), - ] - db, hmap = self._run(_tagtypes, _events) - self.assertEqual(db.get_number_of_events(), 1) - self.assertNotIn(1, hmap) - self.assertIn(2, hmap) - - -# --------------------------------------------------------------------------- -# import_sources — info-field parsing and author/publication split -# --------------------------------------------------------------------------- - -class TestImportSources(unittest.TestCase): - - def _run(self, src_components, src_repo_links, sources_records, - repo_handle_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgSourceComponents', _table(src_components), create=True), \ - mock.patch('libtmg.tmgSourceRepositoryLinks', _table(src_repo_links), create=True), \ - mock.patch('libtmg.tmgSources', _table(sources_records), create=True): - smap = libtmg.import_sources(db, dataset, repo_handle_map) - return db, smap - - def _source_rec(self, **kw): - defaults = dict(dsid=1, majnum=1, mactive=True, - title='Test Source', abbrev='', info='', - text='', fform='', sform='', bform='', reminders='') - defaults.update(kw) - return _Rec(**defaults) - - def test_source_created(self): - db, smap = self._run([], [], [self._source_rec()]) - self.assertEqual(db.get_number_of_sources(), 1) - self.assertIn(1, smap) - - def test_title_set(self): - db, smap = self._run([], [], [self._source_rec(title='My Source')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_title(), 'My Source') - - def test_abbreviation_set(self): - db, smap = self._run([], [], [self._source_rec(abbrev='MySrc')]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_abbreviation(), 'MySrc') - - def test_inactive_source_skipped(self): - db, smap = self._run([], [], [self._source_rec(mactive=False)]) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_wrong_dataset_skipped(self): - db, smap = self._run([], [], [self._source_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_sources(), 0) - - def test_author_element_sets_author(self): - # recno 1 → position 0 in $!& split - components = [_Rec(recno=1, element='[AUTHOR]')] - rec = self._source_rec(info='John Smith') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'John Smith') - - def test_non_author_element_sets_publication_info(self): - components = [_Rec(recno=1, element='[TITLE]')] - rec = self._source_rec(info='Some Title') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertIn('TITLE', src.get_publication_info()) - self.assertIn('Some Title', src.get_publication_info()) - - def test_multiple_authors_joined_with_semicolon(self): - # positions 0 and 1 → recno 1 and 2 - components = [ - _Rec(recno=1, element='[AUTHOR]'), - _Rec(recno=2, element='[EDITOR]'), - ] - rec = self._source_rec(info='Alice$!&Bob') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Alice; Bob') - - def test_empty_info_position_skipped(self): - # position 0 empty, position 1 filled → recno 2 = [AUTHOR] - components = [ - _Rec(recno=1, element='[TITLE]'), - _Rec(recno=2, element='[AUTHOR]'), - ] - rec = self._source_rec(info='$!&Jane Doe') - db, smap = self._run(components, [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(src.get_author(), 'Jane Doe') - self.assertEqual(src.get_publication_info(), '') - - def test_note_fields_become_notes(self): - rec = self._source_rec(text='original text', fform='', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 1) - note = db.get_note_from_handle(src.get_note_list()[0]) - self.assertIn('original text', note.get()) - - def test_multiple_note_fields_each_become_a_note(self): - rec = self._source_rec(text='txt', fform='fn', sform='', bform='') - db, smap = self._run([], [], [rec]) - src = db.get_source_from_handle(smap[1]) - self.assertEqual(len(src.get_note_list()), 2) - - -# --------------------------------------------------------------------------- -# import_places — name reconstruction, type resolution, note parts -# --------------------------------------------------------------------------- - -class TestImportPlaces(unittest.TestCase): - - def _run(self, part_types, place_dict, ppv_records, places_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlacePartType', _table(part_types), create=True), \ - mock.patch('libtmg.tmgPlaceDictionary', _table(place_dict), create=True), \ - mock.patch('libtmg.tmgPlacePartValue', _table(ppv_records), create=True), \ - mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - pmap = libtmg.import_places(db, dataset) - return db, pmap - - # Convenience: build part_type, place_dict, ppv records for a single place - def _setup(self, recno, parts, dataset=1, comment='', shortplace=''): - """ - parts: list of (label, value) e.g. [('City','London'),('Country','UK')] - Returns (part_type_recs, place_dict_recs, ppv_recs, place_recs) - """ - part_type_recs = [] - place_dict_recs = [] - ppv_recs = [] - for i, (label, value) in enumerate(parts): - type_id = i + 1 - uid = i + 100 - part_type_recs.append(_Rec(type=type_id, value=label)) - place_dict_recs.append(_Rec(uid=uid, value=value)) - ppv_recs.append(_Rec(dsid=dataset, recno=recno, type=type_id, uid=uid)) - place_recs = [_Rec(dsid=dataset, recno=recno, - shortplace=shortplace, comment=comment)] - return part_type_recs, place_dict_recs, ppv_recs, place_recs - - def test_city_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'London')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'London') - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_country_only_name_and_type(self): - pt, pd, ppv, pl = self._setup(1, [('Country', 'France')]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.COUNTRY) - - def test_city_state_country_name_order(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Paris'), ('State', 'Île-de-France'), ('Country', 'France') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - # GEO_ORDER: Addressee, Detail, City, County, State, Country - self.assertEqual(place.get_name().get_value(), - 'Paris, Île-de-France, France') - - def test_most_specific_type_wins(self): - # City is more specific than Country in _GEO_ORDER - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Berlin'), ('Country', 'Germany') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - from gramps.gen.lib import PlaceType - self.assertEqual(place.get_type().value, PlaceType.CITY) - - def test_empty_place_skipped(self): - # No parts and no shortplace → nothing imported - place_recs = [_Rec(dsid=1, recno=1, shortplace='', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertEqual(db.get_number_of_places(), 0) - self.assertNotIn(1, pmap) - - def test_shortplace_fallback(self): - # No parts but shortplace set → use it - place_recs = [_Rec(dsid=1, recno=1, shortplace='Somewhere', comment='')] - db, pmap = self._run([], [], [], place_recs) - self.assertIn(1, pmap) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(place.get_name().get_value(), 'Somewhere') - - def test_note_parts_go_to_note(self): - pt, pd, ppv, pl = self._setup(1, [ - ('City', 'Rome'), ('Postal', '00100') - ]) - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - self.assertEqual(len(place.get_note_list()), 1) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('Postal', note.get()) - self.assertIn('00100', note.get()) - - def test_comment_goes_to_note(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Rome')], comment='see also') - db, pmap = self._run(pt, pd, ppv, pl) - place = db.get_place_from_handle(pmap[1]) - note = db.get_note_from_handle(place.get_note_list()[0]) - self.assertIn('see also', note.get()) - - def test_wrong_dataset_skipped(self): - pt, pd, ppv, pl = self._setup(1, [('City', 'Oslo')], dataset=2) - db, pmap = self._run(pt, pd, ppv, pl, dataset=1) - self.assertEqual(db.get_number_of_places(), 0) - - def test_returns_recno_to_handle_map(self): - pt, pd, ppv, pl = self._setup(42, [('City', 'Vienna')]) - db, pmap = self._run(pt, pd, ppv, pl) - self.assertIn(42, pmap) - - -# --------------------------------------------------------------------------- -# link_event_places — event gets its place handle set -# --------------------------------------------------------------------------- - -class TestLinkEventPlaces(unittest.TestCase): - - def _make_event(self, db): - from gramps.gen.db import DbTxn - ev = Event() - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def _make_place(self, db): - from gramps.gen.db import DbTxn - pl = Place() - with DbTxn("setup", db) as t: - db.add_place(pl, t) - return pl.get_handle() - - def test_place_linked_to_event(self): - db = _make_db() - ev_handle = self._make_event(db) - pl_handle = self._make_place(db) - event_handle_map = {1: (ev_handle, 1, 0, 7)} - place_handle_map = {7: pl_handle} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), pl_handle) - - def test_zero_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 0)} - place_handle_map = {0: self._make_place(db)} - libtmg.link_event_places(db, event_handle_map, place_handle_map) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_unknown_placenum_skipped(self): - db = _make_db() - ev_handle = self._make_event(db) - event_handle_map = {1: (ev_handle, 1, 0, 99)} - libtmg.link_event_places(db, event_handle_map, {}) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(event.get_place_handle(), '') - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_event_places(db, {}, {}) # must not raise - libtmg.link_event_places(db, None, None) - - -# --------------------------------------------------------------------------- -# Pure functions: num_to_month, num_to_date, parse_date -# --------------------------------------------------------------------------- -class TestShortPlaceName(unittest.TestCase): - - def _run(self, places_records, placenum, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgPlaces', _table(places_records), create=True): - return libtmg.short_place_name(db, placenum, dataset) - - def test_returns_shortplace(self): - rec = _Rec(dsid=1, recno=5, shortplace='New York ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=5), 'New York') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, recno=1, shortplace='London ', styleid=1, comment='') - self.assertEqual(self._run([rec], placenum=1), 'London') - - def test_wrong_recno_returns_none(self): - rec = _Rec(dsid=1, recno=1, shortplace='Paris', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, recno=1, shortplace='Berlin', styleid=1, comment='') - self.assertIsNone(self._run([rec], placenum=1, dataset=1)) - - -class TestTagTypeName(unittest.TestCase): - - def _run(self, tagtypes_records, eventtype, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes_records), create=True): - return libtmg.tag_type_name(db, eventtype, dataset) - - def test_returns_name(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth ') - self.assertEqual(self._run([rec], eventtype=2), 'Birth') - - def test_trailing_whitespace_stripped(self): - rec = _Rec(dsid=1, etypenum=3, etypename='Death ') - self.assertEqual(self._run([rec], eventtype=3), 'Death') - - def test_wrong_eventtype_returns_none(self): - rec = _Rec(dsid=1, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=99)) - - def test_wrong_dataset_returns_none(self): - rec = _Rec(dsid=2, etypenum=2, etypename='Birth') - self.assertIsNone(self._run([rec], eventtype=2, dataset=1)) - - -# --------------------------------------------------------------------------- -# import_people — name parsing, gender, dataset filter -# --------------------------------------------------------------------------- - -class TestImportPeople(unittest.TestCase): - - def _run(self, names_records, people_records, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgNames', _table(names_records), create=True), \ - mock.patch('libtmg.tmgPeople', _table(people_records), create=True): - per_no_map = libtmg.import_people(db, dataset) - return db, per_no_map - - def _name_rec(self, **kw): - defaults = dict(dsid=1, nper=1, primary=True, srnamedisp='SMITH, John') - defaults.update(kw) - return _Rec(**defaults) - - def _person_rec(self, **kw): - defaults = dict(dsid=1, per_no=1, sex='M') - defaults.update(kw) - return _Rec(**defaults) - - def test_person_created(self): - db, pmap = self._run([self._name_rec()], [self._person_rec()]) - self.assertEqual(db.get_number_of_people(), 1) - - def test_returns_per_no_map(self): - db, pmap = self._run([self._name_rec(nper=5)], [self._person_rec(per_no=5)]) - self.assertIn(5, pmap) - - def test_surname_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'JONES') - - def test_given_name_parsed(self): - db, pmap = self._run([self._name_rec(nper=1, srnamedisp='JONES, Alice')], - [self._person_rec(per_no=1)]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_first_name(), 'Alice') - - def test_male_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='M')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.MALE) - - def test_female_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='F')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.FEMALE) - - def test_unknown_gender(self): - db, pmap = self._run([self._name_rec()], [self._person_rec(sex='?')]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_gender(), Person.UNKNOWN) - - def test_non_primary_name_skipped(self): - db, pmap = self._run( - [self._name_rec(primary=False, srnamedisp='ALT, Name')], - [self._person_rec()] - ) - self.assertEqual(db.get_number_of_people(), 0) - - def test_wrong_dataset_skipped(self): - db, pmap = self._run([self._name_rec(dsid=2)], [self._person_rec(dsid=2)], - dataset=1) - self.assertEqual(db.get_number_of_people(), 0) - - def test_no_comma_surname_only(self): - # srnamedisp with no comma → surname=full string, given='' - db, pmap = self._run([self._name_rec(srnamedisp='SMITH')], - [self._person_rec()]) - p = db.get_person_from_handle(pmap[1]) - self.assertEqual(p.get_primary_name().get_surname(), 'SMITH') - self.assertEqual(p.get_primary_name().get_first_name(), '') - - -# --------------------------------------------------------------------------- -# link_person_events — EventRefs, birth/death special refs -# --------------------------------------------------------------------------- - -class TestLinkPersonEvents(unittest.TestCase): - - def _make_typed_event(self, db, event_type_int): - from gramps.gen.lib import EventType - ev = Event() - ev.set_type(EventType(event_type_int)) - with DbTxn("setup", db) as t: - db.add_event(ev, t) - return ev.get_handle() - - def test_individual_event_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.OCCUPATION) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 1) - - def test_couple_event_not_linked_to_person(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.MARRIAGE) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 2, 0)}) - p = db.get_person_from_handle(phandle) - self.assertEqual(len(p.get_event_ref_list()), 0) - - def test_birth_event_sets_birth_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.BIRTH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_birth_ref()) - self.assertEqual(p.get_birth_ref().ref, ev_handle) - - def test_death_event_sets_death_ref(self): - from gramps.gen.lib import EventType - db, phandle = _add_person(_make_db()) - ev_handle = self._make_typed_event(db, EventType.DEATH) - libtmg.link_person_events(db, - per_no_map={1: phandle}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - p = db.get_person_from_handle(phandle) - self.assertIsNotNone(p.get_death_ref()) - - def test_unknown_person_skipped(self): - from gramps.gen.lib import EventType - db = _make_db() - ev_handle = self._make_typed_event(db, EventType.BIRTH) - # Must not raise even when per1 has no entry in per_no_map - libtmg.link_person_events(db, - per_no_map={}, - event_handle_map={1: (ev_handle, 1, 0, 0)}) - - def test_empty_maps_noop(self): - db = _make_db() - libtmg.link_person_events(db, None, None) - libtmg.link_person_events(db, {}, {}) - - -# --------------------------------------------------------------------------- -# import_families — parent-child grouping, couple events, rel type -# --------------------------------------------------------------------------- - -class TestImportFamilies(unittest.TestCase): - - def _run(self, tagtypes, pc_rels, per_no_by_gender, event_handle_map=None, - dataset=1): - """Create persons from per_no_by_gender={per_no: gender}, run import.""" - import unittest.mock as mock - db = _make_db() - pmap = {} - for per_no, gender in per_no_by_gender.items(): - p = Person() - p.set_gender(gender) - with DbTxn("setup", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_rels), create=True): - libtmg.import_families(db, dataset, pmap, event_handle_map) - return db, pmap - - def _pc(self, parent, child, ptype, primary=True, pnote='', dsid=1): - return _Rec(dsid=dsid, parent=parent, child=child, - ptype=ptype, primary=primary, pnote=pnote) - - def _father_type(self, num=1): - return _Rec(dsid=1, etypenum=num, etypename='Father-Biological') - - def _mother_type(self, num=2): - return _Rec(dsid=1, etypenum=num, etypename='Mother-Biological') - - def test_father_child_creates_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - - def test_mother_child_creates_family(self): - db, pmap = self._run([self._mother_type()], - [self._pc(1, 2, ptype=2)], - {1: Person.FEMALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_mother_handle(), pmap[1]) - - def test_father_and_mother_same_family(self): - db, pmap = self._run( - [self._father_type(1), self._mother_type(2)], - [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)], - {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}, - ) - self.assertEqual(db.get_number_of_families(), 1) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_father_handle(), pmap[1]) - self.assertEqual(fam.get_mother_handle(), pmap[2]) - - def test_child_added_to_family(self): - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(len(fam.get_child_ref_list()), 1) - self.assertEqual(fam.get_child_ref_list()[0].ref, pmap[2]) - - def test_child_ref_type_biological(self): - from gramps.gen.lib import ChildRefType - db, pmap = self._run([self._father_type()], - [self._pc(1, 2, ptype=1)], - {1: Person.MALE, 2: Person.UNKNOWN}) - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_child_ref_list()[0].get_father_relation(), - ChildRefType.BIRTH) - - def test_wrong_dataset_skipped(self): - db, _ = self._run([self._father_type()], - [self._pc(1, 2, ptype=1, dsid=2)], - {1: Person.MALE, 2: Person.UNKNOWN}, dataset=1) - self.assertEqual(db.get_number_of_families(), 0) - - def test_no_per_no_map_is_noop(self): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgTagTypes', _table([]), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table([]), create=True): - libtmg.import_families(db, 1, per_no_map=None) - self.assertEqual(db.get_number_of_families(), 0) - - def test_marriage_event_sets_rel_type_married(self): - from gramps.gen.lib import EventType, FamilyRelType - import unittest.mock as mock - - db = _make_db() - pmap = {} - for per_no, gender in {1: Person.MALE, 2: Person.FEMALE, 3: Person.UNKNOWN}.items(): - p = Person() - p.set_gender(gender) - with DbTxn("s", db) as t: - db.add_person(p, t) - pmap[per_no] = p.get_handle() - - ev = Event() - ev.set_type(EventType(EventType.MARRIAGE)) - with DbTxn("s", db) as t: - db.add_event(ev, t) - - tagtypes = [self._father_type(1), self._mother_type(2)] - pc = [self._pc(1, 3, ptype=1), self._pc(2, 3, ptype=2)] - event_handle_map = {99: (ev.get_handle(), 1, 2, 0)} - - with mock.patch('libtmg.tmgTagTypes', _table(tagtypes), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc), create=True): - libtmg.import_families(db, 1, pmap, event_handle_map) - - fam = db.get_family_from_handle(list(db.get_family_handles())[0]) - self.assertEqual(fam.get_relationship(), FamilyRelType.MARRIED) - self.assertEqual(len(fam.get_event_ref_list()), 1) - - -# --------------------------------------------------------------------------- -# import_repositories — name, type inference, URL, notes -# --------------------------------------------------------------------------- - -class TestImportRepositories(unittest.TestCase): - - def _run(self, repo_records, per_no_map=None, dataset=1): - import unittest.mock as mock - db = _make_db() - with mock.patch('libtmg.tmgRepositories', _table(repo_records), create=True): - repo_map = libtmg.import_repositories(db, dataset, per_no_map) - return db, repo_map - - def _repo_rec(self, **kw): - defaults = dict(dsid=1, recno=1, name='City Library', - abbrev='', rnote='', rperno=0) - defaults.update(kw) - return _Rec(**defaults) - - def test_repository_created(self): - db, rmap = self._run([self._repo_rec()]) - self.assertEqual(db.get_number_of_repositories(), 1) - self.assertIn(1, rmap) - - def test_name_set(self): - db, rmap = self._run([self._repo_rec(name='National Archives')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'National Archives') - - def test_wrong_dataset_skipped(self): - db, rmap = self._run([self._repo_rec(dsid=2)], dataset=1) - self.assertEqual(db.get_number_of_repositories(), 0) - - def test_type_inferred_from_name(self): - from gramps.gen.lib import RepositoryType - db, rmap = self._run([self._repo_rec(name='ancestry.com')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_type().value, RepositoryType.WEBSITE) - - def test_url_added_for_web_repo(self): - db, rmap = self._run([self._repo_rec(name='familysearch.org')]) - repo = db.get_repository_from_handle(rmap[1]) - urls = repo.get_url_list() - self.assertEqual(len(urls), 1) - self.assertIn('familysearch', urls[0].get_path()) - - def test_no_url_for_non_web_repo(self): - db, rmap = self._run([self._repo_rec(name='Local Parish Church')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_url_list()), 0) - - def test_blank_name_falls_back_to_abbrev(self): - db, rmap = self._run([self._repo_rec(name='', abbrev='TNA')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(repo.get_name(), 'TNA') - - def test_note_added_when_rnote_set(self): - db, rmap = self._run([self._repo_rec(rnote='Open Mon-Fri')]) - repo = db.get_repository_from_handle(rmap[1]) - self.assertEqual(len(repo.get_note_list()), 1) - note = db.get_note_from_handle(repo.get_note_list()[0]) - self.assertIn('Open Mon-Fri', note.get()) - - def test_returns_recno_to_handle_map(self): - db, rmap = self._run([self._repo_rec(recno=42)]) - self.assertIn(42, rmap) - - -# --------------------------------------------------------------------------- -# import_citations — creation and attachment to events / persons -# --------------------------------------------------------------------------- - -class TestImportCitations(unittest.TestCase): - - def _run(self, citation_records, names_records=None, pc_records=None, - source_handle_map=None, event_handle_map=None, per_no_map=None, - dataset=1, db=None): - import unittest.mock as mock - if db is None: - db = _make_db() - with mock.patch('libtmg.tmgCitations', _table(citation_records), create=True), \ - mock.patch('libtmg.tmgNames', _table(names_records or []), create=True), \ - mock.patch('libtmg.tmgParentChildRelationships', _table(pc_records or []), create=True): - libtmg.import_citations(db, dataset, - source_handle_map=source_handle_map, - event_handle_map=event_handle_map, - per_no_map=per_no_map) - return db - - def _cit_rec(self, **kw): - defaults = dict(dsid=1, recno=1, majsource=1, stype='E', refrec=1, - exclude=False, subsource='', citref='', citmemo='', - sdsure='', snsure='', sssure='', spsure='', sfsure='') - defaults.update(kw) - return _Rec(**defaults) - - def test_no_source_map_is_noop(self): - db = self._run([self._cit_rec()]) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_created(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec()], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 1) - - def test_excluded_citation_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(exclude=True)], - source_handle_map={1: src.get_handle()}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_wrong_dataset_skipped(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(dsid=2)], - source_handle_map={1: src.get_handle()}, db=db, dataset=1) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_unknown_source_skipped(self): - db = _make_db() - db = self._run([self._cit_rec(majsource=99)], - source_handle_map={1: 'some_handle'}, db=db) - self.assertEqual(db.get_number_of_citations(), 0) - - def test_citation_attached_to_event(self): - db = _make_db() - src = Source() - ev = Event() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_event(ev, t) - ev_handle = ev.get_handle() - - db = self._run([self._cit_rec(stype='E', refrec=7)], - source_handle_map={1: src.get_handle()}, - event_handle_map={7: (ev_handle, 1, 0, 0)}, - db=db) - event = db.get_event_from_handle(ev_handle) - self.assertEqual(len(event.get_citation_list()), 1) - - def test_citation_attached_to_person_via_name(self): - db = _make_db() - src = Source() - p = Person() - with DbTxn("s", db) as t: - db.add_source(src, t) - db.add_person(p, t) - phandle = p.get_handle() - - # name recno=3 maps to nper=5; per_no_map routes nper=5 to phandle - db = self._run( - [self._cit_rec(stype='N', refrec=3)], - names_records=[_Rec(dsid=1, recno=3, nper=5)], - source_handle_map={1: src.get_handle()}, - per_no_map={5: phandle}, - db=db, - ) - person = db.get_person_from_handle(phandle) - self.assertEqual(len(person.get_citation_list()), 1) - - def test_subsource_becomes_page(self): - db = _make_db() - src = Source() - with DbTxn("s", db) as t: - db.add_source(src, t) - db = self._run([self._cit_rec(subsource='p.42')], - source_handle_map={1: src.get_handle()}, db=db) - cit_handle = list(db.get_citation_handles())[0] - cit = db.get_citation_from_handle(cit_handle) - self.assertEqual(cit.get_page(), 'p.42') - - - -# ── TmgProject._read_pjc_config ────────────────────────────────────────── -def _make_minimal_sqz(tmp_dir, pjc_content): - """Create a minimal .SQZ zip containing only a PJC file.""" - import zipfile as _zf - pjc_path = os.path.join(tmp_dir, 'test.pjc') - sqz_path = os.path.join(tmp_dir, 'test.sqz') - with open(pjc_path, 'w', encoding='latin-1') as f: - f.write(pjc_content) - with _zf.ZipFile(sqz_path, 'w') as zf: - zf.write(pjc_path, 'test.pjc') - return sqz_path - - -class _MockUser: - """Minimal stand-in for the Gramps user object used in importData.""" - def __init__(self): - self.error_shown = False - self.error_message = None - self.uistate = None - - def notify_error(self, title, message=''): - self.error_shown = True - self.error_message = message - - def begin_progress(self, *a, **kw): pass - def end_progress(self): pass - def step_progress(self): pass - - -class TestReadPjcConfig(unittest.TestCase): - """Tests for TmgProject._read_pjc_config PJC parsing.""" - - _MINIMAL_PJC = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Researcher]\n" - "Name=Test User\n" - ) - - def _make_project(self, pjc_content, tmp_path): - pjc_file = os.path.join(tmp_path, "test.pjc") - with open(pjc_file, 'w', encoding='latin-1') as f: - f.write(pjc_content) - return libtmg.TmgProject(pjc_file) - - def test_well_formed_pjc_returns_version(self): - """A clean PJC file parses successfully and version() returns a float.""" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(self._MINIMAL_PJC, tmp) - self.assertEqual(project.version(), 11.0) - - def test_malformed_section_header_does_not_raise(self): - """Lines like '[Exho' (no closing bracket) are silently dropped.""" - pjc = ( - "[Stamp]\n" - "PjcVersion=11.0\n" - "[Exho\n" # malformed — the crash trigger - "SomeGarbage\n" - "[Researcher]\n" - "Name=Test User\n" - ) - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - # Must not raise; must still find [Stamp] - self.assertEqual(project.version(), 11.0) - - def test_null_bytes_stripped(self): - """NUL bytes in the PJC file are stripped before parsing.""" - pjc = "[Stamp]\x00\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - self.assertEqual(project.version(), 11.0) - - def test_parse_error_returns_partial_config(self): - """If configparser still raises after filtering, a warning is logged - and a (possibly empty) config object is returned rather than crashing.""" - import logging - # Feed content that survives the filter but still breaks configparser: - # a key=value line before any section header is technically invalid. - pjc = "orphan_key=value\n[Stamp]\nPjcVersion=11.0\n" - with tempfile.TemporaryDirectory() as tmp: - project = self._make_project(pjc, tmp) - with self.assertLogs('.TMGImport', level=logging.WARNING) as cm: - cfg = project._read_pjc_config() - self.assertTrue(any('parse error' in m.lower() or 'parsing' in m.lower() - for m in cm.output)) - # Config object is returned (not None), even if incomplete - self.assertIsNotNone(cfg) - - def test_version_too_old_notifies_user(self): - """A PJC version < 11.0 calls user.notify_error and aborts import.""" - pjc = "[Stamp]\nPjcVersion=10.0\n" # TMG 9.01 or earlier - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for old version") - self.assertIn('9.05', user.error_message or '', - "Error message should mention TMG 9.05") - - def test_missing_pjc_version_notifies_user(self): - """A PJC with no [Stamp]/PjcVersion calls user.notify_error.""" - pjc = "[OtherSection]\nSomeKey=value\n" # no [Stamp] at all - with tempfile.TemporaryDirectory() as tmp: - sqz = _make_minimal_sqz(tmp, pjc) - db = _make_db() - user = _MockUser() - libtmg.importData(db, sqz, user) - self.assertTrue(user.error_shown, - "notify_error should have been called for missing version") - -if __name__ == '__main__': - unittest.main() diff --git a/tests/test_addon_dependencies.py b/tests/test_addon_dependencies.py deleted file mode 100644 index 6baf2a876..000000000 --- a/tests/test_addon_dependencies.py +++ /dev/null @@ -1,421 +0,0 @@ -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2026 Eduard Ralph -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# - -""" -Detect addons that USE another addon without declaring it in ``depends_on``. - -This is the bug class behind Mantis 13707: the WebConnect packs imported -``libwebconnect`` at module load without declaring it, so installing a -pack without libwebconnect already present failed. - -The detector is INDEPENDENT of Gramps' plugin loader by design: - -* it reads ``.gpr.py`` with an exec-shim of its own (no - ``gramps.gen.plug._pluginreg`` import, no PluginManager, no - PluginRegister), because using Gramps' loader would test addons - through the very dependency resolver whose leniency lets the bug - ship — and would tie the test to Gramps' internal, unstable API; -* it loads each registered module in a fresh subprocess with - ``sys.path`` scoped to that addon's directory plus the directories - of its declared ``depends_on`` (so a real missing dep blows up the - way it would on a clean install), and parses the resulting - exception to classify it. - -The Gramps runtime is allowed to be importable from the subprocess — -addons do ``from gi.repository import Gtk`` and ``from gramps.gen.lib -import X`` at load time. The isolation being enforced is addon-from- -addon, not addon-from-Gramps. - -LIMITATION (important): -This catches undeclared addon dependencies that manifest at LOAD time -(top-level imports). It MISSES lazily-imported deps — e.g. a sibling -addon imported inside a function that is not called at module load. -No false positives, but not exhaustive — do not let it be mistaken -for one. - -Failures are bucketed: - -a. ``undeclared_addon_dep`` — the import error names a module that - another addon in this tree provides AND that this addon does not - declare in ``depends_on``. This is a FINDING and fails the test. -b. ``missing_requires_mod`` — the import error names a module the - addon declares in ``requires_mod`` (e.g. ``litellm``). That is an - environment concern, not a dependency-declaration bug; ignored. -c. ``other`` — any other isolated-load failure. Logged so the - information is not lost, but NOT a finding and NOT a test failure. - Examples: GI namespace mismatches, host-environment library issues, - addon import-time side effects requiring full GUI state. -""" - -# ------------------------ -# Python modules -# ------------------------ -import logging -import os -import re -import subprocess -import sys -import textwrap -import unittest -from typing import Any - -LOG = logging.getLogger(__name__) - -ADDONS_ROOT: str = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - -# ------------------------ -# .gpr.py exec shim -# ------------------------ -class _GprSentinel: - """Stands in for any unresolved plugin-type/category constant. - - The shim does not need real enum values — only kwargs captured by - a fake ``register()`` call. Attribute access, calls, and arithmetic - on the sentinel all return the sentinel so common patterns inside - ``.gpr.py`` files do not raise. - """ - - def __repr__(self) -> str: - return "" - - def __getattr__(self, name: str) -> "_GprSentinel": - return self - - def __call__(self, *args: Any, **kwargs: Any) -> "_GprSentinel": - return self - - -_SENT = _GprSentinel() - - -class _PermissiveGlobals(dict): - """Globals dict that returns a sentinel for any unknown plain name. - - CPython's ``LOAD_GLOBAL`` opcode calls ``__getitem__`` (and hence - ``__missing__``) on ``dict`` subclasses, so unresolved plugin-type - constants (``GRAMPLET``, ``REPORT``, ``CATEGORY_TEXT``, …) do not - raise ``NameError`` inside the exec. Dunder names still miss so - Python's own machinery behaves normally. - """ - - def __missing__(self, key: str) -> Any: - if key.startswith("__"): - raise KeyError(key) - return _SENT - - -def _exec_gpr(gpr_path: str) -> list[dict]: - """Exec one ``.gpr.py`` with a fake ``register()`` and return its kwargs. - - :param gpr_path: Absolute path to a ``*.gpr.py`` file. - :returns: One dict per ``register()`` call, with the kwargs verbatim - plus ``_ptype`` for the positional plugin type. - """ - plugins: list[dict] = [] - - def register(ptype: Any, **kwargs: Any) -> None: - kwargs["_ptype"] = ptype - plugins.append(kwargs) - - env = _PermissiveGlobals( - { - "__builtins__": __builtins__, - "__file__": gpr_path, - "__name__": "_gpr_shim", - "register": register, - "_": lambda s, *a, **k: s, - } - ) - with open(gpr_path, "r", encoding="utf-8") as f: - src = f.read() - exec(compile(src, gpr_path, "exec"), env) - return plugins - - -def _index_addons( - addons_root: str, -) -> tuple[dict[str, dict], set[str], list[tuple[str, str]]]: - """Walk ``addons_root`` and build the addon metadata index. - - :returns: ``(id_to_addon, all_modules, skipped)`` where - ``id_to_addon[plugin_id]`` is - ``{directory, modules, depends_on, requires_mod, gpr_files}``, - ``all_modules`` is the set of registered module names across - the whole tree, and ``skipped`` is a list of - ``(gpr_path, error)`` tuples for files whose exec raised. - """ - id_to_addon: dict[str, dict] = {} - all_modules: set[str] = set() - skipped: list[tuple[str, str]] = [] - - for dirname in sorted(os.listdir(addons_root)): - addon_dir = os.path.join(addons_root, dirname) - if not os.path.isdir(addon_dir): - continue - if dirname.startswith("."): - continue - gpr_files = sorted( - os.path.join(addon_dir, f) - for f in os.listdir(addon_dir) - if f.endswith(".gpr.py") - ) - if not gpr_files: - continue - for gpr in gpr_files: - try: - plugins = _exec_gpr(gpr) - except BaseException as exc: # noqa: BLE001 - skipped.append((gpr, f"{type(exc).__name__}: {exc}")) - continue - for plugin in plugins: - pid = plugin.get("id") - fname = plugin.get("fname") - if not isinstance(pid, str) or not isinstance(fname, str): - continue - module = re.sub(r"\.py$", "", fname) - rec = id_to_addon.setdefault( - pid, - { - "directory": addon_dir, - "modules": [], - "depends_on": [], - "requires_mod": [], - "gpr_files": [], - }, - ) - if module not in rec["modules"]: - rec["modules"].append(module) - for dep in plugin.get("depends_on") or []: - if isinstance(dep, str) and dep not in rec["depends_on"]: - rec["depends_on"].append(dep) - for req in plugin.get("requires_mod") or []: - if isinstance(req, str) and req not in rec["requires_mod"]: - rec["requires_mod"].append(req) - if gpr not in rec["gpr_files"]: - rec["gpr_files"].append(gpr) - all_modules.add(module) - return id_to_addon, all_modules, skipped - - -# ------------------------ -# Isolated-load subprocess -# ------------------------ -_LOADER = textwrap.dedent( - """ - import sys, importlib, traceback - # Strip the implicit "" CWD entry. Without this, if the subprocess - # is run from a directory that contains addon subdirectories, PEP - # 420 implicit namespace packages let sibling addons import as - # empty packages — which silently defeats the isolation we are - # trying to enforce. The caller also chdirs to a neutral directory, - # but stripping "" makes the isolation independent of CWD. - sys.path[:] = [p for p in sys.path if p not in ("", ".")] - # Pin the GI namespace versions Gramps itself pins before loading - # any plugin, so addons whose top-level `from gi.repository import X` - # is version-sensitive do not generate false (c) failures from - # ambiguous GI defaults. This is NOT Gramps' loader; it is matching - # the runtime conditions an addon is loaded under. - try: - import gi - for ns, ver in ( - ("Gtk", "3.0"), - ("PangoCairo", "1.0"), - ("OsmGpsMap", "1.0"), - ("GExiv2", "0.10"), - ("Gspell", "1"), - ("GeocodeGlib", "1.0"), - ): - try: - gi.require_version(ns, ver) - except (ValueError, AttributeError): - pass - except ImportError: - pass - target_dir = {target_dir!r} - dep_dirs = {dep_dirs!r} - sys.path[:0] = [target_dir] + list(dep_dirs) - try: - importlib.import_module({module!r}) - except BaseException: - traceback.print_exc() - sys.exit(2) - sys.exit(0) - """ -) - - -def _isolated_load( - target_dir: str, dep_dirs: list[str], module: str, timeout: int = 30 -) -> tuple[int, str]: - """Spawn a subprocess that tries to import ``module`` in isolation. - - The subprocess is run from a neutral CWD (the system temp dir) and - with ``PYTHONPATH`` stripped from the environment, so neither the - parent's working directory nor a stray ``PYTHONPATH`` can leak - sibling-addon paths into the child's ``sys.path``. - - :returns: ``(returncode, stderr)``. Returncode ``-1`` indicates - the subprocess timed out. - """ - code = _LOADER.format(target_dir=target_dir, dep_dirs=dep_dirs, module=module) - env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} - try: - proc = subprocess.run( - [sys.executable, "-I", "-c", code], - capture_output=True, - text=True, - timeout=timeout, - cwd="/tmp", - env=env, - ) - except subprocess.TimeoutExpired: - return -1, "TIMEOUT" - return proc.returncode, proc.stderr - - -_MISSING_NAME_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]") - - -def _classify( - stderr: str, - declared_dep_modules: set[str], - requires_mod: set[str], - all_addon_modules: set[str], -) -> tuple[str, str]: - """Bucket a failed isolated-load. - - A given failure may name multiple missing modules. Bucket (a) wins - over (b) wins over (c), so the highest-signal finding is reported. - """ - missing = [m.split(".")[0] for m in _MISSING_NAME_RE.findall(stderr)] - for name in missing: - if name in all_addon_modules and name not in declared_dep_modules: - return ("a_undeclared_addon_dep", name) - for name in missing: - if name in requires_mod: - return ("b_requires_mod", name) - last_lines = stderr.strip().split("\n")[-5:] - return ("c_other", last_lines[-1] if last_lines else "") - - -# ------------------------------------------------------------ -# -# TestAddonDependencies -# -# ------------------------------------------------------------ -class TestAddonDependencies(unittest.TestCase): - """Fail when any addon imports a sibling addon it does not declare.""" - - id_to_addon: dict[str, dict] = {} - all_modules: set[str] = set() - skipped_gpr: list[tuple[str, str]] = [] - - @classmethod - def setUpClass(cls) -> None: - super().setUpClass() - cls.id_to_addon, cls.all_modules, cls.skipped_gpr = _index_addons(ADDONS_ROOT) - - def test_no_undeclared_addon_dependencies(self) -> None: - """Every addon's registered modules must import in isolation, given - only the directories of its declared ``depends_on``. - - A failure that names a sibling-addon module not listed in - ``depends_on`` is a finding (the #13707 class). A failure that - names a declared ``requires_mod`` is the environment, not the - declaration. Anything else is logged separately. - """ - self.assertGreater( - len(self.id_to_addon), 0, "No addons found — index is empty" - ) - - findings_a: list[str] = [] - findings_c: list[str] = [] - counts = {"pass": 0, "a": 0, "b": 0, "c": 0} - - for pid in sorted(self.id_to_addon): - rec = self.id_to_addon[pid] - target_dir = rec["directory"] - dep_dirs: list[str] = [] - declared_modules: set[str] = set() - for dep_id in rec["depends_on"]: - dep_rec = self.id_to_addon.get(dep_id) - if dep_rec is None: - continue - if dep_rec["directory"] not in dep_dirs: - dep_dirs.append(dep_rec["directory"]) - declared_modules.update(dep_rec["modules"]) - requires_mod_set = set(rec["requires_mod"]) - - for module in rec["modules"]: - rc, err = _isolated_load(target_dir, dep_dirs, module) - if rc == 0: - counts["pass"] += 1 - continue - bucket, detail = _classify( - err, declared_modules, requires_mod_set, self.all_modules - ) - if bucket == "a_undeclared_addon_dep": - counts["a"] += 1 - findings_a.append( - f" {pid} (module {module}) — undeclared addon dep: " - f"{detail}" - ) - elif bucket == "b_requires_mod": - counts["b"] += 1 - else: - counts["c"] += 1 - findings_c.append(f" {pid} (module {module}) — {detail}") - - LOG.info( - "Indexed %d plugins; load pass=%d, bucket a=%d, b=%d, c=%d; " - "gpr exec skipped=%d", - len(self.id_to_addon), - counts["pass"], - counts["a"], - counts["b"], - counts["c"], - len(self.skipped_gpr), - ) - if self.skipped_gpr: - LOG.warning( - "%d .gpr.py file(s) failed exec-shim and were skipped:\n%s", - len(self.skipped_gpr), - "\n".join(f" {p}: {e}" for p, e in self.skipped_gpr), - ) - if findings_c: - LOG.warning( - "%d addon(s) failed isolated load for non-dependency reasons " - "(NOT a finding — environment / GUI-state / import-time side " - "effects):\n%s", - len(findings_c), - "\n".join(findings_c), - ) - - if findings_a: - self.fail( - "Found %d addon(s) that import a sibling addon without " - "declaring it in depends_on:\n%s" - % (len(findings_a), "\n".join(findings_a)) - ) - - -if __name__ == "__main__": - unittest.main() From 55db53f12f9f1cb95772f9dd7213b66bae9168c7 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 16 Jun 2026 18:14:52 +0200 Subject: [PATCH 23/47] Resolve tests/__init__.py add/add: adopt the maintenance/gramps60 GI-pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR added tests/__init__.py as a plain package marker; maintenance/gramps60 independently gained a tests/__init__.py that pins the GTK/GDK 3.0 introspection versions for the repo-root test suite. That add/add is the PR's only merge conflict. Adopt the maintenance/gramps60 version verbatim — the GI-version pin is the functional, canonical content and supersedes the bare marker — so the file is byte-identical on both sides and the conflict resolves with no merge commit. --- tests/__init__.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 6a28a3984..ff68cee43 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,21 +1,19 @@ -# -# Gramps - a GTK+/GNOME based genealogy program -# -# Copyright (C) 2026 Eduard Ralph -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# +"""Pin the GTK / GDK introspection versions for the repo-root test suite. -"""Package marker for the repo-wide Gramps addon test suite.""" +``python3 -m unittest discover -s tests`` imports this package before any test +module under ``tests/``, so requiring the versions here pins the whole suite to +the GTK 3 / GDK 3 stack a real Gramps GUI session uses. A test that imports a +``gramps.gui.*`` module directly never runs the launcher's own +``require_version``; without this the GI stack can resolve to GTK 4 on a host +where that is the default — emitting ``PyGIWarning`` and risking the wrong stack. +""" + +try: + import gi + + gi.require_version("Gdk", "3.0") + gi.require_version("Gtk", "3.0") +except Exception: + # No PyGObject, or the 3.0 typelibs are unavailable — leave the environment + # untouched; this only fixes the version when it can. + pass From 4fc07ba619df9abe04eca75ca08ca26dee1a0195 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Tue, 16 Jun 2026 20:52:40 +0200 Subject: [PATCH 24/47] Gate addon plugin-load test on hard load failures The addon CI suite has a load check, test_load_all_addon_modules, that collects every addon that fails to load into a "hard failures" list but then only logs a warning about them. Its only assertion is that at least one plugin was found, so a real addon that fails to load lets CI pass green while the failure scrolls by unread. The check named a failure class it could never fail on. Make that check gate, the way its two siblings in the same file already do: TestImportPluginSmoke and TestExportPluginSmoke both self.fail on their findings. A non-dependency hard load failure now fails the test directly; dependency skips and subprocess crashes (typically a missing display server in CI) stay advisory and are still only logged. A regression test drives the real production method with a synthetic always-failing addon injected at its load seams and asserts the run now fails rather than passing silently. --- tests/test_plugin_load_gate.py | 150 ++++++++++++++++++++++++++++++ tests/test_plugin_registration.py | 18 +++- 2 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 tests/test_plugin_load_gate.py diff --git a/tests/test_plugin_load_gate.py b/tests/test_plugin_load_gate.py new file mode 100644 index 000000000..020389c72 --- /dev/null +++ b/tests/test_plugin_load_gate.py @@ -0,0 +1,150 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Regression test for the addon plugin-load *gating* decision (PR #820, R-C). + +``TestPluginLoading.test_load_all_addon_modules`` once named ``hard_failures`` +but only ``LOG.warning``-ed them, so a genuine non-dependency load failure +passed silently — its sole assertion was ``assertGreater(len(plugins), 0)``. +These cases drive that **production method itself**, with a synthetic addon +injected at its load seams, and assert that a non-dependency hard failure now +fails the test (``self.fail``) like the sibling smoke tests. + +Why a dedicated module rather than a case inside +``tests/test_plugin_registration.py`` (the brief's named test file): the C4 +runner executes the *whole* selected test module. Running +``test_plugin_registration`` there pulls in the real, registry-backed +``test_load_all_addon_modules``, which — now that it gates — fails on purely +environmental addon-load gaps in a minimal CI image (e.g. a missing GTK icon +theme: "Icon 'stock_link' not present"). That conflates this fix with the +image's addon completeness and is flaky run-to-run. Isolating the regression +here keeps the gate's verification deterministic while still exercising the +real production code path (the production change itself lives in +``tests/test_plugin_registration.py`` exactly as the brief directs). + +Import-light: this module imports the production module (``gramps.gen``-level, +no ``gi``/``gramps.gui`` at load) but runs only the mocked path — no Gramps +plugin registry is booted. +""" + +# ------------------------ +# Python modules +# ------------------------ +from types import SimpleNamespace +from unittest import mock +import os +import unittest + +# ------------------------ +# Gramps specific +# ------------------------ +# NOTE: import the production module, *not* its TestCase classes by name. A bare +# ``from … import TestPluginLoading`` would bind that heavy, registry-backed class +# into this module's namespace, and ``python3 -m unittest tests.test_plugin_load_gate`` +# would then collect and run its real ``test_load_all_addon_modules`` here (booting +# the full plugin load). Reaching the class through ``prod`` keeps only this file's +# own cases discoverable while still driving the real production method. +from tests import test_plugin_registration as prod + + +# ------------------------------------------------------------ +# +# TestPluginLoadingGate +# +# ------------------------------------------------------------ +class TestPluginLoadingGate(unittest.TestCase): + """The load check must *gate* on non-dependency failures, not just log them. + + Each case constructs a real :class:`TestPluginLoading` instance and calls + its production method ``test_load_all_addon_modules`` with one synthetic + addon injected at the load seams (``_get_addon_plugins`` / + ``_check_dependencies`` / ``subprocess.run`` in the production module's own + namespace) — not a copy of the logic and not just the extracted helper — so + the ``self.fail`` wiring is proven end-to-end on the real code path. Plain + :class:`unittest.TestCase`: the registry is stubbed out, so nothing boots. + """ + + @staticmethod + def _fake_plugin() -> SimpleNamespace: + """A minimal :class:`PluginData` stand-in with no declared dependencies.""" + return SimpleNamespace( + id="synthetic_broken_addon", + fpath=os.path.join(prod.ADDONS_ROOT, "SyntheticBrokenAddon"), + include_in_listing=True, + requires_mod=[], + requires_exe=[], + requires_gi=[], + ) + + def _run_load_test_with(self, run_result: SimpleNamespace): + """Drive ``test_load_all_addon_modules`` with one synthetic plugin. + + The synthetic addon has no declared dependencies, so it is not skipped; + ``subprocess.run`` is stubbed to ``run_result`` to simulate a chosen load + outcome. Returns the exception the production method raised, or ``None`` + if it returned without raising (the silent-pass behaviour). + """ + loader = prod.TestPluginLoading("test_load_all_addon_modules") + with mock.patch.object( + prod, "_get_addon_plugins", return_value=[self._fake_plugin()] + ), mock.patch.object( + prod, "_check_dependencies", return_value=[] + ), mock.patch.object( + prod.subprocess, "run", return_value=run_result + ): + try: + loader.test_load_all_addon_modules() + except Exception as exc: # noqa: BLE001 - asserted on by type below + return exc + return None + + def test_hard_failure_gates_the_load_test(self) -> None: + """A synthetic non-dependency load failure must fail the load test. + + With the silent-warning behaviour the production method returned + normally; the gate must instead raise the test's ``failureException`` + (``self.fail``), so this case is red until the gate exists. + """ + outcome = self._run_load_test_with( + SimpleNamespace( + returncode=1, + stderr="ModuleNotFoundError: No module named 'totally_missing_dep'", + ) + ) + self.assertIsInstance( + outcome, + prod.TestPluginLoading.failureException, + "a non-dependency load failure must gate the run (self.fail), " + "not pass silently", + ) + self.assertIn("synthetic_broken_addon", str(outcome)) + self.assertIn("failed to load", str(outcome)) + + def test_clean_load_does_not_gate(self) -> None: + """A plugin that loads cleanly must not fail the load test.""" + outcome = self._run_load_test_with(SimpleNamespace(returncode=0, stderr="")) + self.assertIsNone( + outcome, f"a clean load must not gate, but raised: {outcome!r}" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_plugin_registration.py b/tests/test_plugin_registration.py index b2f2f4dde..39d5192e2 100644 --- a/tests/test_plugin_registration.py +++ b/tests/test_plugin_registration.py @@ -161,7 +161,16 @@ class TestPluginLoading(GrampsTestCase): """ def test_load_all_addon_modules(self) -> None: - """Try to load every addon plugin; collect failures rather than fail fast.""" + """Load every addon plugin; gate on non-dependency load failures. + + Failures are collected rather than failing fast, then classified: + dependency skips and subprocess crashes (typically a missing display + server in CI) are advisory and only logged, while a non-dependency + *hard* load failure fails the test (``self.fail``). This makes + the check a real gate — like the sibling smoke tests + (:class:`TestImportPluginSmoke`, :class:`TestExportPluginSmoke`) — not + an always-pass that merely warns on the failure class it names. + """ plugins = _get_addon_plugins(self.plugin_registry) self.assertGreater(len(plugins), 0, "No addon plugins found to test") @@ -222,10 +231,9 @@ def test_load_all_addon_modules(self) -> None: ) if hard_failures: - LOG.warning( - "%d addon(s) failed to load:\n %s", - len(hard_failures), - "\n ".join(hard_failures), + self.fail( + f"{len(hard_failures)} addon(s) failed to load:\n " + + "\n ".join(hard_failures) ) From 95b77d1ad50acfe9d1d82e5e35636fbcd89e5df7 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Fri, 19 Jun 2026 00:02:23 +0200 Subject: [PATCH 25/47] Provision source-built addon deps in CI instead of silent skip The CI image purged the build toolchain (gcc, python3-dev, pkg-config) right after the Gramps install, and the source-built addon requires_mod that have no wheel on a CI platform (pygraphviz, psycopg2, psycopg) had no system package provisioned on either lane. So pip installing those at CI runtime failed for a missing compiler or header/library, the failure was swallowed by the install step's "|| echo ... (continuing)", and an addon that hard-imports them (e.g. NetworkChart, the PostgreSQL backends) ran a silently degraded suite while the job still reported green. Keep the build toolchain in the image and derive each source-built module's system package from the addons' .gpr.py through the same single-source map as requires_gi/requires_exe, installing it before the runtime pip step on both lanes: the apt lane gets the -dev headers/libpq and compiles the extension, and the conda lane installs the prebuilt conda-forge package so the Windows suites run instead of skipping. A declared requires_mod must now be classified as wheel-only or source-built, and the mapping drift guard fails CI on any module that is neither, so a newly added source-built dependency cannot quietly reopen the coverage gap. The affected addon suites either run with their deps present or fail honestly when a dep cannot be provisioned -- never a silent green. --- .github/docker/gramps-ci/Dockerfile | 13 +- .github/scripts/addon_system_deps.py | 93 +++++++-- .github/workflows/ci.yml | 58 ++++-- tests/test_addon_system_deps.py | 286 +++++++++++++++++++++++++++ 4 files changed, 414 insertions(+), 36 deletions(-) create mode 100644 tests/test_addon_system_deps.py diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index 38a4e9a28..f65a5f949 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -108,7 +108,18 @@ else fi BASH -RUN apt-get purge -y gcc python3-dev pkg-config && apt-get autoremove -y +# NOTE: the build toolchain (gcc, python3-dev, pkg-config) is deliberately KEPT +# in the image — it is not purged here. Addon requires_mod are pip-installed at CI +# runtime (ci.yml "Install addon runtime deps"), and the source-built ones with no +# wheel on the apt lane (pygraphviz, psycopg2) compile a C extension during that +# step. Purging gcc/pkg-config left those builds with no compiler, so they failed +# and were swallowed by the install step's "|| echo … (continuing)" — silently +# degrading the affected addon's coverage while the job stayed green. The per- +# package system headers those builds link against (libgraphviz-dev, libpq-dev on +# apt) are derived from the addons' .gpr.py and installed at CI runtime by the same +# single-source map as requires_gi/requires_exe — see .github/scripts/ +# addon_system_deps.py (MOD_BUILD_PACKAGES) and ci.yml's "Install addon system +# deps" step — so .gpr.py stays the single source of truth. RUN python -c "from gramps.gen.const import VERSION; print('Gramps', VERSION)" \ && python -c "import gi; gi.require_version('Gtk', '3.0'); from gi.repository import Gtk; print('GTK OK')" diff --git a/.github/scripts/addon_system_deps.py b/.github/scripts/addon_system_deps.py index 6e3cef514..3c3480ee4 100644 --- a/.github/scripts/addon_system_deps.py +++ b/.github/scripts/addon_system_deps.py @@ -3,16 +3,23 @@ Addons declare three dependency kinds in their ``.gpr.py``: -* ``requires_mod`` — importable Python modules. pip-installable; ci.yml already - auto-derives these from the ``.gpr.py`` files. Nothing to do here. +* ``requires_mod`` — importable Python modules. pip-installable; ci.yml + auto-derives these from the ``.gpr.py`` files. Most ship a plain wheel and need + nothing more, but a few have no wheel on a CI platform and build from / link + against a system library — ``pygraphviz`` (graphviz), ``psycopg2`` / ``psycopg`` + (libpq). Those need a system package before the pip step can satisfy them, + mapped in ``MOD_BUILD_PACKAGES`` below. * ``requires_gi`` — GObject-introspection typelibs (e.g. ``GooCanvas``). * ``requires_exe`` — system executables (e.g. ``dot`` from graphviz). -The latter two are *system* packages: not pip-installable, named differently per -platform, and Gramps' own ``Requirements`` only *checks* them (never installs). -This module maps each declared ``requires_gi`` namespace / ``requires_exe`` name -to its package on each CI platform and scans the addons for what they declare, -so ci.yml derives the install list from one place instead of a hand-kept list. +All three system kinds above (the GI typelibs, the executables, and the +source-built ``requires_mod`` system packages) are not pip-installable as named, +are named differently per platform, and Gramps' own ``Requirements`` only +*checks* ``requires_mod`` (never installs its system side). This module maps each +declared ``requires_gi`` namespace / ``requires_exe`` name / source-built +``requires_mod`` to its package on each CI platform and scans the addons for what +they declare, so ci.yml derives the install list from one place instead of a +hand-kept list. Platform availability is asymmetric and encoded here: the GTK 3 addon libs (goocanvas, osm-gps-map, gexiv2) exist on Debian/apt but **not on conda-forge**, @@ -25,7 +32,7 @@ addon_system_deps.py --platform apt # space-separated install list addon_system_deps.py --platform conda # (only packages available there) - addon_system_deps.py --unmapped . # declared deps with no map entry; exit 1 if any + addon_system_deps.py --unmapped . # declared GI/exe/mod with no map entry; exit 1 if any """ # ------------------------ @@ -62,6 +69,49 @@ "dot": {"apt": "graphviz", "conda": "graphviz"}, } +# Source-built / system-library requires_mod -> the package that makes the +# module installable+importable, per platform. A requires_mod belongs here ONLY +# when a plain ``pip install `` cannot satisfy it by itself on a CI platform: +# * apt — no wheel, so pip compiles the C extension from source against a -dev +# header (``pygraphviz`` -> libgraphviz-dev; ``psycopg2`` -> libpq-dev), or the +# pure-Python build links a system shared library at import time (``psycopg`` / +# psycopg3 -> libpq, provided by libpq-dev). The generic compiler toolchain +# (gcc, python3-dev, pkg-config) those source builds need stays in the CI +# image (.github/docker/gramps-ci/Dockerfile no longer purges it). +# * conda — conda-forge ships the whole binding prebuilt, so the value is the +# module's own conda-forge package; the conda lane ``mamba install``s it and +# the later pip step finds it already satisfied. Verified present on +# conda-forge win-64: pygraphviz, psycopg2, psycopg. +# Both platforms must PROVISION the dep or fail the install step honestly — a +# value the platform cannot resolve aborts the job (apt-get install / mamba +# install) rather than letting ci.yml's "|| echo … (continuing)" swallow a failed +# build into a silently-degraded green. A genuinely unprovisionable module would +# be a None, but pygraphviz/psycopg2/psycopg are available on both apt and +# conda-forge. +MOD_BUILD_PACKAGES: dict[str, dict[str, str | None]] = { + "pygraphviz": {"apt": "libgraphviz-dev", "conda": "pygraphviz"}, + "psycopg2": {"apt": "libpq-dev", "conda": "psycopg2"}, + "psycopg": {"apt": "libpq-dev", "conda": "psycopg"}, # psycopg3 +} + +# requires_mod that ship a plain pip wheel needing no system package on any CI +# platform. Listed explicitly so the drift guard can tell a wheel-only module +# (nothing to map) from a source-built one that was forgotten: every declared +# requires_mod must be classified as exactly one of WHEEL_ONLY_MODS or +# MOD_BUILD_PACKAGES, else a newly-added source-built dep could silently lose +# coverage again (the build-toolchain gap this module closes). ``--unmapped`` +# fails CI on any requires_mod that is in neither set. +WHEEL_ONLY_MODS: frozenset[str] = frozenset( + { + "boto3", # S3MediaUploader — pure-Python AWS SDK wheel + "dbf", # TMGimporter — pure-Python wheel + "life_line_chart", # LifeLineChartView — pure-Python wheel + "litellm", # ChatWithTree / GrampsChat — pure-Python wheel + "networkx", # NetworkChart — pure-Python wheel + "svgwrite", # LifeLineChartView — pure-Python wheel + } +) + PLATFORMS = ("apt", "conda") @@ -72,6 +122,7 @@ # ------------------------------------------------------------ _GI_RE = re.compile(r"requires_gi\s*=\s*(\[[^\]]*\])") _EXE_RE = re.compile(r"requires_exe\s*=\s*(\[[^\]]*\])") +_MOD_RE = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") def _gpr_files(root: str) -> list[str]: @@ -109,6 +160,10 @@ def scan_executables(root: str) -> set[str]: return _scan(root, _EXE_RE, first_of_tuple=False) +def scan_modules(root: str) -> set[str]: + return _scan(root, _MOD_RE, first_of_tuple=False) + + def addon_requirements(addon_dir: str) -> tuple[set[str], set[str]]: """Return (gi_namespaces, executables) declared by a single addon dir.""" gi: set[str] = set() @@ -138,7 +193,7 @@ def addon_requirements(addon_dir: str) -> tuple[set[str], set[str]]: def packages(platform: str) -> list[str]: """All install-by-name packages available for a platform (full mapped set).""" pkgs: list[str] = [] - for table in (GI_PACKAGES, EXE_PACKAGES): + for table in (GI_PACKAGES, EXE_PACKAGES, MOD_BUILD_PACKAGES): for entry in table.values(): pkg = entry.get(platform) if pkg: @@ -146,11 +201,19 @@ def packages(platform: str) -> list[str]: return sorted(set(pkgs)) -def unmapped(root: str) -> tuple[set[str], set[str]]: - """Declared deps with no entry in the maps at all (drift).""" +def unmapped(root: str) -> tuple[set[str], set[str], set[str]]: + """Declared deps with no entry in the maps at all (drift). + + The third element is every declared ``requires_mod`` classified as *neither* + a wheel-only module nor a source-built one — an unclassified module that, if + it turns out to need a system package, would silently lose coverage. CI fails + on it so a human must classify it (add to ``WHEEL_ONLY_MODS`` or + ``MOD_BUILD_PACKAGES``). + """ return ( scan_gi_namespaces(root) - set(GI_PACKAGES), scan_executables(root) - set(EXE_PACKAGES), + scan_modules(root) - WHEEL_ONLY_MODS - set(MOD_BUILD_PACKAGES), ) @@ -185,17 +248,19 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--unmapped", metavar="ROOT", - help="print declared GI/exe deps with no map entry; exit 1 if any", + help="print declared GI/exe/mod deps with no map entry; exit 1 if any", ) args = parser.parse_args(argv) if args.unmapped is not None: - gi, exe = unmapped(args.unmapped) + gi, exe, mod = unmapped(args.unmapped) for ns in sorted(gi): print(f"gi:{ns}") for name in sorted(exe): print(f"exe:{name}") - return 1 if (gi or exe) else 0 + for name in sorted(mod): + print(f"mod:{name}") + return 1 if (gi or exe or mod) else 0 if args.platform: print(" ".join(packages(args.platform))) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d606be7b9..c69c7c7f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -198,12 +198,16 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install addon system deps (derived from requires_gi / requires_exe) - # System deps (GI typelibs, executables) are not pip-installable and - # gramps only *checks* them, so the image cannot bake them generically - # (its build context excludes addons-source). Derive the apt set from - # every .gpr.py via the single-source map and install it — the container - # runs as root. Mirrors the requires_mod derivation below. + - name: Install addon system deps (requires_gi / requires_exe / source-built requires_mod) + # System deps (GI typelibs, executables, and the -dev headers / libpq that + # source-built requires_mod link against — pygraphviz→libgraphviz-dev, + # psycopg2/psycopg→libpq-dev) are not pip-installable as named and gramps + # only *checks* requires_mod, so the image cannot bake them generically + # (its build context excludes addons-source). Derive the apt set from every + # .gpr.py via the single-source map and install it — the container runs as + # root. MUST run before "Install addon runtime deps" so the headers are + # present when pip compiles the source-built modules; the compiler toolchain + # those builds need lives in the CI image (no longer purged there). shell: bash run: | pkgs=$(python3 .github/scripts/addon_system_deps.py --platform apt) @@ -212,18 +216,21 @@ jobs: apt-get update apt-get install -y --no-install-recommends $pkgs else - echo "no requires_gi / requires_exe declarations found" + echo "no requires_gi / requires_exe / source-built requires_mod declarations found" fi - name: Validate addon system deps are mapped - # Every requires_gi / requires_exe an addon declares must have an entry - # in addon_system_deps.py, so the install list never silently drifts - # from what addons declare (the system-dep analogue of the requires_mod - # find_spec gate below). + # Every requires_gi / requires_exe an addon declares must have an entry in + # addon_system_deps.py, and every requires_mod must be classified as either + # wheel-only or source-built (with a system package), so the install list + # never silently drifts from what addons declare — the build-toolchain + # coverage gap returns the moment a new source-built requires_mod is added + # without a mapping. (The system-dep analogue of the requires_mod find_spec + # gate below.) shell: bash run: | python3 .github/scripts/addon_system_deps.py --unmapped . || { - echo "::error::Addon(s) declare requires_gi/requires_exe with no entry in .github/scripts/addon_system_deps.py — add a mapping row." + echo "::error::Addon(s) declare requires_gi/requires_exe/requires_mod with no entry in .github/scripts/addon_system_deps.py — add a mapping row (or, for a wheel-only module, list it in WHEEL_ONLY_MODS)." exit 1 } @@ -426,12 +433,19 @@ jobs: echo "::warning::Windows lane: branch targets gramps $want but conda-forge ships $have; addons here are validated against $have. Full $want coverage is on the Linux lane (its CI image git-builds $want). See step comment for why conda-Windows cannot build $want." fi - - name: Install addon system deps (derived, conda-available subset) - # Only the conda-forge-available subset (e.g. graphviz). The GTK 3 addon - # GI libs (goocanvas/osm-gps-map/gexiv2) are NOT on conda-forge, so the - # map returns None for them on conda and they are not installed; addons - # needing them skip on Windows by necessity, which run_addon_tests - # tolerates (--platform conda). + - name: Install addon system deps (requires_gi / requires_exe / source-built requires_mod) + # The conda-forge-available subset of the single-source map. This lane + # provisions its OWN deps and must mirror the apt lane: source-built + # requires_mod that conda-forge ships prebuilt — pygraphviz, psycopg2, + # psycopg — are installed here as full conda-forge packages + # (MOD_BUILD_PACKAGES' conda side), so the later pip step finds them already + # satisfied and the affected addons' suites RUN on Windows instead of + # silently skipping. `mamba install` fails the job if a mapped package + # cannot be resolved, so a provisioning gap aborts honestly — never a silent + # green. The GTK 3 addon GI libs (goocanvas/osm-gps-map/gexiv2) are + # genuinely NOT on conda-forge, so the map keeps them at None and they are + # not installed; addons needing only those skip on Windows by necessity, + # which run_addon_tests tolerates for declared GI deps (--platform conda). run: | pkgs=$(python .github/scripts/addon_system_deps.py --platform conda) if [ -n "$pkgs" ]; then @@ -570,12 +584,14 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install addon system deps (derived from requires_gi / requires_exe) + - name: Install addon system deps (requires_gi / requires_exe / source-built requires_mod) # Same as unit-test-linux: derive the apt set from the single-source # map and install it (container runs as root). The plugin registration # test subprocess-loads each addon module, so the GI typelibs those - # modules import must be present here too. (Mapping is drift-guarded in - # unit-test-linux, which this job needs:, so no duplicate gate here.) + # modules import — and the -dev headers / libpq their source-built + # requires_mod compile/link against — must be present here too. (Mapping is + # drift-guarded in unit-test-linux, which this job needs:, so no duplicate + # gate here.) shell: bash run: | pkgs=$(python3 .github/scripts/addon_system_deps.py --platform apt) diff --git a/tests/test_addon_system_deps.py b/tests/test_addon_system_deps.py new file mode 100644 index 000000000..b22e9ea55 --- /dev/null +++ b/tests/test_addon_system_deps.py @@ -0,0 +1,286 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +"""Source-built addon deps must be provisioned in CI — never a silent skip. + +Regression test for the build-toolchain coverage gap (addons-source PR #820): +the CI image purged the build toolchain (gcc/python3-dev/pkg-config) after the +Gramps install, and the system packages a source-built ``requires_mod`` needs +were not provisioned on either lane. So ``pip install pygraphviz`` / ``psycopg2`` +/ ``psycopg`` failed at CI runtime, the failure was swallowed by the install +step's ``|| echo … (continuing)``, and the affected addon's coverage degraded +while the job stayed green — silent coverage loss reported as success. + +The fix restores the invariant *declared addon dependencies are honestly +satisfied or honestly skipped* over the WHOLE category of source-built +``requires_mod``, on BOTH CI lanes: + +* apt — every source-built ``requires_mod`` has its ``-dev`` / libpq package in + the single-source map (``addon_system_deps.MOD_BUILD_PACKAGES``), and the + compiler toolchain stays in the CI image + (``.github/docker/gramps-ci/Dockerfile`` no longer purges it), so pip builds / + links the extension. +* conda — the same modules map to their prebuilt conda-forge package, so the + Windows lane installs them (``mamba install``) instead of silently skipping; + the conda side is NOT ``None``. + +It also closes the category over *future* additions: every declared +``requires_mod`` must be classified (wheel-only or source-built), and the +``--unmapped`` drift guard fails CI on any module that is neither — so a +newly-added source-built dep cannot quietly reopen the gap. + +Why this lives in ``tests/`` and not ``.github/scripts/tests/``: the C4 +red→green runner derives the unittest module name from the test path +(``path.replace('/', '.')``); a ``.github/``-rooted path yields a leading-dot +module name that ``python3 -m unittest`` rejects. ``tests/`` is the repo's real +test package (alongside ``test_plugin_load_gate.py``), so the module name is +importable. Pure stdlib / no ``gi`` / no ``gramps.gui`` imports — it runs under +the headless runner. It exercises the production derivation path (the same +``packages()`` / ``unmapped()`` / CLI that ci.yml calls), not a copy of it. +""" + +# ------------------------ +# Python modules +# ------------------------ +from __future__ import annotations + +import ast +import glob +import io +import os +import re +import sys +import unittest +from contextlib import redirect_stdout + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_HERE) # repo root (tests/ lives directly under it) +_SCRIPTS = os.path.join(_REPO_ROOT, ".github", "scripts") +_DOCKERFILE = os.path.join(_REPO_ROOT, ".github", "docker", "gramps-ci", "Dockerfile") + +sys.path.insert(0, _SCRIPTS) +import addon_system_deps as deps # noqa: E402 + +# The requires_mod that are built from / linked against a system package in CI +# (no plain pip wheel that imports unaided on at least one lane). Kept here as the +# test's own statement of the category, independent of the production map, so a +# regression that drops an entry from MOD_BUILD_PACKAGES is caught rather than +# mirrored. ``psycopg`` (psycopg3, PostgreSQLEnhanced) is included alongside +# ``psycopg2``/``pygraphviz`` — its pure-Python build still needs libpq at import. +_SOURCE_BUILT_MODS = ("pygraphviz", "psycopg2", "psycopg") + +_REQUIRES_MOD_RE = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + + +def _declared_requires_mod() -> set[str]: + """Union of requires_mod across every .gpr.py in the repo (as ci.yml derives).""" + mods: set[str] = set() + for path in glob.glob(os.path.join(_REPO_ROOT, "*", "*.gpr.py")): + try: + with open(path, encoding="utf-8") as fh: + text = fh.read() + except OSError: + continue + for match in _REQUIRES_MOD_RE.finditer(text): + try: + mods.update(ast.literal_eval(match.group(1))) + except (ValueError, SyntaxError): + pass + return mods + + +class SourceBuiltModBuildDeps(unittest.TestCase): + """The install list must carry every source-built requires_mod, per lane.""" + + def test_known_source_built_mods_are_declared_by_some_addon(self): + # Grounds the rest of the suite in real declarations: if these addons ever + # drop the dep this test would otherwise pass vacuously. + declared = _declared_requires_mod() + for mod in _SOURCE_BUILT_MODS: + self.assertIn( + mod, + declared, + f"expected some addon's .gpr.py to declare requires_mod={mod!r}", + ) + + def test_packages_apt_includes_build_headers(self): + apt = deps.packages("apt") + for mod in _SOURCE_BUILT_MODS: + pkg = deps.MOD_BUILD_PACKAGES.get(mod, {}).get("apt") + self.assertIsNotNone( + pkg, + f"source-built requires_mod {mod!r} has no apt package in " + "MOD_BUILD_PACKAGES — its CI pip build/import will fail for a " + "missing header/library", + ) + self.assertIn( + pkg, + apt, + f"{pkg!r} (system package for {mod!r}) missing from packages('apt'); " + "ci.yml would not install it and the source build would be silently skipped", + ) + + def test_packages_conda_provisions_source_built_mods(self): + # The conda (Windows) lane provisions its own deps and must mirror apt: + # pygraphviz/psycopg2/psycopg ship prebuilt on conda-forge, so the conda + # side is the module's own conda-forge package — NEVER None (the + # silent-skip the invariant forbids over the whole category). + conda = deps.packages("conda") + for mod in _SOURCE_BUILT_MODS: + pkg = deps.MOD_BUILD_PACKAGES.get(mod, {}).get("conda") + self.assertIsNotNone( + pkg, + f"source-built requires_mod {mod!r} maps to conda=None — the conda " + "lane would not install it and the failed pip build would be " + "swallowed into a silently-degraded green. It is on conda-forge; map it.", + ) + self.assertIn( + pkg, + conda, + f"{pkg!r} (conda package for {mod!r}) missing from packages('conda'); " + "the Windows lane would skip the addon's suite instead of running it", + ) + + def test_cli_apt_emits_build_headers(self): + # Exercise the exact production entry point ci.yml calls: + # pkgs=$(python3 addon_system_deps.py --platform apt) + buf = io.StringIO() + with redirect_stdout(buf): + rc = deps.main(["--platform", "apt"]) + self.assertEqual(rc, 0) + emitted = buf.getvalue().split() + self.assertIn("libgraphviz-dev", emitted) + self.assertIn("libpq-dev", emitted) + + def test_cli_conda_emits_source_built_mods(self): + # The conda step calls: pkgs=$(python addon_system_deps.py --platform conda) + buf = io.StringIO() + with redirect_stdout(buf): + rc = deps.main(["--platform", "conda"]) + self.assertEqual(rc, 0) + emitted = buf.getvalue().split() + for mod in _SOURCE_BUILT_MODS: + self.assertIn( + mod, + emitted, + f"conda --platform output is missing {mod!r}; the Windows lane " + "would not install it", + ) + + def test_every_declared_source_built_mod_is_provisioned_on_both_lanes(self): + # Category guard: any source-built requires_mod an addon declares must be + # mapped AND surfaced on BOTH lanes, so a newly added one cannot silently + # lose coverage on apt or on conda. + apt = set(deps.packages("apt")) + conda = set(deps.packages("conda")) + for mod in _declared_requires_mod() & set(_SOURCE_BUILT_MODS): + entry = deps.MOD_BUILD_PACKAGES[mod] + for platform, available in (("apt", apt), ("conda", conda)): + pkg = entry.get(platform) + self.assertIsNotNone( + pkg, + f"{mod!r} maps to {platform}=None — silent skip on {platform}", + ) + self.assertIn(pkg, available) + + +class RequiresModCategoryIsComplete(unittest.TestCase): + """Every declared requires_mod must be classified — no silent new gap.""" + + def test_every_declared_requires_mod_is_classified(self): + # The heart of C5(b): each declared requires_mod is EITHER a known + # wheel-only module OR a mapped source-built one. An unclassified module + # is exactly how a future source-built dep would silently lose coverage. + classified = set(deps.WHEEL_ONLY_MODS) | set(deps.MOD_BUILD_PACKAGES) + for mod in _declared_requires_mod(): + self.assertIn( + mod, + classified, + f"requires_mod {mod!r} is in neither WHEEL_ONLY_MODS nor " + "MOD_BUILD_PACKAGES — classify it (a source-built one needs a " + "system-package mapping or its CI coverage silently degrades)", + ) + + def test_unmapped_reports_no_mod_drift(self): + # Drive the production drift guard ci.yml's "Validate addon system deps + # are mapped" step runs (python3 addon_system_deps.py --unmapped .). It + # must report no unmapped requires_mod for the current addon set. + _gi, _exe, mod = deps.unmapped(_REPO_ROOT) + self.assertEqual( + mod, + set(), + f"--unmapped reports unclassified requires_mod {sorted(mod)!r}; " + "ci.yml's mapping gate would fail. Classify each as wheel-only or " + "source-built.", + ) + + def test_unmapped_cli_exit_zero(self): + # The full CLI the validate step invokes must exit 0 (no drift) today. + buf = io.StringIO() + with redirect_stdout(buf): + rc = deps.main(["--unmapped", _REPO_ROOT]) + self.assertEqual( + rc, + 0, + "addon_system_deps.py --unmapped exits non-zero — an addon declares a " + f"GI/exe/mod dep with no mapping:\n{buf.getvalue()}", + ) + + +class CiImageKeepsBuildToolchain(unittest.TestCase): + """The image must keep the compiler toolchain the apt builds use.""" + + @classmethod + def setUpClass(cls): + with open(_DOCKERFILE, encoding="utf-8") as fh: + cls.lines = [ + ln + for ln in fh.read().splitlines() + if ln.strip() and not ln.lstrip().startswith("#") + ] + + def test_toolchain_not_purged(self): + # The defect was a `RUN apt-get purge -y gcc python3-dev pkg-config` after + # the Gramps install, removing the compiler before CI-runtime source + # builds of requires_mod run. + for line in self.lines: + if "apt-get purge" in line or "apt-get remove" in line: + for tool in ("gcc", "pkg-config", "python3-dev"): + self.assertNotIn( + tool, + line, + f"Dockerfile purges build toolchain ({tool!r}) — " + "CI-runtime source builds of requires_mod will have no compiler:\n" + f" {line.strip()}", + ) + + def test_toolchain_is_installed(self): + text = "\n".join(self.lines) + for tool in ("gcc", "pkg-config"): + self.assertIn( + tool, + text, + f"Dockerfile no longer installs {tool!r}; source-built requires_mod " + "cannot compile in CI", + ) + + +if __name__ == "__main__": + unittest.main() From 67ed38d39e0f2c5c8d145433d94a78073f8ce341 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Fri, 19 Jun 2026 00:13:27 +0200 Subject: [PATCH 26/47] ci: single-source requires_mod derivation and is_active helper The addon CI workflow inlined the same requires_mod derivation as an identical Python heredoc in three jobs (unit-test-linux, unit-test-windows, integration-test) and the is_active() bash filter verbatim across six job steps, with the find_spec name-gate triplicated alongside. A one-line change to any of them meant a three- to six-site edit, and the copies could silently diverge. Move the requires_mod derivation and the find_spec validator into a single .github/scripts/addon_python_deps.py that every job calls (--install-list / --check-resolves), and the is_active() filter into .github/scripts/active_addons.sh that each filtering step sources. The .gpr.py files stay the single source of truth, so the derived module list and the active-addon set are unchanged. The module also centralises the import-to-distribution install map (PIL -> Pillow) on the install side only; the find_spec gate keeps validating the raw declared import name, exactly as Gramps does at runtime via check_mod(). No requires_mod=["PIL"] exists in the tree today, so the derived install list is byte-identical to the old heredoc output and the change is behaviour-preserving. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/active_addons.sh | 21 +++ .github/scripts/addon_python_deps.py | 199 ++++++++++++++++++++++ .github/workflows/ci.yml | 237 ++------------------------- tests/test_requires_mod_dedup.py | 164 ++++++++++++++++++ 4 files changed, 396 insertions(+), 225 deletions(-) create mode 100644 .github/scripts/active_addons.sh create mode 100644 .github/scripts/addon_python_deps.py create mode 100644 tests/test_requires_mod_dedup.py diff --git a/.github/scripts/active_addons.sh b/.github/scripts/active_addons.sh new file mode 100644 index 000000000..f68c8f147 --- /dev/null +++ b/.github/scripts/active_addons.sh @@ -0,0 +1,21 @@ +# Single source of truth for the is_active() addon filter, sourced by every +# ci.yml job step that gates on include_in_listing. +# +# An addon is "active" (built and released by make.py, so CI gates on it) unless +# EVERY register() in its .gpr.py sets include_in_listing=False. Those inactive +# addons are skipped by lint, the structure check, compile, and the unit / +# integration test runners. Previously each of those ~6 job steps inlined an +# identical copy of this function; this file is the one place it now lives, so a +# change to the active-addon rule is a one-site edit (per PR #820; Gary Griffin). +# +# Source it from a `shell: bash` step (the function uses `local`, a bashism): +# source .github/scripts/active_addons.sh +is_active() { + local addon="$1" g + for g in "$addon"/*.gpr.py; do + [ -f "$g" ] || continue + grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 + grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 + done + return 1 +} diff --git a/.github/scripts/addon_python_deps.py b/.github/scripts/addon_python_deps.py new file mode 100644 index 000000000..73db1519b --- /dev/null +++ b/.github/scripts/addon_python_deps.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Single source of truth for addon *Python* dependencies (``requires_mod``). + +Sibling to ``addon_system_deps.py`` (which covers the *system* deps — +``requires_gi`` typelibs and ``requires_exe`` executables). This module covers +the third dependency kind a Gramps addon declares in its ``.gpr.py``: + +* ``requires_mod`` — importable Python module names (e.g. ``["psycopg2"]``, + ``["life_line_chart", "svgwrite"]``). pip-installable. + +ci.yml previously inlined an *identical* ``requires_mod`` derivation heredoc in +three jobs (``unit-test-linux``, ``unit-test-windows``, ``integration-test``) +to build the install union, plus a near-identical ``find_spec`` validator +heredoc in each. That copy-paste is the drift this module removes: the +``.gpr.py`` files stay the single source of truth and every job derives the +list from one place. A one-line change is now a one-site edit. + +Self-contained on purpose: pure stdlib, no Gramps import and no third-party +import, so it runs at image-build time before Gramps is installed and on the +bare Windows conda runner. It does NOT depend on any external project. + +Scanning mirrors ``addon_system_deps.py`` deliberately: a regex finds each +``requires_mod = [...]`` assignment and ``ast.literal_eval`` parses the +bracketed list (no executing the ``.gpr.py``). Every real ``requires_mod`` in +addons-source is a flat list of string literals, so a literal-eval parse covers +them all; a non-literal or unreadable declaration is skipped tolerantly (with a +note to stderr) rather than aborting the batch — mirroring the old inline +behaviour. + +``requires_mod`` is the *importable module* name Gramps verifies at runtime via +``importlib.util.find_spec`` (``gramps/gen/utils/requirements.py`` +``Requirements.check_mod``). ``pip install`` wants the *distribution* name, +which differs for a few packages, so the install union maps the known +import→distribution cases (``PIL`` → ``Pillow``) via +``_IMPORT_TO_DISTRIBUTION``. The map is INSTALL-ONLY: ``--check-resolves`` +validates the *raw* declared import name, exactly as Gramps does, so an addon +that declares the PyPI distribution name by mistake (``requires_mod=["Pillow"]`` +when the import name is ``"PIL"``) is still caught. + +CLI:: + + addon_python_deps.py --install-list ROOT # space-separated sorted union + addon_python_deps.py --check-resolves ROOT # fail if a declared import name + # pip-installs but does not import +""" + +from __future__ import annotations + +import argparse +import ast +import glob +import os +import re +import sys + +# Matches a single-line ``requires_mod = ["a", "b"]`` assignment — the same +# shape addon_system_deps.py's _GI_RE/_EXE_RE assume, and the only shape that +# occurs in addons-source. +_MOD_RE = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + +# ``requires_mod`` holds the *importable module* name (declare "PIL", the import +# name, NOT "Pillow", the PyPI distribution — Gramps verifies it with +# ``importlib.util.find_spec``). But ``pip install`` wants the *distribution* +# name, which differs for some packages, so installing the raw import name fails +# (``pip install PIL`` → no such distribution; the package is ``Pillow``). Map +# the known import→distribution cases so the derived install list resolves on +# PyPI. Addons stay correct (import name); only the install side translates. +_IMPORT_TO_DISTRIBUTION = { + "PIL": "Pillow", +} + + +def _gpr_files(root: str) -> list[str]: + return sorted(glob.glob(os.path.join(root, "*", "*.gpr.py"))) + + +def _declared_mods(text: str, path: str) -> list[str]: + """The *raw* (un-mapped) ``requires_mod`` import names declared in one + ``.gpr.py`` body. Tolerant: a non-literal value is skipped with a note to + stderr, mirroring the old inline behaviour.""" + out: list[str] = [] + for m in _MOD_RE.finditer(text): + try: + value = ast.literal_eval(m.group(1)) + except (ValueError, SyntaxError): + print( + f"addon_python_deps: skipping non-literal requires_mod " + f"in {path}: {m.group(1)!r}", + file=sys.stderr, + ) + continue + for mod in value: + if mod: + out.append(mod) + return out + + +def declared_mods(root: str) -> set[str]: + """The sorted-by-caller set of *raw* ``requires_mod`` import names declared + across every addon's ``.gpr.py`` under *root* — the names Gramps verifies + via ``find_spec`` (NOT the install-mapped distribution names). Unreadable + files are skipped with a note to stderr.""" + names: set[str] = set() + for path in _gpr_files(root): + try: + with open(path, encoding="utf-8") as fh: + text = fh.read() + except OSError as exc: + print( + f"addon_python_deps: skipping unreadable {path}: {exc}", + file=sys.stderr, + ) + continue + names.update(_declared_mods(text, path)) + return names + + +def install_list(root: str) -> list[str]: + """Return the sorted union of ``requires_mod`` across every addon's + ``.gpr.py`` under *root*, mapped to pip *distribution* names. This is the + list the ci.yml "Install addon runtime deps" steps pip-install. Best-effort + and tolerant — a file that cannot be read, or a non-literal value, is + skipped (not fatal), so the step installs what it can resolve.""" + return sorted(_IMPORT_TO_DISTRIBUTION.get(mod, mod) for mod in declared_mods(root)) + + +def check_resolves(root: str) -> int: + """Validate that every declared ``requires_mod`` import name which actually + pip-installed also resolves under ``importlib.util.find_spec`` — the same + check Gramps' ``Requirements.check_mod`` performs at runtime. The *raw* + declared name is checked (NOT the install-mapped distribution name), because + that is what Gramps imports. + + Run this *after* the install union has been pip-installed: a name that pip + never installed (an exotic system-dep / image gap, not a PR bug) is skipped; + a name that pip-installed yet still fails ``find_spec`` is a wrong + declaration (e.g. the PyPI distribution ``"Pillow"`` instead of the import + name ``"PIL"``) and fails the gate. Returns 1 if any bad name, else 0. + Preserves the behaviour of the old inline validator heredoc verbatim.""" + import subprocess + from importlib.util import find_spec + + bad: list[str] = [] + for name in sorted(declared_mods(root)): + installed = ( + subprocess.run( + [sys.executable, "-m", "pip", "show", name], + capture_output=True, + ).returncode + == 0 + ) + if not installed: + print(f"~ {name} (pip-install failed earlier, skipping)") + continue + if find_spec(name) is None: + bad.append(name) + print(f"x {name} (pip-installed but find_spec returned None)") + else: + print(f"ok {name}") + + if bad: + print() + print(f"::error::Wrong requires_mod names: {bad}") + print("These pip-install but are not importable. requires_mod is") + print("consumed by gramps' check_mod() via find_spec(), so the") + print("importable module name is required (e.g. 'PIL', not 'Pillow').") + return 1 + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--install-list", + metavar="ROOT", + help="print the sorted, pip-installable union of requires_mod across " + "ROOT/*/*.gpr.py (import names mapped to distribution names)", + ) + group.add_argument( + "--check-resolves", + metavar="ROOT", + help="verify every declared requires_mod import name that pip-installed " + "also resolves under importlib.util.find_spec (Gramps' check_mod); " + "exit 1 if any installs but does not import", + ) + args = parser.parse_args(argv) + + if args.install_list is not None: + print(" ".join(install_list(args.install_list))) + return 0 + return check_resolves(args.check_resolves) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c69c7c7f3..05b45a41c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,15 +73,7 @@ jobs: # step stays self-contained. shell: bash run: | - is_active() { - local addon="$1" g - for g in "$addon"/*.gpr.py; do - [ -f "$g" ] || continue - grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 - grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 - done - return 1 - } + source .github/scripts/active_addons.sh excludes="" for d in */; do d="${d%/}" @@ -116,15 +108,7 @@ jobs: # Skip include_in_listing=False addons (see lint job for rationale). shell: bash run: | - is_active() { - local addon="$1" g - for g in "$addon"/*.gpr.py; do - [ -f "$g" ] || continue - grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 - grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 - done - return 1 - } + source .github/scripts/active_addons.sh failed=0 for gpr in */*.gpr.py; do addon_dir="$(dirname "$gpr")" @@ -158,15 +142,7 @@ jobs: # Skip include_in_listing=False addons (see lint job for rationale). shell: bash run: | - is_active() { - local addon="$1" g - for g in "$addon"/*.gpr.py; do - [ -f "$g" ] || continue - grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 - grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 - done - return 1 - } + source .github/scripts/active_addons.sh skipped="" for d in */; do d="${d%/}" @@ -246,23 +222,7 @@ jobs: # isolation without blocking the rest. shell: bash run: | - addon_mods=$(python3 - <<'PY' - import ast, glob, re - pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") - mods = set() - for f in glob.glob("*/*.gpr.py"): - try: - text = open(f, encoding="utf-8").read() - except OSError: - continue - for m in pat.finditer(text): - try: - mods.update(ast.literal_eval(m.group(1))) - except (ValueError, SyntaxError): - pass - print(" ".join(sorted(mods))) - PY - ) + addon_mods=$(python3 .github/scripts/addon_python_deps.py --install-list .) if [ -n "$addon_mods" ]; then echo "→ addon deps: $addon_mods" for mod in $addon_mods; do @@ -283,46 +243,7 @@ jobs: # system-dep / image gaps, not PR-caused. shell: bash run: | - python3 - <<'PY' - import ast, glob, re, subprocess, sys - from importlib.util import find_spec - - pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") - names = set() - for f in glob.glob("*/*.gpr.py"): - try: - text = open(f, encoding="utf-8").read() - except OSError: - continue - for m in pat.finditer(text): - try: - names.update(ast.literal_eval(m.group(1))) - except (ValueError, SyntaxError): - pass - - bad = [] - for name in sorted(names): - installed = subprocess.run( - [sys.executable, "-m", "pip", "show", name], - capture_output=True, - ).returncode == 0 - if not installed: - print(f"~ {name} (pip-install failed earlier, skipping)") - continue - if find_spec(name) is None: - bad.append(name) - print(f"x {name} (pip-installed but find_spec returned None)") - else: - print(f"ok {name}") - - if bad: - print() - print(f"::error::Wrong requires_mod names: {bad}") - print("These pip-install but are not importable. requires_mod is") - print("consumed by gramps' check_mod() via find_spec(), so the") - print("importable module name is required (e.g. 'PIL', not 'Pillow').") - sys.exit(1) - PY + python3 .github/scripts/addon_python_deps.py --check-resolves . - name: Run per-addon unit tests # Filename convention (all OSes): @@ -341,15 +262,7 @@ jobs: env: PYTHONPATH: . run: | - is_active() { - local addon="$1" g - for g in "$addon"/*.gpr.py; do - [ -f "$g" ] || continue - grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 - grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 - done - return 1 - } + source .github/scripts/active_addons.sh modules="" for f in */tests/test_*.py; do [ -f "$f" ] || continue @@ -459,23 +372,7 @@ jobs: # See unit-test-linux for rationale. Uses `python` (conda-forge # env) to match the surrounding Windows job style. run: | - addon_mods=$(python - <<'PY' - import ast, glob, re - pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") - mods = set() - for f in glob.glob("*/*.gpr.py"): - try: - text = open(f, encoding="utf-8").read() - except OSError: - continue - for m in pat.finditer(text): - try: - mods.update(ast.literal_eval(m.group(1))) - except (ValueError, SyntaxError): - pass - print(" ".join(sorted(mods))) - PY - ) + addon_mods=$(python .github/scripts/addon_python_deps.py --install-list .) if [ -n "$addon_mods" ]; then echo "→ addon deps: $addon_mods" for mod in $addon_mods; do @@ -489,46 +386,7 @@ jobs: # See unit-test-linux for rationale. Uses `python` (conda-forge # env) to match the surrounding Windows job style. run: | - python - <<'PY' - import ast, glob, re, subprocess, sys - from importlib.util import find_spec - - pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") - names = set() - for f in glob.glob("*/*.gpr.py"): - try: - text = open(f, encoding="utf-8").read() - except OSError: - continue - for m in pat.finditer(text): - try: - names.update(ast.literal_eval(m.group(1))) - except (ValueError, SyntaxError): - pass - - bad = [] - for name in sorted(names): - installed = subprocess.run( - [sys.executable, "-m", "pip", "show", name], - capture_output=True, - ).returncode == 0 - if not installed: - print(f"~ {name} (pip-install failed earlier, skipping)") - continue - if find_spec(name) is None: - bad.append(name) - print(f"x {name} (pip-installed but find_spec returned None)") - else: - print(f"ok {name}") - - if bad: - print() - print(f"::error::Wrong requires_mod names: {bad}") - print("These pip-install but are not importable. requires_mod is") - print("consumed by gramps' check_mod() via find_spec(), so the") - print("importable module name is required (e.g. 'PIL', not 'Pillow').") - sys.exit(1) - PY + python .github/scripts/addon_python_deps.py --check-resolves . - name: Run per-addon unit tests # See filename-convention note in unit-test-linux. The Windows @@ -536,15 +394,7 @@ jobs: env: PYTHONPATH: . run: | - is_active() { - local addon="$1" g - for g in "$addon"/*.gpr.py; do - [ -f "$g" ] || continue - grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 - grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 - done - return 1 - } + source .github/scripts/active_addons.sh modules="" for f in */tests/test_*.py; do [ -f "$f" ] || continue @@ -607,23 +457,7 @@ jobs: # requires_mod packages. shell: bash run: | - addon_mods=$(python3 - <<'PY' - import ast, glob, re - pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") - mods = set() - for f in glob.glob("*/*.gpr.py"): - try: - text = open(f, encoding="utf-8").read() - except OSError: - continue - for m in pat.finditer(text): - try: - mods.update(ast.literal_eval(m.group(1))) - except (ValueError, SyntaxError): - pass - print(" ".join(sorted(mods))) - PY - ) + addon_mods=$(python3 .github/scripts/addon_python_deps.py --install-list .) if [ -n "$addon_mods" ]; then echo "→ addon deps: $addon_mods" for mod in $addon_mods; do @@ -637,46 +471,7 @@ jobs: # See unit-test-linux for rationale. shell: bash run: | - python3 - <<'PY' - import ast, glob, re, subprocess, sys - from importlib.util import find_spec - - pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") - names = set() - for f in glob.glob("*/*.gpr.py"): - try: - text = open(f, encoding="utf-8").read() - except OSError: - continue - for m in pat.finditer(text): - try: - names.update(ast.literal_eval(m.group(1))) - except (ValueError, SyntaxError): - pass - - bad = [] - for name in sorted(names): - installed = subprocess.run( - [sys.executable, "-m", "pip", "show", name], - capture_output=True, - ).returncode == 0 - if not installed: - print(f"~ {name} (pip-install failed earlier, skipping)") - continue - if find_spec(name) is None: - bad.append(name) - print(f"x {name} (pip-installed but find_spec returned None)") - else: - print(f"ok {name}") - - if bad: - print() - print(f"::error::Wrong requires_mod names: {bad}") - print("These pip-install but are not importable. requires_mod is") - print("consumed by gramps' check_mod() via find_spec(), so the") - print("importable module name is required (e.g. 'PIL', not 'Pillow').") - sys.exit(1) - PY + python3 .github/scripts/addon_python_deps.py --check-resolves . - name: Run plugin registration tests # shell: bash for consistency with the surrounding steps; the @@ -706,15 +501,7 @@ jobs: env: PYTHONPATH: . run: | - is_active() { - local addon="$1" g - for g in "$addon"/*.gpr.py; do - [ -f "$g" ] || continue - grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 - grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 - done - return 1 - } + source .github/scripts/active_addons.sh modules="" for f in */tests/test_integration*.py; do [ -f "$f" ] || continue diff --git a/tests/test_requires_mod_dedup.py b/tests/test_requires_mod_dedup.py new file mode 100644 index 000000000..8490911a2 --- /dev/null +++ b/tests/test_requires_mod_dedup.py @@ -0,0 +1,164 @@ +"""Regression test for the requires_mod / is_active CI de-duplication. + +ci.yml used to inline an identical ``requires_mod`` derivation heredoc in three +jobs and an identical ``is_active()`` bash helper in ~six job steps. The fix +moves the derivation into ``.github/scripts/addon_python_deps.py`` and the +helper into ``.github/scripts/active_addons.sh``, each consumed from one place. + +This test pins two things, so the duplication cannot silently come back and the +refactor is proven behaviour-preserving: + +1. Behaviour preservation — the single ``addon_python_deps`` module derives the + *same* install union and the *same* raw declared-name set the old inline + heredoc did (computed here by an independent oracle over the real tree). + +2. The DRY invariant, stated per-category — NO inline ``is_active()`` definition + and NO ``requires_mod`` heredoc survive in ci.yml, and EVERY job step that + *calls* ``is_active`` sources the shared helper (a missed step is caught, not + masked by an "at least one source" check). + +Pure stdlib / GUI-import-free on purpose: it imports the production module the +ci.yml jobs call (not a copy), runs headless, and needs no gi / gramps.gui. +""" + +from __future__ import annotations + +import ast +import glob +import os +import re +import sys +import unittest + +# Repo layout is fixed relative to this file: tests/ sits at the addons-source +# root, and the CI scripts live under .github/scripts/ — resolve both from +# __file__ so the test is cwd-independent (the C4 runner cd's into the repo, CI +# discover runs from the root). +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_SCRIPTS = os.path.join(_REPO_ROOT, ".github", "scripts") +_CI_YML = os.path.join(_REPO_ROOT, ".github", "workflows", "ci.yml") +_HELPER = os.path.join(_SCRIPTS, "active_addons.sh") + +if _SCRIPTS not in sys.path: + sys.path.insert(0, _SCRIPTS) + +# Imported at module load: the production module the three ci.yml jobs invoke. +# With the fix reverted (module file removed) this import fails and every test +# below errors — the red half of the red->green contract. +import addon_python_deps # noqa: E402 + +# --- independent oracle: the OLD inline heredoc algorithm, verbatim ---------- +_OLD_RE = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") +# Install-name map the heredoc lacked; install-only (find_spec gate stays raw). +_INSTALL_MAP = {"PIL": "Pillow"} + + +def _read(path): + with open(path, encoding="utf-8") as fh: + return fh.read() + + +def _old_raw_union(root): + mods = set() + for fn in sorted(glob.glob(os.path.join(root, "*", "*.gpr.py"))): + try: + text = _read(fn) + except OSError: + continue + for m in _OLD_RE.finditer(text): + try: + mods.update(ast.literal_eval(m.group(1))) + except (ValueError, SyntaxError): + pass + return {m for m in mods if m} + + +def _ci_steps(): + """Yield each ci.yml step body as a string. A step starts at a 6-space + `- ` line and runs until the next one (no YAML dependency needed).""" + lines = _read(_CI_YML).splitlines(keepends=True) + steps, cur = [], None + for line in lines: + if re.match(r"^ - ", line): + if cur is not None: + steps.append("".join(cur)) + cur = [line] + elif cur is not None: + cur.append(line) + if cur is not None: + steps.append("".join(cur)) + return steps + + +class RequiresModDerivationDedup(unittest.TestCase): + """The derivation is single-sourced AND behaviour-preserving.""" + + def test_install_list_matches_old_heredoc(self): + old = _old_raw_union(_REPO_ROOT) + expected = sorted(_INSTALL_MAP.get(m, m) for m in old) + self.assertEqual(addon_python_deps.install_list(_REPO_ROOT), expected) + + def test_declared_raw_names_match_old_heredoc(self): + # The find_spec gate consumes RAW import names — these must equal the + # old union exactly (the install map must NOT leak into them). + self.assertEqual( + addon_python_deps.declared_mods(_REPO_ROOT), _old_raw_union(_REPO_ROOT) + ) + + def test_install_map_is_install_only(self): + # PIL maps to Pillow on the install side, but the raw declared-name set + # never contains the distribution name (so Gramps' find_spec gate, which + # the module's --check-resolves mirrors, keeps checking the import name). + self.assertNotIn("Pillow", addon_python_deps.declared_mods(_REPO_ROOT)) + + def test_no_requires_mod_heredoc_remains(self): + text = _read(_CI_YML) + self.assertEqual( + re.findall(r"requires_mod\s*=\s*\(\[", text), + [], + "a requires_mod derivation heredoc still lives inline in ci.yml", + ) + + def test_three_jobs_consume_the_module(self): + text = _read(_CI_YML) + self.assertEqual( + len(re.findall(r"addon_python_deps\.py --install-list", text)), 3 + ) + self.assertEqual( + len(re.findall(r"addon_python_deps\.py --check-resolves", text)), 3 + ) + + +class IsActiveHelperDedup(unittest.TestCase): + """is_active() lives in one sourced helper, consumed by EVERY filtering step.""" + + def test_helper_file_defines_is_active(self): + self.assertTrue(os.path.isfile(_HELPER)) + self.assertIn("is_active()", _read(_HELPER)) + + def test_no_inline_is_active_definition_remains(self): + text = _read(_CI_YML) + self.assertEqual( + re.findall(r"is_active\(\)\s*\{", text), + [], + "an inline is_active() definition still lives in ci.yml", + ) + + def test_every_is_active_call_site_sources_the_helper(self): + # Per-category invariant: every job step that CALLS is_active must also + # source the shared helper. Asserting "sourced at least once" would miss + # a step that calls a now-undefined is_active; this checks each site. + calling = [s for s in _ci_steps() if re.search(r'is_active\s+"', s)] + self.assertGreaterEqual( + len(calling), 6, "expected >=6 active-addon filtering steps" + ) + for step in calling: + self.assertIn( + "source .github/scripts/active_addons.sh", + step, + "a step calls is_active without sourcing active_addons.sh:\n" + step, + ) + + +if __name__ == "__main__": + unittest.main() From 0fdb8732dd030a33b199bcd0465bc775ef929929 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Fri, 19 Jun 2026 23:49:32 +0200 Subject: [PATCH 27/47] =?UTF-8?q?ci(addon-deps):=20expand=20requires=5Fmod?= =?UTF-8?q?=20import=E2=86=92pip=20table=20to=20match=20gramps=20#2308?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI install step maps requires_mod import names to PyPI distribution names (e.g. PIL→Pillow). gramps PR #2308 ("PyPI wheel installer for addon module dependencies") adds the authoritative table _IMPORT_TO_PYPI in gramps/gen/utils/pypi.py with ~12 entries; this copy had one. Once #2308 merges, an addon declaring e.g. cv2 / yaml / sklearn in requires_mod installs correctly in Gramps but this CI's install union would pip install the bare import name and fail the addon-unit step. Mirror the 11 missing entries now (all current, correct PyPI names — valid independent of #2308) so CI installs the same distribution Gramps does. Closes the drift flagged on PR 820 (upstream PR 2308). No live addon declares an at-risk name today, so this is pre-emptive. When #2308 merges, single-source the table from gramps' module instead of hand-mirroring it. Co-Authored-By: Claude Opus 4.8 --- .github/scripts/addon_python_deps.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/scripts/addon_python_deps.py b/.github/scripts/addon_python_deps.py index 73db1519b..f0ee58733 100644 --- a/.github/scripts/addon_python_deps.py +++ b/.github/scripts/addon_python_deps.py @@ -65,8 +65,26 @@ # (``pip install PIL`` → no such distribution; the package is ``Pillow``). Map # the known import→distribution cases so the derived install list resolves on # PyPI. Addons stay correct (import name); only the install side translates. +# +# Kept in step with Gramps' own install-time table, ``_IMPORT_TO_PYPI`` in +# ``gramps/gen/utils/pypi.py`` (gramps PR #2308) — the authority Gramps will use +# to install ``requires_mod`` deps in frozen/Flatpak/pip-less environments. The +# entries below mirror that table so an addon declaring any of these import names +# installs the same distribution in CI as Gramps does at runtime. Once #2308 +# merges, single-source this from that module rather than hand-mirroring it. _IMPORT_TO_DISTRIBUTION = { "PIL": "Pillow", + "cv2": "opencv-python", + "sklearn": "scikit-learn", + "yaml": "PyYAML", + "dateutil": "python-dateutil", + "bs4": "beautifulsoup4", + "serial": "pyserial", + "usb": "pyusb", + "nacl": "PyNaCl", + "Crypto": "pycryptodome", + "OpenSSL": "pyOpenSSL", + "wx": "wxPython", } From b9974e5286ff7b5ca6ae07eade0d39cefa84a800 Mon Sep 17 00:00:00 2001 From: Eduard Ralph Date: Mon, 22 Jun 2026 20:57:49 +0200 Subject: [PATCH 28/47] ci: discover + load nested-package addon tests (#52) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unit/integration jobs discovered tests with a one-level glob (`*/tests/test_*.py`) and the runner loaded them by dotted name from the repo root, never putting the addon's own dir on sys.path. So a nested-package addon's suites (tests under `/tests//`) were not found, and addon code importing a top-level package (`from name_processor… import`, WebSearch's `from models import`) could not resolve. - ci.yml: enable `shopt -s globstar` and recurse the three discovery loops (`*/tests/**/test_*.py`, and `**/test_integration*.py`). Existing guards (is_active, the basename filters, the Sqlite skip, the dotted conversion) are unaffected. - run_addon_tests.py: in the worker, APPEND the addon's own directory to sys.path before loading — mirroring Gramps' plugin loader (gramps/gen/plug/_manager.py). Append (not prepend) so the repo-root shared `tests` Gramps-emulation env still wins; the module is still loaded by full dotted name from the repo root, so package-relative imports keep working. `--root` is threaded into the worker. Strict superset of today's behaviour: flat addons unchanged. Validated: the full NameSuite nested suite (upstream PR 941) — 20 modules, 234 tests — discovers, loads, and passes as-is; WebSearch's `from models import` now resolves (its remaining Gtk/Gdk-version error is the local GTK4 env / the separate #38 Gdk-pin gap, not this change). Out of scope: dsblank's `test/`+`*_test.py`+pytest convention is intentionally not matched (tracked separately). tests/test_run_addon_tests_paths.py covers a nested Model-B addon (top- level lib import + shared-env-not-shadowed) and a flat Model-B addon. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/scripts/run_addon_tests.py | 18 +++- .github/workflows/ci.yml | 9 +- tests/test_run_addon_tests_paths.py | 144 ++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 tests/test_run_addon_tests_paths.py diff --git a/.github/scripts/run_addon_tests.py b/.github/scripts/run_addon_tests.py index 6491aa0e5..12016fa9a 100644 --- a/.github/scripts/run_addon_tests.py +++ b/.github/scripts/run_addon_tests.py @@ -76,7 +76,7 @@ def _bootstrap_gi() -> None: # worker: runs ONE module in this (sub)process # # ------------------------------------------------------------ -def _run_worker(modname: str) -> int: +def _run_worker(modname: str, root: str = ".") -> int: """Run a single module; print a machine-readable result line; exit 0. The parent classifies pass/fail from the printed counts and its own platform @@ -84,6 +84,18 @@ def _run_worker(modname: str) -> int: indistinguishable from an interpreter crash). """ _bootstrap_gi() + # Put the addon's own directory on sys.path, mirroring how Gramps' plugin + # loader (gramps/gen/plug/_manager.py) inserts the addon dir before importing + # a plugin. APPEND (not prepend) so the repo-root shared `tests` environment + # still takes precedence: this lets a nested-package addon's top-level + # imports (e.g. ``from name_processor… import``) resolve without shadowing + # the shared Gramps-emulation test env. The module is still loaded by its + # full dotted name from the repo root, so package-relative imports keep + # working too. + addon = modname.split(".", 1)[0] + addon_dir = os.path.join(root, addon) + if addon_dir not in sys.path: + sys.path.append(addon_dir) try: suite = unittest.defaultTestLoader.loadTestsFromName(modname) except Exception as exc: # import-time failure @@ -109,7 +121,7 @@ def _classify(modname: str, platform: str, root: str) -> tuple[bool, str]: satisfiable = deps.addon_satisfiable_on(os.path.join(root, addon), platform) proc = subprocess.Popen( - [sys.executable, os.path.abspath(__file__), "--worker", modname], + [sys.executable, os.path.abspath(__file__), "--worker", modname, "--root", root], stdout=subprocess.PIPE, stderr=None, # stream the test output straight to the CI log text=True, @@ -180,7 +192,7 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.worker: - return _run_worker(args.worker) + return _run_worker(args.worker, args.root) if not args.platform: parser.error("--platform is required in parent mode") diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05b45a41c..8893400a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -264,7 +264,8 @@ jobs: run: | source .github/scripts/active_addons.sh modules="" - for f in */tests/test_*.py; do + shopt -s globstar # also match nested-package tests//test_*.py + for f in */tests/**/test_*.py; do [ -f "$f" ] || continue addon="${f%%/*}" is_active "$addon" || continue @@ -396,7 +397,8 @@ jobs: run: | source .github/scripts/active_addons.sh modules="" - for f in */tests/test_*.py; do + shopt -s globstar # also match nested-package tests//test_*.py + for f in */tests/**/test_*.py; do [ -f "$f" ] || continue addon="${f%%/*}" is_active "$addon" || continue @@ -503,7 +505,8 @@ jobs: run: | source .github/scripts/active_addons.sh modules="" - for f in */tests/test_integration*.py; do + shopt -s globstar # also match nested-package tests//test_integration*.py + for f in */tests/**/test_integration*.py; do [ -f "$f" ] || continue addon="${f%%/*}" is_active "$addon" || continue diff --git a/tests/test_run_addon_tests_paths.py b/tests/test_run_addon_tests_paths.py new file mode 100644 index 000000000..a9e5d7a3b --- /dev/null +++ b/tests/test_run_addon_tests_paths.py @@ -0,0 +1,144 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Tests for ``run_addon_tests.py``'s per-addon import root (issue #52). + +The runner appends each addon's own directory to ``sys.path`` before loading +its tests, mirroring how Gramps' plugin loader puts the addon dir on the path. +This lets an addon's tests import the addon's top-level modules +(``from import …``) — including a nested-package addon whose test +modules live under ``/tests//`` — while the repo-root shared +``tests`` environment still wins (APPEND, not prepend). + +Driven via subprocess against synthetic addon trees in a temp dir, so no gramps +install is needed. +""" + +# ------------------------ +# Python modules +# ------------------------ +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + + +ADDONS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RUN_ADDON_TESTS = os.path.join( + ADDONS_ROOT, ".github", "scripts", "run_addon_tests.py" +) + + +@unittest.skipUnless( + os.path.isfile(RUN_ADDON_TESTS), "run_addon_tests.py not present on this branch" +) +class RunAddonTestsPathsTest(unittest.TestCase): + """The runner must load nested- and flat-package addon tests by adding the + addon dir to sys.path, without shadowing the shared repo-root tests env.""" + + def setUp(self) -> None: + self.root = tempfile.mkdtemp(prefix="run_addon_tests_paths_") + # Stand-in for the repo-root shared Gramps-emulation env (PR 950): + # a top-level `tests` package the addon tests may import. + self._write("tests/__init__.py", "") + self._write( + "tests/gramps_test_env.py", 'SENTINEL = "repo-root-shared-env"\n' + ) + + def tearDown(self) -> None: + shutil.rmtree(self.root, ignore_errors=True) + + def _write(self, relpath: str, content: str) -> None: + path = os.path.join(self.root, relpath) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as fp: + fp.write(content) + + def _run(self, modname: str) -> subprocess.CompletedProcess: + env = os.environ.copy() + env["PYTHONPATH"] = "." # repo root on sys.path, as ci.yml sets it + return subprocess.run( + [ + sys.executable, + RUN_ADDON_TESTS, + "--platform", + "apt", + "--root", + self.root, + modname, + ], + cwd=self.root, + env=env, + capture_output=True, + text=True, + check=False, + ) + + def test_nested_package_addon_loads(self) -> None: + """A nested-package addon: test module under tests//, importing + the addon's top-level lib AND the shared repo-root tests env.""" + # Namespace-package top (no SynthAddon/__init__.py); top-level lib module. + self._write("SynthAddon/synthlib.py", "VALUE = 42\n") + self._write("SynthAddon/tests/__init__.py", "") + self._write("SynthAddon/tests/sub/__init__.py", "") + self._write( + "SynthAddon/tests/sub/test_nested.py", + "import unittest\n" + "from synthlib import VALUE\n" # per-addon import root + "from tests.gramps_test_env import SENTINEL\n" # shared env not shadowed + "\n" + "class T(unittest.TestCase):\n" + " def test_addon_lib(self):\n" + " self.assertEqual(VALUE, 42)\n" + " def test_shared_env(self):\n" + ' self.assertEqual(SENTINEL, "repo-root-shared-env")\n', + ) + + result = self._run("SynthAddon.tests.sub.test_nested") + out = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, "runner failed:\n%s" % out) + self.assertIn("ok SynthAddon.tests.sub.test_nested", result.stdout, out) + + def test_flat_model_b_addon_loads(self) -> None: + """A flat addon whose test imports a top-level addon module + (WebSearch-style) — currently broken without the per-addon import root.""" + self._write("FlatAddon/flatlib.py", "FLAT = 7\n") + self._write("FlatAddon/tests/__init__.py", "") + self._write( + "FlatAddon/tests/test_flat.py", + "import unittest\n" + "from flatlib import FLAT\n" + "\n" + "class T(unittest.TestCase):\n" + " def test_flat(self):\n" + " self.assertEqual(FLAT, 7)\n", + ) + + result = self._run("FlatAddon.tests.test_flat") + out = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, "runner failed:\n%s" % out) + self.assertIn("ok FlatAddon.tests.test_flat", result.stdout, out) + + +if __name__ == "__main__": + unittest.main() From 427f8c3a90bba46825b0fa5fe3b9b89a80bb3486 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 00:26:17 +0200 Subject: [PATCH 29/47] ci: classify PIL and pymongo as wheel-only requires_mod The merge of upstream maintenance/gramps60 brought in two requires_mod declarations the dependency drift gate had never seen: PIL (EditExifMetadata) and pymongo (MongoDB). Neither was classified in WHEEL_ONLY_MODS or MOD_BUILD_PACKAGES, so addon_system_deps.py --unmapped exited 1, failing the 'Validate addon system deps are mapped' CI step and three tests in tests/test_addon_system_deps.py. Both ship binary wheels on every CI platform and need no system package, so they belong in WHEEL_ONLY_MODS. PIL is declared by import name per the requires_mod contract; the install side already maps PIL -> Pillow via addon_python_deps.py. --- .github/scripts/addon_system_deps.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/scripts/addon_system_deps.py b/.github/scripts/addon_system_deps.py index 3c3480ee4..b66cac2dd 100644 --- a/.github/scripts/addon_system_deps.py +++ b/.github/scripts/addon_system_deps.py @@ -103,11 +103,15 @@ # fails CI on any requires_mod that is in neither set. WHEEL_ONLY_MODS: frozenset[str] = frozenset( { + "PIL", # EditExifMetadata — Pillow ships binary wheels on every CI + # platform; declared by import name, addon_python_deps maps + # PIL→Pillow on the install side "boto3", # S3MediaUploader — pure-Python AWS SDK wheel "dbf", # TMGimporter — pure-Python wheel "life_line_chart", # LifeLineChartView — pure-Python wheel "litellm", # ChatWithTree / GrampsChat — pure-Python wheel "networkx", # NetworkChart — pure-Python wheel + "pymongo", # MongoDB — ships binary wheels on every CI platform "svgwrite", # LifeLineChartView — pure-Python wheel } ) From ad98e1d3020b8d20eb3b22721c1b36174c3721cc Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 00:39:10 +0200 Subject: [PATCH 30/47] ci: single-source requires_mod handling from gramps' installer (gramps PR #2308) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gramps 6.1 now ships the authority this machinery was hand-mirroring: gramps/gen/utils/pypi.py (gramps PR #2308, 7f94428b13 — not backported to 6.0) with the import->distribution table _IMPORT_TO_PYPI, and a Requirements.check_mod that really imports the module after find_spec. Wire addon_python_deps.py to both, per lane, at lookup time: - _distribution_map() prefers the installed gramps' _IMPORT_TO_PYPI so 6.1+ lanes install exactly what Gramps' own installer would, new upstream entries included; the local _IMPORT_TO_DISTRIBUTION becomes a fallback mirror for lanes where gramps is absent or predates 6.1 (the gramps60 image, the conda-forge 6.0.x Windows lane, bare runners). The table dict is read directly rather than via resolve_pypi_name(), whose per-name LOG.warning advises declaring the PyPI name — the opposite of the import-name contract the gate enforces. - --check-resolves delegates to the installed gramps' Requirements().check_mod, so the gate matches whichever series the lane ships: find_spec-only on 6.0, find_spec plus a real import on 6.1+. Stdlib find_spec remains the fallback where gramps is not importable (never a CI lane). - Fix a latent probe bug the PIL declaration makes live: installed-ness was probed with 'pip show ', but pip only knows the distribution name ('pip show PIL' fails with Pillow installed), so the one name the mapping machinery exists for was silently skipped, never validated. Probe by the mapped distribution name; keep judging by the raw import name (the install-only-map invariant). All gramps imports are lazy and guarded with (Exception, SystemExit) — a half-installed gramps raises SystemExit from ResourcePath at import, not ImportError — so the module stays importable pure-stdlib. tests/test_addon_python_deps.py pins the two seams and the probe fix hermetically (fake gramps module trees via mock.patch.dict), plus a sync-guard asserting mirror == authority wherever gramps >= 6.1 is importable (the gramps61 lanes; skips elsewhere, where the mirror is inert). ci.yml changes are comment-only. --- .github/scripts/addon_python_deps.py | 138 +++++++++++----- .github/workflows/ci.yml | 26 +-- tests/test_addon_python_deps.py | 229 +++++++++++++++++++++++++++ 3 files changed, 348 insertions(+), 45 deletions(-) create mode 100644 tests/test_addon_python_deps.py diff --git a/.github/scripts/addon_python_deps.py b/.github/scripts/addon_python_deps.py index f0ee58733..fe4b38587 100644 --- a/.github/scripts/addon_python_deps.py +++ b/.github/scripts/addon_python_deps.py @@ -27,15 +27,18 @@ note to stderr) rather than aborting the batch — mirroring the old inline behaviour. -``requires_mod`` is the *importable module* name Gramps verifies at runtime via -``importlib.util.find_spec`` (``gramps/gen/utils/requirements.py`` -``Requirements.check_mod``). ``pip install`` wants the *distribution* name, -which differs for a few packages, so the install union maps the known -import→distribution cases (``PIL`` → ``Pillow``) via -``_IMPORT_TO_DISTRIBUTION``. The map is INSTALL-ONLY: ``--check-resolves`` -validates the *raw* declared import name, exactly as Gramps does, so an addon -that declares the PyPI distribution name by mistake (``requires_mod=["Pillow"]`` -when the import name is ``"PIL"``) is still caught. +``requires_mod`` is the *importable module* name Gramps verifies at runtime +(``gramps/gen/utils/requirements.py`` ``Requirements.check_mod`` — bare +``find_spec`` on gramps 6.0; ``find_spec`` plus a real import on 6.1+, gramps +PR #2308). ``pip install`` wants the *distribution* name, which differs for a +few packages, so the install union maps the known import→distribution cases +(``PIL`` → ``Pillow``) — single-sourced from Gramps' own ``_IMPORT_TO_PYPI`` +table when the installed gramps ships one (6.1+), with the local +``_IMPORT_TO_DISTRIBUTION`` mirror as fallback. The map is INSTALL-ONLY: +``--check-resolves`` validates the *raw* declared import name, exactly as +Gramps does, so an addon that declares the PyPI distribution name by mistake +(``requires_mod=["Pillow"]`` when the import name is ``"PIL"``) is still +caught. CLI:: @@ -52,6 +55,7 @@ import os import re import sys +from collections.abc import Callable # Matches a single-line ``requires_mod = ["a", "b"]`` assignment — the same # shape addon_system_deps.py's _GI_RE/_EXE_RE assume, and the only shape that @@ -66,12 +70,16 @@ # the known import→distribution cases so the derived install list resolves on # PyPI. Addons stay correct (import name); only the install side translates. # -# Kept in step with Gramps' own install-time table, ``_IMPORT_TO_PYPI`` in -# ``gramps/gen/utils/pypi.py`` (gramps PR #2308) — the authority Gramps will use -# to install ``requires_mod`` deps in frozen/Flatpak/pip-less environments. The -# entries below mirror that table so an addon declaring any of these import names -# installs the same distribution in CI as Gramps does at runtime. Once #2308 -# merges, single-source this from that module rather than hand-mirroring it. +# FALLBACK MIRROR of Gramps' own install-time table, ``_IMPORT_TO_PYPI`` in +# ``gramps/gen/utils/pypi.py`` (gramps PR #2308, merged into gramps 6.1 as +# 7f94428b13; not on 6.0) — the authority Gramps uses to install +# ``requires_mod`` deps in frozen/Flatpak/pip-less environments. At lookup time +# ``_distribution_map()`` prefers that table from the installed gramps, so on +# 6.1+ lanes new upstream entries take effect without touching this file; this +# mirror serves lanes where gramps is absent or predates 6.1 (the gramps60 +# image, the conda-forge 6.0.x Windows lane, bare runners). The sync-guard test +# ``tests/test_addon_python_deps.py::GrampsTableSync`` pins mirror == authority +# wherever the authority is importable. _IMPORT_TO_DISTRIBUTION = { "PIL": "Pillow", "cv2": "opencv-python", @@ -88,6 +96,54 @@ } +def _distribution_map() -> dict[str, str]: + """The import→distribution table, preferring the authority over the mirror. + + The installed Gramps (6.1+) ships the authoritative table, + ``_IMPORT_TO_PYPI`` in ``gramps/gen/utils/pypi.py``; read it so CI installs + exactly the distribution Gramps' own installer would. Fall back to the + local ``_IMPORT_TO_DISTRIBUTION`` mirror when gramps is absent (image + build, bare runner) or predates 6.1. The table dict is read directly rather + than calling ``resolve_pypi_name()``: same data, but without one + ``LOG.warning`` per translated name in every CI step — and that warning + advises declaring the PyPI name, the opposite of the import-name contract + ``--check-resolves`` enforces. + + A half-installed gramps can raise ``SystemExit`` at import (ResourcePath + aborts when its resources are missing), so the guard is broader than + ``ImportError``. + """ + try: + from gramps.gen.utils import pypi as _gramps_pypi + except (Exception, SystemExit): + return _IMPORT_TO_DISTRIBUTION + table = getattr(_gramps_pypi, "_IMPORT_TO_PYPI", None) + return dict(table) if table else _IMPORT_TO_DISTRIBUTION + + +def _module_checker() -> tuple[str, Callable[[str], bool]]: + """A ``(label, check)`` pair implementing Gramps' requires_mod gate. + + Delegates to the installed gramps' ``Requirements().check_mod`` so the + gate matches whichever series this lane ships: bare ``find_spec`` on 6.0, + ``find_spec`` plus a real import on 6.1+ (gramps PR #2308). The stdlib + ``find_spec`` fallback only applies where gramps is not importable (a dev + box, never a CI lane) and deliberately stays find_spec-only: the gate + exists to catch *declaration* bugs, and really importing every declared + mod on an arbitrary machine is slow and side-effectful. + """ + try: + from gramps.gen.utils.requirements import Requirements + except (Exception, SystemExit): + from importlib.util import find_spec + + return ( + "stdlib find_spec (gramps not importable)", + lambda name: find_spec(name) is not None, + ) + return "gramps Requirements().check_mod", Requirements().check_mod + + def _gpr_files(root: str) -> list[str]: return sorted(glob.glob(os.path.join(root, "*", "*.gpr.py"))) @@ -139,40 +195,48 @@ def install_list(root: str) -> list[str]: list the ci.yml "Install addon runtime deps" steps pip-install. Best-effort and tolerant — a file that cannot be read, or a non-literal value, is skipped (not fatal), so the step installs what it can resolve.""" - return sorted(_IMPORT_TO_DISTRIBUTION.get(mod, mod) for mod in declared_mods(root)) + table = _distribution_map() + return sorted(table.get(mod, mod) for mod in declared_mods(root)) def check_resolves(root: str) -> int: """Validate that every declared ``requires_mod`` import name which actually - pip-installed also resolves under ``importlib.util.find_spec`` — the same - check Gramps' ``Requirements.check_mod`` performs at runtime. The *raw* - declared name is checked (NOT the install-mapped distribution name), because - that is what Gramps imports. - - Run this *after* the install union has been pip-installed: a name that pip - never installed (an exotic system-dep / image gap, not a PR bug) is skipped; - a name that pip-installed yet still fails ``find_spec`` is a wrong - declaration (e.g. the PyPI distribution ``"Pillow"`` instead of the import - name ``"PIL"``) and fails the gate. Returns 1 if any bad name, else 0. - Preserves the behaviour of the old inline validator heredoc verbatim.""" + pip-installed also passes Gramps' own dependency gate, + ``Requirements().check_mod`` — bare ``find_spec`` on gramps 6.0, + ``find_spec`` plus a real import on 6.1+ (gramps PR #2308) — so the gate + matches whichever series this lane ships. The *raw* declared name is + checked (NOT the install-mapped distribution name), because that is what + Gramps imports; installed-ness however is probed with the *mapped* + distribution name, because that is the only name pip knows (``pip show + PIL`` fails even with Pillow installed). + + Run this *after* the install union has been pip-installed: a name whose + distribution pip never installed (an exotic system-dep / image gap, not a + PR bug) is skipped; a name that pip-installed yet still fails the gate is a + wrong declaration (e.g. the PyPI distribution ``"Pillow"`` instead of the + import name ``"PIL"``) — or, on 6.1+, a module that installs but cannot + import — and fails the run. Returns 1 if any bad name, else 0.""" import subprocess - from importlib.util import find_spec + label, check = _module_checker() + print(f"dep gate: {label}") + table = _distribution_map() bad: list[str] = [] for name in sorted(declared_mods(root)): + dist = table.get(name, name) installed = ( subprocess.run( - [sys.executable, "-m", "pip", "show", name], + [sys.executable, "-m", "pip", "show", dist], capture_output=True, ).returncode == 0 ) if not installed: - print(f"~ {name} (pip-install failed earlier, skipping)") + print(f"~ {name} (pip never installed {dist}, skipping)") continue - if find_spec(name) is None: + if not check(name): bad.append(name) - print(f"x {name} (pip-installed but find_spec returned None)") + print(f"x {name} (installed as {dist} but fails Gramps' dep gate)") else: print(f"ok {name}") @@ -180,8 +244,9 @@ def check_resolves(root: str) -> int: print() print(f"::error::Wrong requires_mod names: {bad}") print("These pip-install but are not importable. requires_mod is") - print("consumed by gramps' check_mod() via find_spec(), so the") - print("importable module name is required (e.g. 'PIL', not 'Pillow').") + print("consumed by gramps' check_mod() — find_spec, plus a real import") + print("on gramps 6.1+ — so the importable module name is required") + print("(e.g. 'PIL', not 'Pillow').") return 1 return 0 @@ -202,8 +267,9 @@ def main(argv: list[str] | None = None) -> int: "--check-resolves", metavar="ROOT", help="verify every declared requires_mod import name that pip-installed " - "also resolves under importlib.util.find_spec (Gramps' check_mod); " - "exit 1 if any installs but does not import", + "also passes Gramps' dep gate (Requirements().check_mod when gramps is " + "importable, stdlib find_spec otherwise); exit 1 if any installs but " + "does not import", ) args = parser.parse_args(argv) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8893400a0..a06555656 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,7 +201,7 @@ jobs: # wheel-only or source-built (with a system package), so the install list # never silently drifts from what addons declare — the build-toolchain # coverage gap returns the moment a new source-built requires_mod is added - # without a mapping. (The system-dep analogue of the requires_mod find_spec + # without a mapping. (The system-dep analogue of the requires_mod dep # gate below.) shell: bash run: | @@ -216,7 +216,11 @@ jobs: # (gramps/gui/plug/_windows.py __on_install_clicked → req.install → # gen/utils/requirements.py). Keeps .gpr.py files as the single # source of truth for addon deps — no parallel list to maintain - # in the image or workflow. Best-effort: a package needing exotic + # in the image or workflow. Import→distribution names (PIL→Pillow) + # come from the installed gramps' own _IMPORT_TO_PYPI table + # (gen/utils/pypi.py, gramps 6.1+, PR #2308), with a local mirror + # in addon_python_deps.py for lanes on 6.0. Best-effort: a package + # needing exotic # system deps (pygraphviz → graphviz-dev, psycopg2 → libpq-dev) # may fail here; the affected addon's tests will skip or fail in # isolation without blocking the rest. @@ -234,13 +238,17 @@ jobs: - name: Validate requires_mod names against Gramps' dep gate # Cross-check: every requires_mod entry that pip successfully - # installed in the previous step must also resolve via - # find_spec(), since that is what Gramps' Addon Manager calls - # (gramps/gen/utils/requirements.py:check_mod). A name that - # pip-installs but does not import is a declaration bug — e.g. - # requires_mod=["Pillow"] when the importable name is "PIL". - # Pip-install failures upstream are skipped: those are - # system-dep / image gaps, not PR-caused. + # installed in the previous step must also pass Gramps' own dep + # gate — the script delegates to the installed gramps' + # Requirements().check_mod (gramps/gen/utils/requirements.py): + # find_spec on 6.0; find_spec plus a real import on 6.1+ (gramps + # PR #2308), so the gate matches whichever series this lane + # ships. A name that pip-installs but does not import is a + # declaration bug — e.g. requires_mod=["Pillow"] when the + # importable name is "PIL". Pip-installed-ness is probed by the + # mapped distribution name (pip only knows "Pillow", not "PIL"); + # names whose distribution never installed are skipped: those + # are system-dep / image gaps, not PR-caused. shell: bash run: | python3 .github/scripts/addon_python_deps.py --check-resolves . diff --git a/tests/test_addon_python_deps.py b/tests/test_addon_python_deps.py new file mode 100644 index 000000000..0263d30d9 --- /dev/null +++ b/tests/test_addon_python_deps.py @@ -0,0 +1,229 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +"""The requires_mod machinery must track Gramps' own (gramps PR #2308). + +gramps 6.1 ships an install-time authority for addon Python deps: +``gramps/gen/utils/pypi.py`` with the import→distribution table +``_IMPORT_TO_PYPI``, and a ``Requirements.check_mod`` that really imports the +module after ``find_spec`` (merged as 7f94428b13; not backported to 6.0). +``addon_python_deps.py`` accounts for both at lookup time: + +* ``_distribution_map()`` prefers the installed gramps' ``_IMPORT_TO_PYPI`` + over the local ``_IMPORT_TO_DISTRIBUTION`` fallback mirror, so 6.1+ lanes + install exactly what Gramps' own installer would — including entries added + upstream after this file was written. +* ``_module_checker()`` delegates ``--check-resolves`` to the installed + gramps' ``Requirements().check_mod``, so the gate is find_spec-only on 6.0 + and find_spec-plus-real-import on 6.1+, per lane, automatically. +* the pip-installed-ness probe uses the *mapped distribution* name — ``pip + show PIL`` fails even with Pillow installed, so probing the raw import name + silently skipped exactly the declarations the mapping machinery exists for. + +These tests pin the two seams and the probe fix hermetically (fake gramps +module trees injected via ``mock.patch.dict(sys.modules, ...)``; no network, +no real gramps needed), plus one sync-guard that compares the fallback mirror +against the real authority table wherever gramps >= 6.1 is importable — the +gramps61 CI lanes — and skips elsewhere. Mirror drift can only originate from +a 6.1+ table change, which exactly those lanes catch. + +Import guards use ``except (Exception, SystemExit)`` throughout: a +half-installed gramps (raw source checkout on sys.path) raises ``SystemExit`` +from ResourcePath at import, not ImportError. +""" + +# ------------------------ +# Python modules +# ------------------------ +from __future__ import annotations + +import io +import os +import sys +import types +import unittest +from contextlib import redirect_stdout +from unittest import mock + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_HERE) # repo root (tests/ lives directly under it) +_SCRIPTS = os.path.join(_REPO_ROOT, ".github", "scripts") + +sys.path.insert(0, _SCRIPTS) +import addon_python_deps as deps # noqa: E402 + + +def _fake_gramps_tree(**leaves: types.ModuleType) -> dict[str, types.ModuleType]: + """A sys.modules overlay for ``gramps.gen.utils`` plus the given leaves. + + ``leaves`` maps a leaf module name (``pypi``, ``requirements``) to a fake + module; the returned dict contains the full parent chain with attributes + linked, ready for ``mock.patch.dict(sys.modules, ...)``. + """ + gramps = types.ModuleType("gramps") + gen = types.ModuleType("gramps.gen") + utils = types.ModuleType("gramps.gen.utils") + gramps.gen = gen + gen.utils = utils + tree = {"gramps": gramps, "gramps.gen": gen, "gramps.gen.utils": utils} + for name, module in leaves.items(): + setattr(utils, name, module) + tree[f"gramps.gen.utils.{name}"] = module + return tree + + +# The overlay that makes every ``import gramps...`` raise ImportError, even on +# a machine where the real gramps is importable (None in sys.modules halts the +# import of the top-level package before any submodule is considered). +_NO_GRAMPS = {"gramps": None} + + +class GrampsTableSync(unittest.TestCase): + """The fallback mirror must equal Gramps' authority table (gramps >= 6.1).""" + + def test_local_mirror_matches_gramps_table(self) -> None: + try: + from gramps.gen.utils import pypi + except (Exception, SystemExit): + self.skipTest( + "gramps.gen.utils.pypi not importable — the sync-guard gates " + "on gramps >= 6.1 lanes; the mirror is inert here" + ) + # A loud failure on rename: at runtime getattr degrades safely to the + # mirror, so THIS assertion is the only place an upstream rename of + # _IMPORT_TO_PYPI surfaces. + self.assertTrue( + hasattr(pypi, "_IMPORT_TO_PYPI"), + "gramps.gen.utils.pypi no longer exposes _IMPORT_TO_PYPI — " + "update _distribution_map() and this mirror sync-guard", + ) + self.assertEqual( + dict(pypi._IMPORT_TO_PYPI), + deps._IMPORT_TO_DISTRIBUTION, + "_IMPORT_TO_DISTRIBUTION has drifted from gramps' " + "_IMPORT_TO_PYPI — re-sync the fallback mirror in " + ".github/scripts/addon_python_deps.py", + ) + + +class DistributionMapSeam(unittest.TestCase): + """_distribution_map(): authority when present, mirror otherwise.""" + + def test_prefers_gramps_table(self) -> None: + fake_pypi = types.ModuleType("gramps.gen.utils.pypi") + fake_pypi._IMPORT_TO_PYPI = {"PIL": "Pillow", "fake_mod": "fake-dist"} + with mock.patch.dict(sys.modules, _fake_gramps_tree(pypi=fake_pypi)): + table = deps._distribution_map() + self.assertEqual(table["fake_mod"], "fake-dist") + self.assertEqual(table["PIL"], "Pillow") + + def test_falls_back_without_gramps(self) -> None: + with mock.patch.dict(sys.modules, _NO_GRAMPS): + table = deps._distribution_map() + self.assertEqual(table, deps._IMPORT_TO_DISTRIBUTION) + + def test_falls_back_when_table_attr_missing(self) -> None: + # Pins the safe-degradation path an upstream rename would take. + fake_pypi = types.ModuleType("gramps.gen.utils.pypi") + with mock.patch.dict(sys.modules, _fake_gramps_tree(pypi=fake_pypi)): + table = deps._distribution_map() + self.assertEqual(table, deps._IMPORT_TO_DISTRIBUTION) + + +class ModuleCheckerSeam(unittest.TestCase): + """_module_checker(): Gramps' own gate when present, find_spec otherwise.""" + + def test_delegates_to_gramps_check_mod(self) -> None: + calls: list[str] = [] + + class _FakeRequirements: + def check_mod(self, name: str) -> bool: + calls.append(name) + return name == "good_mod" + + fake_req = types.ModuleType("gramps.gen.utils.requirements") + fake_req.Requirements = _FakeRequirements + with mock.patch.dict(sys.modules, _fake_gramps_tree(requirements=fake_req)): + label, check = deps._module_checker() + self.assertIn("check_mod", label) + self.assertTrue(check("good_mod")) + self.assertFalse(check("bad_mod")) + self.assertEqual(calls, ["good_mod", "bad_mod"]) + + def test_stdlib_fallback(self) -> None: + with mock.patch.dict(sys.modules, _NO_GRAMPS): + label, check = deps._module_checker() + self.assertIn("find_spec", label) + self.assertTrue(check("os")) + self.assertFalse(check("definitely_not_a_module_xyz")) + + +class CheckResolvesGate(unittest.TestCase): + """check_resolves(): probe by distribution name, judge by import name.""" + + def _run(self, *, check, pip_ok_for: set[str], recorded: list[list[str]]): + """Drive check_resolves with one declared mod, ``PIL``, hermetically.""" + + def fake_run(argv, **kwargs): + recorded.append(list(argv)) + rc = 0 if argv[-1] in pip_ok_for else 1 + return types.SimpleNamespace(returncode=rc) + + out = io.StringIO() + with ( + mock.patch.object(deps, "declared_mods", return_value={"PIL"}), + mock.patch.object( + deps, "_distribution_map", return_value={"PIL": "Pillow"} + ), + mock.patch.object( + deps, "_module_checker", return_value=("test gate", check) + ), + mock.patch("subprocess.run", side_effect=fake_run), + redirect_stdout(out), + ): + rc = deps.check_resolves(".") + return rc, out.getvalue() + + def test_probe_uses_distribution_name(self) -> None: + # Regression: `pip show PIL` fails even with Pillow installed, so the + # old raw-name probe skipped ("~") the one declaration the mapping + # machinery exists for — never validating it. + recorded: list[list[str]] = [] + rc, out = self._run( + check=lambda name: True, pip_ok_for={"Pillow"}, recorded=recorded + ) + self.assertEqual(rc, 0) + pip_show = [argv for argv in recorded if "show" in argv] + self.assertTrue(pip_show and pip_show[0][-1] == "Pillow", pip_show) + self.assertNotIn("~", out) + self.assertIn("ok PIL", out) + + def test_installed_but_unresolvable_fails(self) -> None: + recorded: list[list[str]] = [] + rc, out = self._run( + check=lambda name: False, pip_ok_for={"Pillow"}, recorded=recorded + ) + self.assertEqual(rc, 1) + self.assertIn("x PIL", out) + self.assertIn("::error::", out) + + +if __name__ == "__main__": + unittest.main() From 5911ebf403847c3aa3cdf391c08e71a6a850c7f2 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 02:20:39 +0200 Subject: [PATCH 31/47] tests: make the no-gramps overlay defeat-proof; pin the resolve gate to the raw name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two gaps the adversarial review found in the test module added for gramps PR #2308: 1. The {"gramps": None} overlay does not force ImportError when a gramps submodule is already cached — `from gramps.gen.utils.requirements import Requirements` short-circuits on the cached submodule and never consults the None-ed parent. unittest discovery imports test_plugin_registration, which caches gramps.gen.utils.requirements at module level, so on the integration lane test_stdlib_fallback took the DELEGATED path and false-red'd (verified: label 'gramps Requirements().check_mod' instead of 'stdlib find_spec'). Replace the constant with _no_gramps_overlay(), which also None-outs every already-cached gramps.* key. Both fallback tests now seed a cached fake gramps tree first, then overlay — so they prove the overlay defeats a pre-cached submodule rather than merely an absent gramps. 2. CheckResolvesGate's stub checkers were constant lambdas, so a mutant that judged the mapped DISTRIBUTION name (check(dist)) instead of the raw import name survived — in production that would make gramps 6.1's check_mod("Pillow") fail a correct requires_mod=["PIL"]. The checkers now record their argument and assert they were asked about "PIL". Also corrects the sync-guard docstring: the guard is latent until these workflows land on maintenance/gramps61 (no current lane imports gramps >= 6.1), not something 'those lanes catch' today. --- tests/test_addon_python_deps.py | 92 ++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 20 deletions(-) diff --git a/tests/test_addon_python_deps.py b/tests/test_addon_python_deps.py index 0263d30d9..9fd1f7d45 100644 --- a/tests/test_addon_python_deps.py +++ b/tests/test_addon_python_deps.py @@ -40,9 +40,15 @@ These tests pin the two seams and the probe fix hermetically (fake gramps module trees injected via ``mock.patch.dict(sys.modules, ...)``; no network, no real gramps needed), plus one sync-guard that compares the fallback mirror -against the real authority table wherever gramps >= 6.1 is importable — the -gramps61 CI lanes — and skips elsewhere. Mirror drift can only originate from -a 6.1+ table change, which exactly those lanes catch. +against the real authority table wherever gramps >= 6.1 is importable. + +NOTE on the sync-guard's reach: it is LATENT until these workflows land on +``maintenance/gramps61``. Today no CI lane imports a gramps that ships +``gramps.gen.utils.pypi`` — the gramps60 image predates it, and the conda +Windows lane pins 6.0.x — so every lane skips the guard; it self-activates +when the pipeline reaches the 6.1 branch. Keep it: the cost is one skipped +test until then, and it is the only place a mirror-vs-authority drift is +caught once 6.1 arrives. Import guards use ``except (Exception, SystemExit)`` throughout: a half-installed gramps (raw source checkout on sys.path) raises ``SystemExit`` @@ -89,10 +95,23 @@ def _fake_gramps_tree(**leaves: types.ModuleType) -> dict[str, types.ModuleType] return tree -# The overlay that makes every ``import gramps...`` raise ImportError, even on -# a machine where the real gramps is importable (None in sys.modules halts the -# import of the top-level package before any submodule is considered). -_NO_GRAMPS = {"gramps": None} +def _no_gramps_overlay() -> dict[str, None]: + """A sys.modules overlay under which every ``import gramps...`` fails. + + A top-level ``{"gramps": None}`` alone is NOT enough: ``from + gramps.gen.utils.requirements import Requirements`` short-circuits on a + cached ``sys.modules['gramps.gen.utils.requirements']`` and never consults + the None-ed parent. And a cached submodule is the norm on CI — unittest + discovery imports ``test_plugin_registration``, which imports + ``gramps.gen.utils.requirements`` at module level, before these tests run. + So None-out the top-level name AND every already-cached ``gramps.*`` key; + that restores the ImportError the fallback branches must see. + """ + overlay: dict[str, None] = {"gramps": None} + for name in list(sys.modules): + if name == "gramps" or name.startswith("gramps."): + overlay[name] = None + return overlay class GrampsTableSync(unittest.TestCase): @@ -135,8 +154,15 @@ def test_prefers_gramps_table(self) -> None: self.assertEqual(table["PIL"], "Pillow") def test_falls_back_without_gramps(self) -> None: - with mock.patch.dict(sys.modules, _NO_GRAMPS): - table = deps._distribution_map() + # Seed a cached gramps.gen.utils.pypi FIRST (as the integration lane's + # discovery would leave gramps submodules cached), then apply the + # overlay on top: the fallback must still fire, proving the overlay + # defeats a pre-cached submodule and not merely an absent gramps. + fake_pypi = types.ModuleType("gramps.gen.utils.pypi") + fake_pypi._IMPORT_TO_PYPI = {"PIL": "Pillow", "seeded": "seeded-dist"} + with mock.patch.dict(sys.modules, _fake_gramps_tree(pypi=fake_pypi)): + with mock.patch.dict(sys.modules, _no_gramps_overlay()): + table = deps._distribution_map() self.assertEqual(table, deps._IMPORT_TO_DISTRIBUTION) def test_falls_back_when_table_attr_missing(self) -> None: @@ -168,11 +194,24 @@ def check_mod(self, name: str) -> bool: self.assertEqual(calls, ["good_mod", "bad_mod"]) def test_stdlib_fallback(self) -> None: - with mock.patch.dict(sys.modules, _NO_GRAMPS): - label, check = deps._module_checker() - self.assertIn("find_spec", label) - self.assertTrue(check("os")) - self.assertFalse(check("definitely_not_a_module_xyz")) + # Seed a cached gramps.gen.utils.requirements FIRST (exactly what + # unittest discovery of test_plugin_registration leaves behind on the + # integration lane), then overlay: the checker must STILL fall back to + # find_spec. Without the full-tree overlay this false-red'd — the + # delegated path was taken because the submodule was already cached. + fake_req = types.ModuleType("gramps.gen.utils.requirements") + + class _FakeRequirements: + def check_mod(self, name: str) -> bool: # pragma: no cover + return True + + fake_req.Requirements = _FakeRequirements + with mock.patch.dict(sys.modules, _fake_gramps_tree(requirements=fake_req)): + with mock.patch.dict(sys.modules, _no_gramps_overlay()): + label, check = deps._module_checker() + self.assertIn("find_spec", label) + self.assertTrue(check("os")) + self.assertFalse(check("definitely_not_a_module_xyz")) class CheckResolvesGate(unittest.TestCase): @@ -206,23 +245,36 @@ def test_probe_uses_distribution_name(self) -> None: # old raw-name probe skipped ("~") the one declaration the mapping # machinery exists for — never validating it. recorded: list[list[str]] = [] - rc, out = self._run( - check=lambda name: True, pip_ok_for={"Pillow"}, recorded=recorded - ) + checked: list[str] = [] + + def check(name: str) -> bool: + checked.append(name) + return True + + rc, out = self._run(check=check, pip_ok_for={"Pillow"}, recorded=recorded) self.assertEqual(rc, 0) pip_show = [argv for argv in recorded if "show" in argv] self.assertTrue(pip_show and pip_show[0][-1] == "Pillow", pip_show) self.assertNotIn("~", out) self.assertIn("ok PIL", out) + # The gate must JUDGE the raw import name, never the distribution name — + # gramps' check_mod("Pillow") would wrongly fail a correct requires_mod + # =["PIL"]. (Kills the M1b mutant that passed `dist` to the checker.) + self.assertEqual(checked, ["PIL"]) def test_installed_but_unresolvable_fails(self) -> None: recorded: list[list[str]] = [] - rc, out = self._run( - check=lambda name: False, pip_ok_for={"Pillow"}, recorded=recorded - ) + checked: list[str] = [] + + def check(name: str) -> bool: + checked.append(name) + return False + + rc, out = self._run(check=check, pip_ok_for={"Pillow"}, recorded=recorded) self.assertEqual(rc, 1) self.assertIn("x PIL", out) self.assertIn("::error::", out) + self.assertEqual(checked, ["PIL"]) if __name__ == "__main__": From 0704d72a3968dc9de6aa7ba2210aa3f7f8dbf6ab Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 02:25:59 +0200 Subject: [PATCH 32/47] tests: give the heredoc-revert guard teeth; pin the dedup oracle's map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two adversarial-review findings in the dedup test: 1. test_no_requires_mod_heredoc_remains was a tautology. Its guard, re.findall(r"requires_mod\s*=\s*\(\[", text), can never match the old heredoc's own source line pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") because that line contains the LITERAL characters \s* while the guard's \s* matches whitespace. Pasting the removed heredoc back into ci.yml kept the test green. Replace it with assertNotIn of the heredoc's literal fragments (re.compile(r"requires_mod and requires_mod\s*), both of which appear verbatim in the pre-dedup ci.yml (6x) and nowhere in the current file — so a revert now reds. 2. test_install_list_matches_old_heredoc's oracle hardcodes _INSTALL_MAP= {PIL: Pillow}. Since #2308, production install_list() consults gramps' authoritative _IMPORT_TO_PYPI when a gramps >= 6.1 is importable, so on a 6.1 lane an upstream table entry for any declared mod would flip this oracle red for a non-regression. Pin _distribution_map to the local mirror inside the comparison; comment the two-step fix (GrampsTableSync reds -> re-sync mirror -> extend _INSTALL_MAP). --- tests/test_requires_mod_dedup.py | 37 ++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/tests/test_requires_mod_dedup.py b/tests/test_requires_mod_dedup.py index 8490911a2..a468f3a69 100644 --- a/tests/test_requires_mod_dedup.py +++ b/tests/test_requires_mod_dedup.py @@ -29,6 +29,7 @@ import re import sys import unittest +from unittest import mock # Repo layout is fixed relative to this file: tests/ sits at the addons-source # root, and the CI scripts live under .github/scripts/ — resolve both from @@ -96,7 +97,21 @@ class RequiresModDerivationDedup(unittest.TestCase): def test_install_list_matches_old_heredoc(self): old = _old_raw_union(_REPO_ROOT) expected = sorted(_INSTALL_MAP.get(m, m) for m in old) - self.assertEqual(addon_python_deps.install_list(_REPO_ROOT), expected) + # Pin the import->distribution table to the LOCAL mirror for this + # comparison. The oracle (_INSTALL_MAP) states the old heredoc's + # behaviour plus the install-only map; production install_list() now + # ALSO consults gramps' authoritative _IMPORT_TO_PYPI when a gramps + # >= 6.1 is importable (the gramps61 lanes). Without this pin, an + # upstream table gaining a mapping for a *declared* mod would flip this + # oracle red for a change that is not a regression here. If that + # happens: the GrampsTableSync guard reds first (re-sync the mirror in + # addon_python_deps.py), then extend _INSTALL_MAP above to match. + with mock.patch.object( + addon_python_deps, + "_distribution_map", + return_value=dict(addon_python_deps._IMPORT_TO_DISTRIBUTION), + ): + self.assertEqual(addon_python_deps.install_list(_REPO_ROOT), expected) def test_declared_raw_names_match_old_heredoc(self): # The find_spec gate consumes RAW import names — these must equal the @@ -112,12 +127,22 @@ def test_install_map_is_install_only(self): self.assertNotIn("Pillow", addon_python_deps.declared_mods(_REPO_ROOT)) def test_no_requires_mod_heredoc_remains(self): + # The previous assertion (re.findall(r"requires_mod\s*=\s*\(\[", ...)) + # was a tautology: the old inline heredoc's distinctive line was + # pat = re.compile(r"requires_mod\s*=\s*(\[[^\]]*\])") + # whose text contains the LITERAL characters `\s*`, which the guard's + # own `\s*` (matching whitespace) can never match — so it never bit, + # and pasting the heredoc back in stayed green. Match the heredoc's + # own literal fragments instead; both appear verbatim in the pre-dedup + # ci.yml and in none of the current file. text = _read(_CI_YML) - self.assertEqual( - re.findall(r"requires_mod\s*=\s*\(\[", text), - [], - "a requires_mod derivation heredoc still lives inline in ci.yml", - ) + for fragment in ('re.compile(r"requires_mod', r"requires_mod\s*"): + self.assertNotIn( + fragment, + text, + "a requires_mod derivation heredoc still lives inline in ci.yml " + f"(found {fragment!r})", + ) def test_three_jobs_consume_the_module(self): text = _read(_CI_YML) From 716389d738daa1b82afbe29a3d86d558109b9688 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 02:33:10 +0200 Subject: [PATCH 33/47] tests: scan the Dockerfile toolchain check as logical lines, cover python3-dev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The build-toolchain assertions were line-based substring checks and let three regressions through (adversarial-review mutants, all verified surviving): - M4c: a multi-line `RUN apt-get purge -y \` with gcc/python3-dev on the continuation lines — invisible to a per-physical-line scan. - M4d: `apt-get -y purge gcc` (flag before the verb) — the old 'apt-get purge' substring test missed it. - M4e: dropping python3-dev from the install — test_toolchain_is_installed only checked gcc and pkg-config. Join backslash continuations into logical lines first; match any apt-get removal verb (purge/remove/autoremove) in any flag order via regex; require each of gcc, pkg-config AND python3-dev to appear as a whole token on an actual `apt-get install` line (not merely somewhere in the file — a comment or a purge line must not satisfy it). The real Dockerfile still passes. --- tests/test_addon_system_deps.py | 61 +++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/tests/test_addon_system_deps.py b/tests/test_addon_system_deps.py index b22e9ea55..719151553 100644 --- a/tests/test_addon_system_deps.py +++ b/tests/test_addon_system_deps.py @@ -244,41 +244,66 @@ def test_unmapped_cli_exit_zero(self): ) +# The compiler toolchain a source-built requires_mod (pygraphviz, psycopg2, …) +# needs at CI runtime. All three must stay installed and unpurged in the image. +_TOOLCHAIN = ("gcc", "pkg-config", "python3-dev") +# `apt-get` followed by any flags/words and then a removal verb / an install. +_APT_PURGE_RE = re.compile(r"\bapt-get\b(?:\s+\S+)*?\s+(?:purge|remove|autoremove)\b") +_APT_INSTALL_RE = re.compile(r"\bapt-get\b(?:\s+\S+)*?\s+install\b") + + +def _mentions(text: str, tool: str): + """A whole-token match for a package name (so pkg-config != pkg-configurator).""" + return re.search(rf"(? Date: Mon, 20 Jul 2026 02:35:24 +0200 Subject: [PATCH 34/47] ci: least-privilege token, drift gate on the Windows lane, pinned ruff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three security/robustness fixes from the adversarial review: - Add top-level 'permissions: contents: read' to ci.yml. The workflow runs addon-influenced code (pip package hooks from requires_mod; addon import-time code in the load/test lanes) and had no permissions block, so on push runs that code inherited the ambient default token scope. Pulling the public GHCR image needs no packages scope. (docker-build.yml already scopes its token.) - Add the '--unmapped' drift gate to unit-test-windows, before its pip step. The gate previously existed only on unit-test-linux; unit-test-windows 'needs: setup' alone (not unit-test-linux), so a novel requires_mod name reached 'pip install' unguarded on the Windows lane — arbitrary package execution — even though Linux blocked it. Both lanes now gate independently. - Pin ruff==0.15.22 in the CI image. Unpinned, a ruff release could flip the lint verdict (new rule in the E9/F63/F7/F82 selection) on any rebuild with no repo change; the pin makes the gate reproducible. Bump deliberately. Dedup pins (exactly-3 --install-list/--check-resolves, is_active sourcing) unaffected: the new step calls addon_system_deps.py, and permissions is not a job step. --- .github/docker/gramps-ci/Dockerfile | 5 ++++- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index f65a5f949..9cc3d7c81 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -79,7 +79,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # requires_mod)" step pip-installs them at CI runtime from every # .gpr.py's requires_mod list, matching what Gramps' Addon Manager does # for an end user. Keeps .gpr.py the single source of truth. -RUN pip install --no-cache-dir PyGObject pycairo orjson ruff +# ruff is PINNED: an unpinned ruff would change the lint verdict on rebuild (a +# new rule in the E9/F63/F7/F82 selection flips a lane red — or silently stops +# flagging — with no repo change). Bump deliberately, not by rebuild drift. +RUN pip install --no-cache-dir PyGObject pycairo orjson ruff==0.15.22 # Install gramps: PyPI first, SHA-pinned git clone as fallback. RUN /bin/bash -eo pipefail <<'BASH' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a06555656..fc23ba822 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,15 @@ on: pull_request: branches: [maintenance/gramps**] +# Least privilege. Every job only reads the repo; the CI image is a public +# GHCR package, so pulling it needs no `packages` scope. This workflow runs +# addon-influenced code (pip package hooks derived from requires_mod, and +# addon import-time code in the load/test lanes), so the ambient token it +# hands that code must not carry write scopes. Presumes the gramps-ci package +# is public (the one-time visibility flip in .github/CI-MAINTAINER.md). +permissions: + contents: read + jobs: # ----------------------------------------------------------------- # Setup — derive the branch suffix (gramps60 / gramps61 / …) from @@ -377,6 +386,19 @@ jobs: echo "no conda-available addon system deps to install" fi + - name: Validate addon system deps are mapped + # Same drift gate as unit-test-linux (see its comment). This lane runs + # it independently and BEFORE its own pip step: unit-test-windows only + # `needs: setup`, not unit-test-linux, so it must not lean on the Linux + # gate — otherwise a novel requires_mod would reach `pip install` + # unguarded here (arbitrary package execution) while Linux blocks it. + # `python` (conda-forge env) to match the surrounding Windows style. + run: | + python .github/scripts/addon_system_deps.py --unmapped . || { + echo "::error::Addon(s) declare requires_gi/requires_exe/requires_mod with no entry in .github/scripts/addon_system_deps.py — add a mapping row (or, for a wheel-only module, list it in WHEEL_ONLY_MODS)." + exit 1 + } + - name: Install addon runtime deps (derived from requires_mod) # See unit-test-linux for rationale. Uses `python` (conda-forge # env) to match the surrounding Windows job style. From 02f0ba7a5440b703a2d313df1adbefafb0ecd4b8 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 02:38:26 +0200 Subject: [PATCH 35/47] ci-scripts: fail the dep gate when a wheel-only requires_mod never installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dep-resolution gate skipped ('~') every requires_mod whose distribution pip never installed, treating install failure as an environment gap. For source-built modules (pygraphviz, psycopg2 — need a system -dev package) that is right. But a wheel-only module (WHEEL_ONLY_MODS: PIL, boto3, litellm, networkx, pymongo, svgwrite, dbf, life_line_chart) ships a pure/binary wheel that installs on every CI platform, so a miss there is a real provisioning regression that the previous 'skip' let scroll by green — the fail-open hole the review flagged for the addons that have no tests of their own. check_resolves now splits the not-installed branch on the classification set (imported from the sibling addon_system_deps): wheel-only misses FAIL with a distinct ::error::, source-built misses stay advisory skips. Tests exercise both against the real WHEEL_ONLY_MODS (not mocked), and assert a never- installed wheel is never even handed to the dep checker. --- .github/scripts/addon_python_deps.py | 42 +++++++++++++++---- .github/workflows/ci.yml | 9 ++-- tests/test_addon_python_deps.py | 63 +++++++++++++++++++++++++--- 3 files changed, 96 insertions(+), 18 deletions(-) diff --git a/.github/scripts/addon_python_deps.py b/.github/scripts/addon_python_deps.py index fe4b38587..5c2aa06cb 100644 --- a/.github/scripts/addon_python_deps.py +++ b/.github/scripts/addon_python_deps.py @@ -57,6 +57,13 @@ import sys from collections.abc import Callable +# Sibling module in the same directory: the wheel-only vs source-built +# classification of every declared requires_mod. Running this file by path +# already puts its dir on sys.path[0]; the insert also covers being imported +# (the tests, and any embedding). Pure stdlib either way — no gramps import. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from addon_system_deps import WHEEL_ONLY_MODS # noqa: E402 + # Matches a single-line ``requires_mod = ["a", "b"]`` assignment — the same # shape addon_system_deps.py's _GI_RE/_EXE_RE assume, and the only shape that # occurs in addons-source. @@ -210,18 +217,23 @@ def check_resolves(root: str) -> int: distribution name, because that is the only name pip knows (``pip show PIL`` fails even with Pillow installed). - Run this *after* the install union has been pip-installed: a name whose - distribution pip never installed (an exotic system-dep / image gap, not a - PR bug) is skipped; a name that pip-installed yet still fails the gate is a - wrong declaration (e.g. the PyPI distribution ``"Pillow"`` instead of the - import name ``"PIL"``) — or, on 6.1+, a module that installs but cannot - import — and fails the run. Returns 1 if any bad name, else 0.""" + Run this *after* the install union has been pip-installed. A name whose + distribution pip never installed is judged by its category: a **wheel-only** + module (``WHEEL_ONLY_MODS``) ships a pure/binary wheel that installs on + every CI platform, so a miss is a real provisioning regression and FAILS + the gate; a **source-built** module (``MOD_BUILD_PACKAGES``, e.g. pygraphviz + / psycopg2) can legitimately miss on an image/system gap and stays an + advisory skip. A name that pip-installed yet still fails the gate is a wrong + declaration (e.g. the PyPI distribution ``"Pillow"`` instead of the import + name ``"PIL"``) — or, on 6.1+, a module that installs but cannot import — + and fails the run. Returns 1 if any bad or missing-wheel name, else 0.""" import subprocess label, check = _module_checker() print(f"dep gate: {label}") table = _distribution_map() bad: list[str] = [] + missing_wheels: list[str] = [] for name in sorted(declared_mods(root)): dist = table.get(name, name) installed = ( @@ -232,7 +244,14 @@ def check_resolves(root: str) -> int: == 0 ) if not installed: - print(f"~ {name} (pip never installed {dist}, skipping)") + if name in WHEEL_ONLY_MODS: + missing_wheels.append(name) + print(f"x {name} (wheel-only, but pip never installed {dist})") + else: + print( + f"~ {name} (pip never installed {dist}, skipping — " + "source-built/system-dep)" + ) continue if not check(name): bad.append(name) @@ -247,8 +266,13 @@ def check_resolves(root: str) -> int: print("consumed by gramps' check_mod() — find_spec, plus a real import") print("on gramps 6.1+ — so the importable module name is required") print("(e.g. 'PIL', not 'Pillow').") - return 1 - return 0 + if missing_wheels: + print() + print(f"::error::Wheel-only requires_mod never pip-installed: {missing_wheels}") + print("These declare a pure/binary wheel that installs on every CI") + print("platform, so a failed install is a provisioning regression, not") + print("an environment gap — investigate the install step's output above.") + return 1 if (bad or missing_wheels) else 0 def main(argv: list[str] | None = None) -> int: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc23ba822..a440b3775 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,9 +255,12 @@ jobs: # ships. A name that pip-installs but does not import is a # declaration bug — e.g. requires_mod=["Pillow"] when the # importable name is "PIL". Pip-installed-ness is probed by the - # mapped distribution name (pip only knows "Pillow", not "PIL"); - # names whose distribution never installed are skipped: those - # are system-dep / image gaps, not PR-caused. + # mapped distribution name (pip only knows "Pillow", not "PIL"). + # A name whose distribution never installed is judged by category: + # a wheel-only module (WHEEL_ONLY_MODS) ships a wheel that installs + # everywhere, so a miss is a provisioning regression and FAILS the + # gate; a source-built module (pygraphviz/psycopg2) can miss on an + # image/system gap and stays an advisory skip. shell: bash run: | python3 .github/scripts/addon_python_deps.py --check-resolves . diff --git a/tests/test_addon_python_deps.py b/tests/test_addon_python_deps.py index 9fd1f7d45..16245880b 100644 --- a/tests/test_addon_python_deps.py +++ b/tests/test_addon_python_deps.py @@ -217,8 +217,23 @@ def check_mod(self, name: str) -> bool: # pragma: no cover class CheckResolvesGate(unittest.TestCase): """check_resolves(): probe by distribution name, judge by import name.""" - def _run(self, *, check, pip_ok_for: set[str], recorded: list[list[str]]): - """Drive check_resolves with one declared mod, ``PIL``, hermetically.""" + def _run( + self, + *, + check, + pip_ok_for: set[str], + recorded: list[list[str]], + declared: set[str] | None = None, + dist_map: dict[str, str] | None = None, + ): + """Drive check_resolves hermetically. + + Defaults to one declared mod, ``PIL`` → ``Pillow``. Pass ``declared`` / + ``dist_map`` to exercise the wheel-only vs source-built branches against + the REAL classification sets (WHEEL_ONLY_MODS is not mocked). + """ + declared = {"PIL"} if declared is None else declared + dist_map = {"PIL": "Pillow"} if dist_map is None else dist_map def fake_run(argv, **kwargs): recorded.append(list(argv)) @@ -227,10 +242,8 @@ def fake_run(argv, **kwargs): out = io.StringIO() with ( - mock.patch.object(deps, "declared_mods", return_value={"PIL"}), - mock.patch.object( - deps, "_distribution_map", return_value={"PIL": "Pillow"} - ), + mock.patch.object(deps, "declared_mods", return_value=declared), + mock.patch.object(deps, "_distribution_map", return_value=dist_map), mock.patch.object( deps, "_module_checker", return_value=("test gate", check) ), @@ -276,6 +289,44 @@ def check(name: str) -> bool: self.assertIn("::error::", out) self.assertEqual(checked, ["PIL"]) + def test_wheel_only_never_installed_fails(self) -> None: + # A wheel-only dep (PIL is in the real WHEEL_ONLY_MODS) that pip never + # installed is a provisioning regression, not an environment gap: the + # gate must FAIL, and must not even consult the dep checker. + self.assertIn("PIL", deps.WHEEL_ONLY_MODS) # guard the fixture premise + recorded: list[list[str]] = [] + checked: list[str] = [] + + def check(name: str) -> bool: + checked.append(name) + return True + + rc, out = self._run( + check=check, pip_ok_for=set(), recorded=recorded + ) # nothing installs + self.assertEqual(rc, 1) + self.assertIn("wheel-only", out) + self.assertIn("::error::Wheel-only", out) + self.assertEqual( + checked, [], "a never-installed wheel must not be gate-checked" + ) + + def test_source_built_never_installed_stays_advisory(self) -> None: + # A source-built dep (pygraphviz is in the real MOD_BUILD_PACKAGES, not + # WHEEL_ONLY_MODS) may legitimately miss on an image/system gap: advisory + # skip, rc 0. + self.assertNotIn("pygraphviz", deps.WHEEL_ONLY_MODS) # guard the premise + recorded: list[list[str]] = [] + rc, out = self._run( + check=lambda name: True, + pip_ok_for=set(), + recorded=recorded, + declared={"pygraphviz"}, + dist_map={}, + ) + self.assertEqual(rc, 0) + self.assertIn("~ pygraphviz", out) + if __name__ == "__main__": unittest.main() From e3fb3f7d086f30907b1137bcdbac0f00a2cb1e0c Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 02:49:00 +0200 Subject: [PATCH 36/47] =?UTF-8?q?ci-scripts:=20run=5Faddon=5Ftests=20?= =?UTF-8?q?=E2=80=94=20bounded=20timeout,=20honest=20load-failure=20taxono?= =?UTF-8?q?my,=20zero-test=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four adversarial-review findings in the per-addon test runner: F1 — the per-module timeout was defeated by a grandchild. proc.kill() reaped only the worker; the follow-up proc.communicate() then blocked until any process that inherited the stdout pipe exited, so a test that spawned a long-lived child hung the run to the job cap (verified: a 2s timeout took 16s). The worker now runs in its own process group (start_new_session, POSIX) and a timeout os.killpg's the group, with a bounded follow-up communicate(). A new posix-only test spawns a sleep-120 child inheriting stdout and asserts the run returns in < 45s. F2/F3 — the load-failure taxonomy was inverted AND never fired. A module-level ImportError (the common missing-dep shape) was not raised by loadTestsFromName — since Python 3.5 it is swallowed into a _FailedTest that errors at run time — so it reached the parent as an anonymous broke=1 and hard-failed with no satisfiability check, while a SyntaxError on an unsatisfiable platform was *excused* as a dep skip. The worker now probes the import explicitly (importlib.import_module) and tags it kind=dep|other; the parent excuses only dep-shaped failures on an unsatisfiable platform, and fails a non-dep-shaped load error (SyntaxError, an import-time code bug) on every platform. F4 — a module that loaded but collected zero tests reported ok (unittest exits 0 on an empty suite). It now FAILS with a clear message. F10 — a non-integer RUN_ADDON_TESTS_TIMEOUT crashed the runner at import with a raw ValueError; it is now ignored with a stderr note and the default used. Tests driven via subprocess on synthetic addon trees, using GooCanvas (apt-provisioned, conda:None) as the satisfiable/unsatisfiable lever. --- .github/scripts/run_addon_tests.py | 123 ++++++++++++++++++++---- tests/test_run_addon_tests_paths.py | 139 ++++++++++++++++++++++++++-- 2 files changed, 236 insertions(+), 26 deletions(-) diff --git a/.github/scripts/run_addon_tests.py b/.github/scripts/run_addon_tests.py index 12016fa9a..945f1350c 100644 --- a/.github/scripts/run_addon_tests.py +++ b/.github/scripts/run_addon_tests.py @@ -15,13 +15,20 @@ clock. A test that hangs (e.g. a DB import that blocks on a platform) would otherwise hang the whole CI job indefinitely — neither plain unittest nor xmlrunner has a timeout. A module that exceeds the limit is killed and - reported as a FAILURE, so the job stays bounded and names the culprit. - -3. **Honest skip accounting.** unittest exits 0 when every test SKIPS, so a - wholly-skipped module reads as a pass. This runner FAILS such a module — - UNLESS the addon's declared system deps are unavailable on this platform - (e.g. goocanvas/osm-gps-map are not on conda-forge), in which case the skip - is expected and tolerated (the map lives in ``addon_system_deps.py``). + reported as a FAILURE, so the job stays bounded and names the culprit. The + worker runs in its own process group (POSIX) so the timeout reaps any + children it spawned, not just the worker — otherwise a hung grandchild + holding the stdout pipe defeats the timeout. + +3. **Honest skip accounting.** unittest exits 0 when every test SKIPS, and also + when a module collects ZERO tests — both read as a pass. This runner FAILS a + wholly-skipped or zero-test module, UNLESS the addon's declared system deps + are unavailable on this platform (e.g. goocanvas/osm-gps-map are not on + conda-forge), in which case the skip is expected and tolerated (the map + lives in ``addon_system_deps.py``). A module that fails to LOAD is excused as + a platform skip only when the failure is dependency-shaped (ImportError, or + gi's absent-typelib ValueError); a SyntaxError or a bug in the addon's own + import-time code is a real defect and always FAILS, on every platform. Usage:: @@ -39,7 +46,10 @@ from __future__ import annotations import argparse +import importlib import os +import re +import signal import subprocess import sys import unittest @@ -47,10 +57,26 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import addon_system_deps as deps # noqa: E402 -# Per-module wall clock. Generous enough for a legitimate DB-backed suite, small -# enough that a hung test is caught promptly instead of running to the job cap. -# Overridable via env for tuning/testing. -MODULE_TIMEOUT_S = int(os.environ.get("RUN_ADDON_TESTS_TIMEOUT", "300")) + +def _module_timeout() -> int: + """Per-module wall clock (seconds). Generous enough for a legitimate + DB-backed suite, small enough that a hung test is caught promptly instead of + running to the job cap. Overridable via ``RUN_ADDON_TESTS_TIMEOUT``; a + non-integer value is ignored with a note rather than crashing the runner.""" + raw = os.environ.get("RUN_ADDON_TESTS_TIMEOUT", "") + if raw: + try: + return int(raw) + except ValueError: + print( + f"run_addon_tests: ignoring non-integer " + f"RUN_ADDON_TESTS_TIMEOUT={raw!r}; using default 300", + file=sys.stderr, + ) + return 300 + + +MODULE_TIMEOUT_S = _module_timeout() # Markers the worker prints so the parent can read the outcome without needing # xmlrunner (820's env has only stdlib unittest). @@ -58,6 +84,20 @@ _LOADERROR = "__RESULT__ loaderror" +def _dep_shaped(exc: BaseException) -> bool: + """Whether a module load failure is a missing-dependency shape. + + Only these may be excused as an expected platform skip when the addon's + declared deps are unavailable: an ``ImportError`` (missing module), or the + ``ValueError`` ``gi.require_version`` raises for an absent typelib + (``Namespace X not available``). Everything else — SyntaxError, a bug in the + addon's own import-time code — is a real defect and must never be excused. + """ + if isinstance(exc, ImportError): + return True + return isinstance(exc, ValueError) and "not available" in str(exc) + + def _bootstrap_gi() -> None: """Pin the GI versions the Gramps GUI launcher pins, before tests import.""" try: @@ -96,11 +136,19 @@ def _run_worker(modname: str, root: str = ".") -> int: addon_dir = os.path.join(root, addon) if addon_dir not in sys.path: sys.path.append(addon_dir) + # Probe the import EXPLICITLY first. loadTestsFromName does not raise on a + # module-level ImportError/SyntaxError — since Python 3.5 it swallows the + # error into a _FailedTest placeholder that only errors when run, so the + # failure would reach the parent as an anonymous `broke=1` with its shape + # lost. Importing here surfaces the real exception so it can be classified + # (dependency-shaped vs a code bug). try: - suite = unittest.defaultTestLoader.loadTestsFromName(modname) + importlib.import_module(modname) except Exception as exc: # import-time failure - print(f"{_LOADERROR} {exc!r}", flush=True) + kind = "dep" if _dep_shaped(exc) else "other" + print(f"{_LOADERROR} kind={kind} {exc!r}", flush=True) return 0 + suite = unittest.defaultTestLoader.loadTestsFromName(modname) result = unittest.TextTestRunner(verbosity=2).run(suite) broke = len(result.failures) + len(result.errors) print( @@ -121,17 +169,40 @@ def _classify(modname: str, platform: str, root: str) -> tuple[bool, str]: satisfiable = deps.addon_satisfiable_on(os.path.join(root, addon), platform) proc = subprocess.Popen( - [sys.executable, os.path.abspath(__file__), "--worker", modname, "--root", root], + [ + sys.executable, + os.path.abspath(__file__), + "--worker", + modname, + "--root", + root, + ], stdout=subprocess.PIPE, stderr=None, # stream the test output straight to the CI log text=True, + # Own process group so a timeout can reap the worker AND any children it + # spawned. Without this, proc.kill() kills only the worker and the + # follow-up communicate() blocks until a grandchild that inherited the + # stdout pipe exits — a hung grandchild defeats the timeout entirely. + start_new_session=(os.name == "posix"), # setsid; raises on Windows ) try: # communicate() enforces the wall clock and reaps the process. stdout, _ = proc.communicate(timeout=MODULE_TIMEOUT_S) except subprocess.TimeoutExpired: - proc.kill() - proc.communicate() + if os.name == "posix": + try: + os.killpg(proc.pid, signal.SIGKILL) # group id == worker pid + except (ProcessLookupError, PermissionError): + proc.kill() + else: + proc.kill() + try: + # Bounded: a surviving grandchild holding the stdout pipe must not + # hang the run a second time. + proc.communicate(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() return True, f" FAIL {modname} — timed out after {MODULE_TIMEOUT_S}s (hung)" out_lines = stdout.splitlines() @@ -143,6 +214,16 @@ def _classify(modname: str, platform: str, root: str) -> tuple[bool, str]: ) if result_line.startswith(_LOADERROR): + kind_m = re.search(r"\bkind=(\w+)", result_line) + dep_shaped = bool(kind_m) and kind_m.group(1) == "dep" + if not dep_shaped: + # A non-dependency load failure (SyntaxError, a bug in the addon's + # import-time code) is a real defect on every platform — never + # excusable as a "deps unavailable here" skip. + return True, ( + f" FAIL {modname} — load error (not dependency-shaped; " + "a code bug, not a platform skip)" + ) if satisfiable: return True, f" FAIL {modname} — load error" return False, ( @@ -160,7 +241,15 @@ def _classify(modname: str, platform: str, root: str) -> tuple[bool, str]: if broke: return True, f" FAIL {modname} — {broke} failed/errored" - if ran > 0 and skipped == ran: + if ran == 0: + # The module loaded but collected no tests (a class not subclassing + # TestCase, a mistyped method name, a refactor that broke collection). + # unittest exits 0 on this, so it would read as green — fail it. + return True, ( + f" FAIL {modname} — module loaded but collected zero tests " + "(empty or misnamed test module reads as green)" + ) + if skipped == ran: if satisfiable: return True, ( f" FAIL {modname} — all {ran} tests skipped " diff --git a/tests/test_run_addon_tests_paths.py b/tests/test_run_addon_tests_paths.py index a9e5d7a3b..b57593d59 100644 --- a/tests/test_run_addon_tests_paths.py +++ b/tests/test_run_addon_tests_paths.py @@ -44,9 +44,7 @@ ADDONS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -RUN_ADDON_TESTS = os.path.join( - ADDONS_ROOT, ".github", "scripts", "run_addon_tests.py" -) +RUN_ADDON_TESTS = os.path.join(ADDONS_ROOT, ".github", "scripts", "run_addon_tests.py") @unittest.skipUnless( @@ -61,9 +59,7 @@ def setUp(self) -> None: # Stand-in for the repo-root shared Gramps-emulation env (PR 950): # a top-level `tests` package the addon tests may import. self._write("tests/__init__.py", "") - self._write( - "tests/gramps_test_env.py", 'SENTINEL = "repo-root-shared-env"\n' - ) + self._write("tests/gramps_test_env.py", 'SENTINEL = "repo-root-shared-env"\n') def tearDown(self) -> None: shutil.rmtree(self.root, ignore_errors=True) @@ -74,15 +70,22 @@ def _write(self, relpath: str, content: str) -> None: with open(path, "w", encoding="utf-8") as fp: fp.write(content) - def _run(self, modname: str) -> subprocess.CompletedProcess: + def _run( + self, + modname: str, + platform: str = "apt", + extra_env: dict | None = None, + ) -> subprocess.CompletedProcess: env = os.environ.copy() env["PYTHONPATH"] = "." # repo root on sys.path, as ci.yml sets it + if extra_env: + env.update(extra_env) return subprocess.run( [ sys.executable, RUN_ADDON_TESTS, "--platform", - "apt", + platform, "--root", self.root, modname, @@ -104,7 +107,7 @@ def test_nested_package_addon_loads(self) -> None: self._write( "SynthAddon/tests/sub/test_nested.py", "import unittest\n" - "from synthlib import VALUE\n" # per-addon import root + "from synthlib import VALUE\n" # per-addon import root "from tests.gramps_test_env import SENTINEL\n" # shared env not shadowed "\n" "class T(unittest.TestCase):\n" @@ -139,6 +142,124 @@ def test_flat_model_b_addon_loads(self) -> None: self.assertEqual(result.returncode, 0, "runner failed:\n%s" % out) self.assertIn("ok FlatAddon.tests.test_flat", result.stdout, out) + # ------------------------------------------------------------------ + # Outcome classification (load-failure taxonomy, zero-test, timeout, + # tolerant timeout env) — adversarial-review findings F1-F4/F10. + # GooCanvas is apt-provisioned but conda:None (addon_system_deps.GI_PACKAGES), + # so a `requires_gi=[("GooCanvas","2.0")]` addon is unsatisfiable on conda + # and satisfiable on apt — the lever these tests use. + # ------------------------------------------------------------------ + def _write_gi_addon(self, addon: str, test_body: str) -> None: + self._write( + f"{addon}/{addon.lower()}.gpr.py", + f'register(GRAMPLET, id="{addon}", requires_gi=[("GooCanvas", "2.0")])\n', + ) + self._write(f"{addon}/tests/__init__.py", "") + self._write(f"{addon}/tests/test_x.py", test_body) + + def test_syntax_error_fails_even_on_unsatisfiable_platform(self) -> None: + # A SyntaxError is a code bug, never a dependency shape — it must FAIL + # even where the addon's declared GI dep is unavailable (conda). + self._write_gi_addon("GiSyntax", "def broken(:\n pass\n") + result = self._run("GiSyntax.tests.test_x", platform="conda") + out = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, out) + self.assertIn("not dependency-shaped", out) + + def test_dep_import_error_skips_on_unsatisfiable_platform(self) -> None: + # A dependency-shaped load failure (ImportError) where the addon's deps + # are unavailable is an expected platform skip. + self._write_gi_addon( + "GiDepSkip", "import definitely_not_installed_xyz # noqa\n" + ) + result = self._run("GiDepSkip.tests.test_x", platform="conda") + out = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, out) + self.assertIn("skip", result.stdout) + + def test_dep_import_error_fails_on_satisfiable_platform(self) -> None: + # The same ImportError where the addon declares no unsatisfiable dep + # (satisfiable on apt) is a real failure, not a skip. + self._write( + "PlainAddon/plainaddon.gpr.py", 'register(GRAMPLET, id="PlainAddon")\n' + ) + self._write("PlainAddon/tests/__init__.py", "") + self._write( + "PlainAddon/tests/test_x.py", + "import definitely_not_installed_xyz # noqa\n", + ) + result = self._run("PlainAddon.tests.test_x", platform="apt") + out = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, out) + + def test_zero_collected_tests_fails(self) -> None: + # A module that loads but collects no tests reads as green under plain + # unittest; the runner must fail it. + self._write( + "EmptyAddon/emptyaddon.gpr.py", 'register(GRAMPLET, id="EmptyAddon")\n' + ) + self._write("EmptyAddon/tests/__init__.py", "") + self._write( + "EmptyAddon/tests/test_x.py", + "class NotATestCase:\n def test_nope(self):\n pass\n", + ) + result = self._run("EmptyAddon.tests.test_x", platform="apt") + out = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, out) + self.assertIn("zero tests", out) + + def test_non_integer_timeout_env_is_tolerated(self) -> None: + # A non-integer RUN_ADDON_TESTS_TIMEOUT must not crash the runner. + self._write("OkAddon/okaddon.gpr.py", 'register(GRAMPLET, id="OkAddon")\n') + self._write("OkAddon/tests/__init__.py", "") + self._write( + "OkAddon/tests/test_x.py", + "import unittest\n" + "class T(unittest.TestCase):\n" + " def test_ok(self):\n" + " self.assertTrue(True)\n", + ) + result = self._run( + "OkAddon.tests.test_x", + platform="apt", + extra_env={"RUN_ADDON_TESTS_TIMEOUT": "soon"}, + ) + out = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, out) + self.assertIn("ignoring non-integer", result.stderr) + + @unittest.skipUnless(os.name == "posix", "process-group kill is POSIX-only") + def test_timeout_reaps_grandchild_holding_stdout(self) -> None: + # A test that spawns a long-lived child inheriting the worker's stdout + # must not defeat the timeout: the whole process group is reaped and the + # follow-up communicate() is bounded, so the runner returns promptly. + import time + + self._write( + "HangAddon/hangaddon.gpr.py", 'register(GRAMPLET, id="HangAddon")\n' + ) + self._write("HangAddon/tests/__init__.py", "") + self._write( + "HangAddon/tests/test_x.py", + "import subprocess, sys, time, unittest\n" + "class T(unittest.TestCase):\n" + " def test_hang(self):\n" + # child inherits stdout (the worker's pipe); then the test blocks + " subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(120)'])\n" + " time.sleep(120)\n", + ) + start = time.monotonic() + result = self._run( + "HangAddon.tests.test_x", + platform="apt", + extra_env={"RUN_ADDON_TESTS_TIMEOUT": "3"}, + ) + elapsed = time.monotonic() - start + out = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, out) + self.assertIn("timed out", out) + self.assertLess(elapsed, 45, f"timeout not bounded (took {elapsed:.1f}s)") + if __name__ == "__main__": unittest.main() From 4a4a2b4896222fc744aec23d45fdb4bcca03c2e0 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 02:54:10 +0200 Subject: [PATCH 37/47] ci-scripts: tolerate a non-string requires_* entry instead of crashing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An author copying the requires_gi tuple shape into requires_mod (requires_mod=[("psycopg2", ">=2")]) writes a valid literal that both dep CLIs then choke on: the tuple reaches sorted() mixing tuple and str (TypeError), or a list entry is unhashable for a set .add() — replacing the --unmapped drift gate's readable diagnostic with a raw traceback, and violating addon_python_deps' own 'skip tolerantly, don't abort the batch' contract. Guard on isinstance(entry, str) in both scanners (addon_python_deps._declared_ mods and addon_system_deps._scan / addon_requirements): a non-string entry is skipped with a stderr note, like a non-literal value already is. Tests in both modules feed a tuple entry and assert the scan yields only the string names and does not raise; the real tree's --unmapped stays clean. --- .github/scripts/addon_python_deps.py | 15 +++++++++++++-- .github/scripts/addon_system_deps.py | 19 +++++++++++++++---- tests/test_addon_python_deps.py | 16 ++++++++++++++++ tests/test_addon_system_deps.py | 17 +++++++++++++++++ 4 files changed, 61 insertions(+), 6 deletions(-) diff --git a/.github/scripts/addon_python_deps.py b/.github/scripts/addon_python_deps.py index 5c2aa06cb..cb9ce4d35 100644 --- a/.github/scripts/addon_python_deps.py +++ b/.github/scripts/addon_python_deps.py @@ -171,8 +171,19 @@ def _declared_mods(text: str, path: str) -> list[str]: ) continue for mod in value: - if mod: - out.append(mod) + if not mod: + continue + if not isinstance(mod, str): + # e.g. requires_mod=[("psycopg2", ">=2")] — an author copying + # the requires_gi tuple shape. A non-str entry would crash the + # later sorted()/mapping; skip it tolerantly with a note. + print( + f"addon_python_deps: skipping non-string requires_mod entry " + f"in {path}: {mod!r}", + file=sys.stderr, + ) + continue + out.append(mod) return out diff --git a/.github/scripts/addon_system_deps.py b/.github/scripts/addon_system_deps.py index b66cac2dd..2ff13d02d 100644 --- a/.github/scripts/addon_system_deps.py +++ b/.github/scripts/addon_system_deps.py @@ -151,8 +151,19 @@ def _scan(root: str, pattern: re.Pattern, first_of_tuple: bool) -> set[str]: for entry in _literal(match.group(1)): if first_of_tuple and isinstance(entry, (tuple, list)): entry = entry[0] if entry else None - if entry: - found.add(entry) + if not entry: + continue + if not isinstance(entry, str): + # A non-str entry (e.g. requires_mod=[("psycopg2", ">=2")], + # or a nested list) would be unhashable for .add() or crash a + # later sorted()/lookup — skip it tolerantly with a note. + print( + f"addon_system_deps: skipping non-string requires_* " + f"entry in {path}: {entry!r}", + file=sys.stderr, + ) + continue + found.add(entry) return found @@ -180,11 +191,11 @@ def addon_requirements(addon_dir: str) -> tuple[set[str], set[str]]: for match in _GI_RE.finditer(text): for entry in _literal(match.group(1)): ns = entry[0] if isinstance(entry, (tuple, list)) else entry - if ns: + if isinstance(ns, str) and ns: gi.add(ns) for match in _EXE_RE.finditer(text): for entry in _literal(match.group(1)): - if entry: + if isinstance(entry, str) and entry: exe.add(entry) return gi, exe diff --git a/tests/test_addon_python_deps.py b/tests/test_addon_python_deps.py index 16245880b..b3f4c4f28 100644 --- a/tests/test_addon_python_deps.py +++ b/tests/test_addon_python_deps.py @@ -328,5 +328,21 @@ def test_source_built_never_installed_stays_advisory(self) -> None: self.assertIn("~ pygraphviz", out) +class NonStringRequiresMod(unittest.TestCase): + """A tuple-shaped requires_mod entry must be skipped, not crash the CLI.""" + + def test_tuple_entry_skipped_not_fatal(self) -> None: + import tempfile + + with tempfile.TemporaryDirectory() as root: + addon = os.path.join(root, "Addon") + os.makedirs(addon) + with open(os.path.join(addon, "x.gpr.py"), "w", encoding="utf-8") as fh: + fh.write('requires_mod = [("psycopg2", ">=2"), "svgwrite"]\n') + # Would previously TypeError in sorted() mixing tuple and str. + self.assertEqual(deps.declared_mods(root), {"svgwrite"}) + self.assertEqual(deps.install_list(root), ["svgwrite"]) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_addon_system_deps.py b/tests/test_addon_system_deps.py index 719151553..7e7f6c5ee 100644 --- a/tests/test_addon_system_deps.py +++ b/tests/test_addon_system_deps.py @@ -307,5 +307,22 @@ def test_toolchain_is_installed(self): ) +class NonStringRequiresEntry(unittest.TestCase): + """A tuple/list requires_* entry must be skipped, not crash the scanner.""" + + def test_tuple_requires_mod_skipped_not_fatal(self): + import tempfile + + with tempfile.TemporaryDirectory() as root: + addon = os.path.join(root, "Addon") + os.makedirs(addon) + with open(os.path.join(addon, "x.gpr.py"), "w", encoding="utf-8") as fh: + fh.write('requires_mod = [("psycopg2", ">=2"), "svgwrite"]\n') + # scan_modules / the --unmapped CLI would previously TypeError on the + # tuple entry (unhashable for a set, or sorted() mixing types). + self.assertEqual(deps.scan_modules(root), {"svgwrite"}) + self.assertEqual(deps.unmapped(root), (set(), set(), set())) + + if __name__ == "__main__": unittest.main() From c5fe06a4e86c4e8d94dddc1629105c7dcaaf8634 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 03:10:58 +0200 Subject: [PATCH 38/47] ci: make is_active ast-based, per-register, and comment-proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit active_addons.sh's is_active() grepped each .gpr.py for include_in_listing: an =True anywhere, or the absence of the flag, meant active. Two latent bugs the adversarial review flagged: (1) it is file-granular, so an addon with one register(include_in_listing=False) plus a sibling register() that omits the flag was read INACTIVE, though make.py builds/releases it (it reads include_in_listing per registration, default True) — CI would skip lint, structure, compile and tests for an addon that ships; (2) grep has no comment awareness, so a flag mentioned only in a # comment flips the result. Move the rule into a new pure-stdlib active_addons.py that parses each gpr with ast and classifies per register() call: active iff any register omits the flag or sets it to anything but the literal False; an unparsable or register-less gpr is tolerantly active (never silently drop an addon). active_addons.sh now calls it once at source time (python3, python fallback for the conda Windows lane) and is_active() is a membership test — one interpreter per sourcing step, not one per addon. ci.yml is untouched, so the dedup invariants hold. Verified behaviour-identical to the old grep across all 146 current addon dirs; test_active_addons pins the semantics and embeds that whole-tree oracle, so the first gpr to exercise the difference trips the test for human review rather than silently changing CI's gated set. --- .github/scripts/active_addons.py | 127 +++++++++++++++++++ .github/scripts/active_addons.sh | 36 ++++-- tests/test_active_addons.py | 201 +++++++++++++++++++++++++++++++ 3 files changed, 354 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/active_addons.py create mode 100644 tests/test_active_addons.py diff --git a/.github/scripts/active_addons.py b/.github/scripts/active_addons.py new file mode 100644 index 000000000..757862c0d --- /dev/null +++ b/.github/scripts/active_addons.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Single source of truth for the *active addon* rule (include_in_listing). + +An addon is **active** — built and released by ``make.py``, so CI gates on it — +when at least one ``register()`` call in its ``.gpr.py`` file(s) would be listed. +``make.py`` reads each registration's ``include_in_listing`` with a default of +``True`` (``p.get("include_in_listing", True)``), so a plugin is listed unless +its registration explicitly sets ``include_in_listing=False``. An addon whose +EVERY registration sets it to ``False`` is inactive; CI skips it in lint, the +structure check, compile, and the test runners. + +This module is the parser behind ``active_addons.sh``'s ``is_active()``; the +shell helper calls ``--list`` once and greps the result. The rule here is: + +* **per-register**, matching ``make.py`` — one ``register(..., include_in_listing + =False)`` does not make the addon inactive if a sibling ``register()`` omits + the flag (defaults True) or sets it truthy. (The previous grep helper was + file-granular: any ``include_in_listing=`` with no ``=True`` in the whole file + read as inactive, disagreeing with make.py on that mixed shape.) +* **comment-proof** — a flag mentioned only in a ``#`` comment is ignored, + because the source is parsed with ``ast`` rather than grepped. +* **tolerant** — a ``.gpr.py`` that fails to parse, or a dir whose gpr has no + ``register()`` call at all, counts as ACTIVE. The default never silently drops + an addon from CI; the worst case is gating one that make.py would not build. + +Pure stdlib, no gramps import, never executes the ``.gpr.py`` (ast.parse only). + +CLI:: + + active_addons.py --list [ROOT] # active addon dir names, one per line + active_addons.py --check DIR # exit 0 if DIR is active, 1 if not +""" + +from __future__ import annotations + +import argparse +import ast +import glob +import os +import sys + + +def _register_calls(tree: ast.AST): + """Yield every ``register(...)`` call node in a parsed .gpr.py.""" + for node in ast.walk(tree): + if isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name) and func.id == "register": + yield node + + +def _register_is_listed(call: ast.Call) -> bool: + """Whether one ``register()`` call would be listed by make.py. + + Listed unless it carries ``include_in_listing=`` set to the literal + ``False``. Omitted → default True → listed. A non-literal value (a variable + / expression we cannot evaluate statically) is treated as listed, so the + tolerant default never hides an addon. + """ + for kw in call.keywords: + if kw.arg == "include_in_listing": + value = kw.value + if isinstance(value, ast.Constant) and value.value is False: + return False + return True + return True # omitted → make.py default True + + +def addon_is_active(addon_dir: str) -> bool: + """Whether an addon directory is active (built/released → CI gates on it).""" + gprs = sorted(glob.glob(os.path.join(addon_dir, "*.gpr.py"))) + if not gprs: + return False # not an addon (no descriptor) — nothing to gate + saw_register = False + for gpr in gprs: + try: + with open(gpr, encoding="utf-8") as fh: + tree = ast.parse(fh.read(), filename=gpr) + except (OSError, SyntaxError, ValueError): + return True # cannot analyse → tolerant active + for call in _register_calls(tree): + saw_register = True + if _register_is_listed(call): + return True + # gpr(s) exist but declared no register() at all → tolerant active; + # otherwise every register set include_in_listing=False → inactive. + return not saw_register + + +def active_addons(root: str) -> list[str]: + """Sorted names of the active addon directories directly under *root*.""" + names: list[str] = [] + for gpr in sorted(glob.glob(os.path.join(root, "*", "*.gpr.py"))): + addon_dir = os.path.dirname(gpr) + name = os.path.basename(addon_dir) + if name not in names and addon_is_active(addon_dir): + names.append(name) + return sorted(names) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "--list", + nargs="?", + const=".", + metavar="ROOT", + help="print active addon dir names under ROOT (default: cwd)", + ) + group.add_argument( + "--check", + metavar="DIR", + help="exit 0 if the addon DIR is active, 1 if not", + ) + args = parser.parse_args(argv) + + if args.check is not None: + return 0 if addon_is_active(args.check) else 1 + print("\n".join(active_addons(args.list))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/active_addons.sh b/.github/scripts/active_addons.sh index f68c8f147..c1c2cca30 100644 --- a/.github/scripts/active_addons.sh +++ b/.github/scripts/active_addons.sh @@ -1,21 +1,37 @@ # Single source of truth for the is_active() addon filter, sourced by every # ci.yml job step that gates on include_in_listing. # -# An addon is "active" (built and released by make.py, so CI gates on it) unless -# EVERY register() in its .gpr.py sets include_in_listing=False. Those inactive -# addons are skipped by lint, the structure check, compile, and the unit / +# An addon is "active" (built and released by make.py, so CI gates on it) when +# at least one register() in its .gpr.py would be listed — include_in_listing +# omitted (make.py default True) or set to anything but the literal False. Only +# an addon whose EVERY register() sets include_in_listing=False is inactive. +# Those are skipped by lint, the structure check, compile, and the unit / # integration test runners. Previously each of those ~6 job steps inlined an # identical copy of this function; this file is the one place it now lives, so a # change to the active-addon rule is a one-site edit (per PR #820; Gary Griffin). # +# The rule itself lives in active_addons.py (ast-based, per-register, +# comment-proof — a grep cannot tell one register's flag from another's, nor a +# real flag from one in a comment). This helper computes the active set ONCE at +# source time and is_active() is a membership test, so a sourcing step starts +# one interpreter, not one per addon. python3, falling back to python for the +# conda Windows lane which ships only `python`. A helper failure aborts the +# sourcing step (its shell runs under set -e) rather than silently marking +# every addon inactive. +# # Source it from a `shell: bash` step (the function uses `local`, a bashism): # source .github/scripts/active_addons.sh +_AA_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if command -v python3 >/dev/null 2>&1; then _AA_PY=python3; else _AA_PY=python; fi +_ACTIVE_ADDONS="$("$_AA_PY" "$_AA_DIR/active_addons.py" --list .)" || { + echo "::error::active_addons.py failed — cannot determine active addons" >&2 + return 1 2>/dev/null || exit 1 +} + is_active() { - local addon="$1" g - for g in "$addon"/*.gpr.py; do - [ -f "$g" ] || continue - grep -qE 'include_in_listing[[:space:]]*=[[:space:]]*True' "$g" && return 0 - grep -qE 'include_in_listing[[:space:]]*=' "$g" || return 0 - done - return 1 + # Accept a dir name or path; compare on the basename (strip trailing slash + # then any leading path), matching the names active_addons.py --list prints. + local name="${1%/}" + name="${name##*/}" + printf '%s\n' "$_ACTIVE_ADDONS" | grep -qxF "$name" } diff --git a/tests/test_active_addons.py b/tests/test_active_addons.py new file mode 100644 index 000000000..62a2db705 --- /dev/null +++ b/tests/test_active_addons.py @@ -0,0 +1,201 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Eduard Ralph +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +"""The active-addon rule (include_in_listing) must be per-register and correct. + +``active_addons.py`` replaced a file-granular grep with an ast parse. These +tests pin the semantics that make it *more correct* than the grep (per-register, +comment-proof, tolerant defaults) AND assert it is behaviour-identical to the +old grep rule over the real addon tree today — so the change ships proven +equivalent, and the first gpr that exercises the difference trips the oracle +test (a human then confirms the intent and updates the oracle), rather than +silently changing which addons CI gates on. + +Pure stdlib; the bash integration test is skipped where bash is absent. +""" + +from __future__ import annotations + +import glob +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_HERE) +_SCRIPTS = os.path.join(_REPO_ROOT, ".github", "scripts") +_HELPER_SH = os.path.join(_SCRIPTS, "active_addons.sh") + +sys.path.insert(0, _SCRIPTS) +import active_addons as aa # noqa: E402 + + +class ActiveAddonSemantics(unittest.TestCase): + """Per-register, comment-proof, tolerant classification.""" + + def setUp(self) -> None: + self.root = tempfile.mkdtemp(prefix="active_addons_") + + def tearDown(self) -> None: + shutil.rmtree(self.root, ignore_errors=True) + + def _addon(self, name: str, gpr_body: str) -> str: + d = os.path.join(self.root, name) + os.makedirs(d, exist_ok=True) + with open( + os.path.join(d, f"{name.lower()}.gpr.py"), "w", encoding="utf-8" + ) as fh: + fh.write(gpr_body) + return d + + def test_sibling_register_without_flag_is_active(self) -> None: + # One register sets False, a sibling omits it (make.py default True) → + # the addon IS built/released → active. (The old grep read this file as + # inactive — the correctness bug this change fixes.) + d = self._addon( + "Mixed", + 'register(TOOL, id="a", include_in_listing=False)\n' + 'register(GRAMPLET, id="b")\n', + ) + self.assertTrue(aa.addon_is_active(d)) + + def test_all_registers_false_is_inactive(self) -> None: + d = self._addon( + "AllFalse", + 'register(TOOL, id="a", include_in_listing=False)\n' + 'register(GRAMPLET, id="b", include_in_listing=False)\n', + ) + self.assertFalse(aa.addon_is_active(d)) + + def test_explicit_true_is_active(self) -> None: + d = self._addon("T", 'register(TOOL, id="a", include_in_listing=True)\n') + self.assertTrue(aa.addon_is_active(d)) + + def test_flag_only_in_comment_is_ignored(self) -> None: + # A False in a comment must not flip an otherwise-listed addon inactive. + d = self._addon( + "Commented", + "# include_in_listing=False (historical note)\n" + 'register(GRAMPLET, id="a")\n', + ) + self.assertTrue(aa.addon_is_active(d)) + + def test_no_register_is_active(self) -> None: + d = self._addon("NoReg", "PLUGINS = [] # descriptor with no register()\n") + self.assertTrue(aa.addon_is_active(d)) + + def test_unparsable_gpr_is_active(self) -> None: + d = self._addon("Broken", "register(TOOL, id= # truncated\n") + self.assertTrue(aa.addon_is_active(d)) + + def test_non_literal_flag_is_active(self) -> None: + # A value we cannot evaluate statically must not be assumed False. + d = self._addon( + "Dynamic", + 'LISTED = True\nregister(GRAMPLET, id="a", include_in_listing=LISTED)\n', + ) + self.assertTrue(aa.addon_is_active(d)) + + def test_dir_without_gpr_not_listed(self) -> None: + d = os.path.join(self.root, "NotAnAddon") + os.makedirs(d) + self.assertFalse(aa.addon_is_active(d)) + + +class BehaviourIdentityWithOldGrep(unittest.TestCase): + """The ast rule must match the old file-granular grep over the real tree.""" + + @staticmethod + def _old_grep_active(addon_dir: str) -> bool: + # The exact rule active_addons.sh used to inline: per FILE, an + # include_in_listing=True (anywhere) or the ABSENCE of any + # include_in_listing= makes the addon active; else inactive. + gprs = sorted(glob.glob(os.path.join(addon_dir, "*.gpr.py"))) + for gpr in gprs: + with open(gpr, encoding="utf-8") as fh: + text = fh.read() + if re.search(r"include_in_listing[ \t]*=[ \t]*True", text): + return True + if not re.search(r"include_in_listing[ \t]*=", text): + return True + return False + + def test_ast_matches_grep_over_whole_tree(self) -> None: + dirs = sorted( + { + os.path.dirname(g) + for g in glob.glob(os.path.join(_REPO_ROOT, "*", "*.gpr.py")) + } + ) + self.assertGreater(len(dirs), 100, "addon tree not found from test location") + diffs = [ + os.path.basename(d) + for d in dirs + if self._old_grep_active(d) != aa.addon_is_active(d) + ] + self.assertEqual( + diffs, + [], + "active_addons.py disagrees with the old grep rule on: " + f"{diffs}. This is the per-register/comment-proof semantic change " + "biting a real addon for the first time — confirm the new (correct) " + "classification is intended, then update this oracle to match.", + ) + + +@unittest.skipUnless(shutil.which("bash"), "bash not available") +class ShellHelperIntegration(unittest.TestCase): + """active_addons.sh's is_active() must agree with active_addons.py.""" + + def test_sourced_is_active_matches_check(self) -> None: + root = tempfile.mkdtemp(prefix="active_addons_sh_") + try: + for name, body in ( + ("Active", 'register(GRAMPLET, id="a")\n'), + ("Inactive", 'register(TOOL, id="a", include_in_listing=False)\n'), + ): + d = os.path.join(root, name) + os.makedirs(d) + with open(os.path.join(d, f"{name.lower()}.gpr.py"), "w") as fh: + fh.write(body) + script = ( + f"source {_HELPER_SH}\n" + "is_active Active && echo A:active || echo A:inactive\n" + "is_active Inactive && echo I:active || echo I:inactive\n" + ) + result = subprocess.run( + ["bash", "-c", script], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + self.assertIn("A:active", result.stdout, result.stderr) + self.assertIn("I:inactive", result.stdout, result.stderr) + finally: + shutil.rmtree(root, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() From dd7e73c1e47b3820bf71a69bfe202a13f462c6bd Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 03:11:57 +0200 Subject: [PATCH 39/47] ci: document why DynamicWeb and Sqlite tests stay out of the unit lanes Two silent test exclusions the review flagged as unexplained: - The unit-test glob is */tests/**/test_*.py, which never matches DynamicWeb/ test_dynamicweb.py (a root-level module). That is deliberate: it is a nose-era dev harness (nose is gone on py3.12, it asserts a USER_PLUGINS install and drives Gramps.py from a source checkout) that would hard-fail if run, and its CI-shaped sibling DynamicWeb/tests/test_dwr_tree_names.py IS matched. State it at the glob so it does not read as an accidental gap. - Sqlite/tests/test_sqlite.py is hard-excluded in both unit lanes with no reason given. Record it: the test needs GRAMPS_RESOURCES + example/gramps/ example.gramps from a gramps SOURCE checkout (the wheels ship only gramps/ + share) and writes fixed /tmp paths; un-excluding is a follow-up. Comment-only; no behaviour change. --- .github/workflows/ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a440b3775..1b4608cca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -285,6 +285,14 @@ jobs: source .github/scripts/active_addons.sh modules="" shopt -s globstar # also match nested-package tests//test_*.py + # Glob is scoped to */tests/**/ ON PURPOSE. The only root-level test + # module in the tree, DynamicWeb/test_dynamicweb.py, is a nose-era dev + # harness (nose is gone on py3.12; it asserts a USER_PLUGINS install and + # drives Gramps.py from a source checkout) — unrunnable here, and it + # declares no requires_gi, so it would hard-fail if matched. Its + # CI-shaped sibling DynamicWeb/tests/test_dwr_tree_names.py IS matched, + # so DynamicWeb keeps coverage. The runner can load a root-level module, + # so this is a policy choice, not a limitation. for f in */tests/**/test_*.py; do [ -f "$f" ] || continue addon="${f%%/*}" @@ -294,6 +302,12 @@ jobs: test_windows_*) continue ;; esac case "$f" in + # Sqlite/tests/test_sqlite.py predates this pipeline: it needs + # GRAMPS_RESOURCES + example/gramps/example.gramps from a gramps + # SOURCE checkout (the pip/conda wheels ship only gramps/ + share, + # not example/) and writes fixed /tmp paths with no cleanup. Un- + # excluding is a follow-up: provide example.gramps in the lane and + # port the test to tempfile. Sqlite/tests/test_sqlite.py) continue ;; esac mod="${f%.py}" @@ -430,6 +444,8 @@ jobs: run: | source .github/scripts/active_addons.sh modules="" + # Glob scoped to */tests/**/ (see unit-test-linux for why DynamicWeb's + # root-level test module is deliberately not matched). shopt -s globstar # also match nested-package tests//test_*.py for f in */tests/**/test_*.py; do [ -f "$f" ] || continue @@ -440,6 +456,8 @@ jobs: test_linux_*) continue ;; esac case "$f" in + # excluded — see unit-test-linux for the rationale (needs a gramps + # source checkout's example.gramps; a follow-up). Sqlite/tests/test_sqlite.py) continue ;; esac mod="${f%.py}" From 8b28323b7464d94ad49a9a87ffadf5dc607378c7 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 03:16:55 +0200 Subject: [PATCH 40/47] ci: correct the Windows-lane gramps-provenance story (PyPI, not conda-forge) Comments in environment.yml and ci.yml claimed the conda Windows lane gets gramps from conda-forge and that the pin 'self-heals' when 6.1 reaches conda-forge. Both are wrong: gramps is installed from PyPI via environment.yml's pip: block, and the pin 'gramps>=6.0,<6.1' can never resolve 6.1 regardless of conda-forge. On a gramps61+ branch the lane therefore validates addons against 6.0.x until BOTH 6.1 is published on PyPI AND the pin is bumped by hand. Correct the comments to name PyPI as the source and the two conditions; the ci.yml 'Report gramps-vs-branch series' step already warns on the divergence (logic unchanged). CI-MAINTAINER.md's 'no workflow edits required' for a new branch now notes the one file that IS a manual per-branch bump: environment.yml. Docs/comments only; no behaviour change. --- .github/CI-MAINTAINER.md | 11 ++++++++++- .github/environment.yml | 13 +++++++------ .github/workflows/ci.yml | 27 +++++++++++++-------------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/.github/CI-MAINTAINER.md b/.github/CI-MAINTAINER.md index 69324a203..252ad32aa 100644 --- a/.github/CI-MAINTAINER.md +++ b/.github/CI-MAINTAINER.md @@ -68,11 +68,20 @@ git branch maintenance/gramps62 maintenance/gramps61 git push origin maintenance/gramps62 ``` -No workflow edits required. The workflows derive everything from +No *workflow* edits required — the workflows derive everything from `github.ref_name`. The first push fires the same race described above — re-run the failed CI jobs once `Build Docker Images` finishes. +One file is NOT auto-derived and must be bumped by hand on the new +branch: **`.github/environment.yml`**'s gramps pin (`gramps>=6.0,<6.1`). +The conda Windows lane installs gramps from PyPI through that pin, so +until you bump it to the new series (e.g. `>=6.2,<6.3`) — and the +series is actually published on PyPI — the Windows lane keeps +validating addons against the old series. `ci.yml`'s "Report +gramps-vs-branch series" step prints a loud `::warning::` while the two +diverge; it does not fail. + The setup job's regex (`gramps[0-9][0-9]`) requires a two-digit suffix. When Gramps 10.0 opens this regex needs updating in two places (`ci.yml` setup job and `docker-build.yml` params step). diff --git a/.github/environment.yml b/.github/environment.yml index f1c7c9ae7..0d73f7cbd 100644 --- a/.github/environment.yml +++ b/.github/environment.yml @@ -11,12 +11,13 @@ dependencies: # requires_mod — single source of truth. Keep only the stable base # here (Gramps + orjson for plugin registration). # - # conda-forge has no gramps 6.1 yet, so on a maintenance/gramps61 (or later) - # branch this resolves to 6.0.x — i.e. the Windows lane validates addons - # against conda-forge's newest in-range gramps, not the branch's exact series - # (the Linux CI image git-builds the exact series; conda-Windows cannot — see - # ci.yml's "Report gramps-vs-branch series" step). When 6.1 reaches - # conda-forge the pin picks it up automatically. + # NOTE: gramps is installed from PyPI via the `pip:` block below (conda-forge + # is NOT the source), and the pin is a deliberate per-series bound. `<6.1` + # can never resolve 6.1, so on a maintenance/gramps61 (or later) branch this + # file MUST be bumped (e.g. to ">=6.1,<6.2") — the workflows derive the series + # automatically, but this pin does not and will otherwise validate addons + # against 6.0.x forever (see .github/CI-MAINTAINER.md and ci.yml's "Report + # gramps-vs-branch series" step, which warns loudly when the two diverge). - pip: - "gramps>=6.0,<6.1" - orjson diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b4608cca..7a6bad817 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -356,20 +356,19 @@ jobs: python -c "import gramps, gi; print('deps OK')" - name: Report gramps-vs-branch series (Windows lane caveat) - # The Linux lane git-builds the branch's exact gramps in its CI image - # (.github/docker/gramps-ci/Dockerfile, PyPI-first/git-fallback). The - # conda-forge Windows lane CANNOT match that: conda-forge has no gramps - # 6.1 yet, and gramps' own Windows build targets MSYS2 UCRT64, not conda - # — building 6.1 from git here fails in gramps' build hook (build_intl's - # `msgfmt --xml` cannot locate the shared-mime-info/appstream ITS rules, - # which are absent in the conda env). So on a maintenance/gramps61 (or - # later) branch this lane validates addons against conda-forge's newest - # in-range gramps (6.0.x today) rather than the branch's series. This - # step surfaces that honestly; it does NOT fail. Addon tests that depend - # on series-exact gramps behaviour skip themselves on Windows (e.g. - # TMGimporter's real-DB import tests) and run on the Linux lane instead. - # When gramps 6.1 reaches conda-forge, environment.yml's pin picks it up - # and this caveat disappears on its own. + # The Linux lane runs the branch's exact gramps in its CI image + # (.github/docker/gramps-ci/Dockerfile, PyPI-first / git-tip fallback). + # The conda Windows lane installs gramps from PyPI via environment.yml's + # `pip:` block (NOT from conda-forge), pinned `gramps>=6.0,<6.1`. So on a + # maintenance/gramps61 (or later) branch it validates addons against + # 6.0.x, not the branch's series — for two independent reasons: the pin's + # `<6.1` excludes 6.1, and 6.1 is not published on PyPI yet anyway. This + # step surfaces the mismatch honestly; it does NOT fail. Addon tests that + # depend on series-exact gramps behaviour skip themselves on Windows + # (e.g. TMGimporter's real-DB import tests) and run on the Linux lane + # instead. The caveat clears only when BOTH hold: gramps 6.1 is on PyPI, + # AND environment.yml's pin has been bumped on that branch (it does not + # self-heal — see .github/CI-MAINTAINER.md). run: | suffix="${{ needs.setup.outputs.branch_suffix }}" # e.g. gramps61 digits="${suffix#gramps}" # e.g. 61 From 92529bbbb158c59d75f4a6aa75a00ed7a53d6f5d Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 03:17:47 +0200 Subject: [PATCH 41/47] ci: lowercase the GHCR image ref so mixed-case fork owners can pull MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The container image ref embedded ${{ github.repository }} verbatim. GHCR references must be lowercase and docker-build's metadata-action lowercases what it pushes, so a fork whose owner has uppercase letters (e.g. EdUralph) pushed a lowercase image but ci.yml tried to pull the mixed-case ref — an invalid reference that fails every container job at init. Lowercase it with ${GITHUB_REPOSITORY,,} in the setup step (no new step, so the is_active dedup pins are untouched). Upstream (all-lowercase owner) is unaffected. --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a6bad817..b00efc63b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,14 @@ jobs: *) echo "::error::unexpected ref '$ref' (suffix '$suffix')"; exit 1 ;; esac echo "branch_suffix=$suffix" >> "$GITHUB_OUTPUT" - echo "ci_image=ghcr.io/${{ github.repository }}/gramps-ci:$suffix" >> "$GITHUB_OUTPUT" + # Lowercase the owner/name: GHCR refs must be lowercase, and + # docker-build's metadata-action lowercases what it PUSHES, so the + # pull side must match — otherwise a fork whose owner has uppercase + # letters pulls an invalid (mixed-case) reference and every container + # job fails at init. ${VAR,,} is a bash lowercasing expansion; there + # is no expression-level equivalent in Actions. + repo_lc="${GITHUB_REPOSITORY,,}" + echo "ci_image=ghcr.io/${repo_lc}/gramps-ci:$suffix" >> "$GITHUB_OUTPUT" # ----------------------------------------------------------------- # Lint (ci container) From 1127965a7e21c33f68fff21551db1f3dac647f5c Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 03:19:59 +0200 Subject: [PATCH 42/47] =?UTF-8?q?ci:=20docker-build=20lifecycle=20hardenin?= =?UTF-8?q?g=20=E2=80=94=20PR=20build-without-push,=20no-cache,=20weekly?= =?UTF-8?q?=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The image lifecycle was the review's weakest subsystem. Three additions to docker-build.yml, none changing the normal push path: - pull_request trigger scoped to .github/docker/** and this workflow: builds the image to validate a Dockerfile change before merge, WITHOUT pushing (the login step is skipped and push: is false via github.event_name), so a fork PR never touches the packages:write token. params derives the series from github.base_ref on a PR. - workflow_dispatch 'no-cache' boolean input wired to build-push-action, for a from-scratch rebuild that picks up base-image/apt and new gramps patch releases the buildx cache key would otherwise skip. - weekly schedule + a weekly-rebuild fan-out job that dispatches a no-cache rebuild per maintenance branch (permissions: actions: write; existing job guarded off schedule). Documented as inert until the workflows reach the default branch, since scheduled runs fire only from there. CI-MAINTAINER.md gains an 'Image lifecycle' section covering all of the above plus the known one-push image lag (ci.yml pulls the moving tag while the rebuild runs, so an image-affecting push tests against the previous image — re-run the affected jobs after the build; the Dockerfile ruff pin lands one push late for the same reason). --- .github/CI-MAINTAINER.md | 38 ++++++++++++++++++- .github/workflows/docker-build.yml | 60 +++++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/.github/CI-MAINTAINER.md b/.github/CI-MAINTAINER.md index 252ad32aa..01be87151 100644 --- a/.github/CI-MAINTAINER.md +++ b/.github/CI-MAINTAINER.md @@ -14,8 +14,9 @@ an unreleased branch see [CONTRIBUTING.md](../CONTRIBUTING.md#work-towards-a-mer 1. [One-time setup when the PR first merges](#one-time-setup-when-the-pr-first-merges) 2. [Creating a new maintenance branch](#creating-a-new-maintenance-branch) 3. [When a Gramps minor release lands on PyPI](#when-a-gramps-minor-release-lands-on-pypi) -4. [Diagnostic log markers](#diagnostic-log-markers) -5. [Optional future-proofing knobs](#optional-future-proofing-knobs) +4. [Image lifecycle](#image-lifecycle) +5. [Diagnostic log markers](#diagnostic-log-markers) +6. [Optional future-proofing knobs](#optional-future-proofing-knobs) ## One-time setup when the PR first merges @@ -109,6 +110,39 @@ waiting for the next push: The `::notice::` line in the build log confirms the switch. +## Image lifecycle + +`docker-build.yml` owns the `gramps-ci:` image. How it rebuilds: + +- **On push** to `maintenance/gramps**` — builds and pushes the image + for that branch. This is the normal path. +- **On pull_request** touching `.github/docker/**` or + `docker-build.yml` — builds the image **without pushing** (the login + step is skipped and `push:` is false), so a Dockerfile change is + validated before merge. A fork PR never touches the push credential. + A green PR build proves "the image still builds"; it does **not** + update the tag CI pulls. +- **workflow_dispatch** with **`no-cache: true`** — a from-scratch + rebuild that ignores the buildx layer cache. Use this to pick up a + new base-image (apt security) update or a newly published gramps + patch release that the cache key would otherwise skip. +- **Weekly `schedule`** — the `weekly-rebuild` job fans out a + `no-cache` dispatch to every `maintenance/grampsNN` branch. + **Caveat:** GitHub runs scheduled workflows only from the repo's + **default branch**, so this is inert until these workflows exist + there (upstream's default is `maintenance/gramps61`). That is by + design — remove/adjust it if the default branch never carries the + pipeline. + +**Known one-push image lag (not fixed, be aware):** `ci.yml` and +`docker-build.yml` both fire on the same push, and `ci.yml` pulls the +moving `gramps-ci:` tag while the rebuild is still running. So a +push that changes the image (e.g. a Dockerfile edit) has its CI run +against the **previous** image. After the `Build Docker Images` run for +that push finishes, **re-run the affected CI jobs** to test against the +new image. For the same reason, the `ruff` version pinned in the +Dockerfile takes effect one push late. + ## Diagnostic log markers When investigating a CI failure, the install-step output in diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6ebd19ed5..4534a9d9f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -7,7 +7,27 @@ on: # in exchange for guaranteeing gramps-ci: exists on the # first push to any newly-created maintenance branch. branches: [maintenance/gramps**] + pull_request: + # Build-only validation (no push — see the build step's `push:` and the + # login step's `if:`) when a PR touches the image definition, so a broken + # Dockerfile is caught before it merges instead of on the next push. + branches: [maintenance/gramps**] + paths: + - ".github/docker/**" + - ".github/workflows/docker-build.yml" + schedule: + # Weekly no-cache refresh so base-image (apt security) updates and new + # gramps patch releases enter the image even when nothing changes the + # buildx cache key. Fans out to every maintenance branch (see the + # weekly-rebuild job). NOTE: scheduled runs execute only from the repo's + # DEFAULT branch — inert until these workflows exist there. + - cron: "23 4 * * 1" workflow_dispatch: + inputs: + no-cache: + description: "Rebuild without the buildx layer cache (fresh base image + gramps)" + type: boolean + default: false env: REGISTRY: ghcr.io @@ -20,11 +40,17 @@ permissions: jobs: build-ci: name: Build gramps-ci + # The schedule event is handled by weekly-rebuild (fan-out); this job runs + # for push / pull_request / workflow_dispatch. + if: github.event_name != 'schedule' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Log in to GHCR + # Skip on pull_request: a fork PR's token is read-only and this build + # does not push, so the push credential is never exercised on a PR. + if: github.event_name != 'pull_request' uses: docker/login-action@v3 with: registry: ghcr.io @@ -48,7 +74,7 @@ jobs: id: params shell: bash run: | - ref="${{ github.ref_name }}" + ref="${{ github.base_ref || github.ref_name }}" suffix="${ref#maintenance/}" case "$suffix" in gramps[0-9][0-9]) ;; @@ -77,11 +103,41 @@ jobs: uses: docker/build-push-action@v6 with: context: .github/docker/gramps-ci - push: true + # Push everywhere EXCEPT pull_request, where this is a build-only + # validation that must not touch GHCR. + push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | GRAMPS_SERIES=${{ steps.params.outputs.series }} GRAMPS_FALLBACK_SHA=${{ steps.params.outputs.fallback_sha }} + # `no-cache` (workflow_dispatch input; empty/false on other events) + # forces a from-scratch rebuild — used by the weekly refresh. + no-cache: ${{ inputs.no-cache == true }} cache-from: type=gha cache-to: type=gha,mode=max + + weekly-rebuild: + name: Weekly no-cache refresh (fan-out) + # The scheduled event fires only on the default branch; dispatch a no-cache + # rebuild for every maintenance branch that carries this workflow. Inert + # until these workflows reach the default branch — by design, not a bug. + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + permissions: + contents: read + actions: write # gh workflow run + steps: + - name: Dispatch a no-cache rebuild per maintenance branch + env: + GH_TOKEN: ${{ github.token }} + run: | + git ls-remote --heads "https://github.com/${GITHUB_REPOSITORY}.git" \ + 'refs/heads/maintenance/gramps*' \ + | awk -F'refs/heads/' '{print $2}' \ + | grep -E '^maintenance/gramps[0-9][0-9]$' \ + | while read -r branch; do + gh workflow run docker-build.yml --repo "$GITHUB_REPOSITORY" \ + --ref "$branch" -f no-cache=true \ + || echo "::warning::dispatch failed for $branch (workflow not on that branch yet?)" + done From 2a2f778c5816d50dc92234002c19e4b0d26fa323 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Mon, 20 Jul 2026 04:02:37 +0200 Subject: [PATCH 43/47] ci-scripts: load addon tests via unittest, not a raw import_module probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The import-probe added for the load-failure taxonomy used importlib.import_module(modname), which is fragile: the worker appends the addon's own directory to sys.path so the addon's tests can import its top-level modules, and for an addon whose main module shares the addon's directory name (e.g. CalculateEstimatedDates/CalculateEstimatedDates.py) that appended entry shadows the namespace-package directory, so a dotted import_module resolves CalculateEstimatedDates to the .py MODULE and 'CalculateEstimatedDates.tests' then raises 'not a package'. 11 real addons failed this way on CI (gramps-independent — the fresh image would fail too). unittest's own loadTestsFromName resolves the package correctly, so load via it and classify the load failure from whichever form it takes: a RAISED exception (some Python versions raise SyntaxError) or a DEFERRED _FailedTest placeholder (ImportError, wrapped as an ImportError whose message embeds the original traceback). _dep_shaped inspects the terminal exception in that embedded traceback so a wrapped SyntaxError is still classified as a code bug, not a dependency skip. Verified across the F2/F3/F4 fixtures. --- .github/scripts/run_addon_tests.py | 93 +++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 16 deletions(-) diff --git a/.github/scripts/run_addon_tests.py b/.github/scripts/run_addon_tests.py index 945f1350c..163917e02 100644 --- a/.github/scripts/run_addon_tests.py +++ b/.github/scripts/run_addon_tests.py @@ -46,7 +46,6 @@ from __future__ import annotations import argparse -import importlib import os import re import signal @@ -84,18 +83,72 @@ def _module_timeout() -> int: _LOADERROR = "__RESULT__ loaderror" +def _terminal_exc_name(text: str) -> str | None: + """The exception class name on the last ``Type: message`` line of a + traceback string (``ModuleNotFoundError``, ``SyntaxError``, …), or None.""" + for line in reversed(text.strip().splitlines()): + m = re.match(r"^([A-Za-z_][\w.]*(?:Error|Exception|Warning)):", line.strip()) + if m: + return m.group(1).rsplit(".", 1)[-1] + return None + + def _dep_shaped(exc: BaseException) -> bool: """Whether a module load failure is a missing-dependency shape. Only these may be excused as an expected platform skip when the addon's - declared deps are unavailable: an ``ImportError`` (missing module), or the - ``ValueError`` ``gi.require_version`` raises for an absent typelib - (``Namespace X not available``). Everything else — SyntaxError, a bug in the - addon's own import-time code — is a real defect and must never be excused. + declared deps are unavailable: a missing module (ImportError / + ModuleNotFoundError), or the ``ValueError`` ``gi.require_version`` raises for + an absent typelib (``Namespace X not available``). A SyntaxError or a bug in + the addon's own import-time code is a real defect and must never be excused. + + Complication: ``unittest``'s loader wraps EVERY import-time exception — + including SyntaxError — into an ``ImportError`` whose message embeds the + original traceback (``Failed to import test module: …``). So for that + wrapper we cannot go by type; inspect the terminal exception in the embedded + traceback instead. """ + if isinstance(exc, ValueError): + return "not available" in str(exc) if isinstance(exc, ImportError): - return True - return isinstance(exc, ValueError) and "not available" in str(exc) + msg = str(exc) + if "Failed to import test module" in msg: + terminal = _terminal_exc_name(msg) + # Unknown terminal → treat as dep (tolerant: a missing dep is the + # common case and we would rather skip than falsely fail). + return terminal in (None, "ImportError", "ModuleNotFoundError") + return True # a bare ImportError (not the loader wrapper) + return False + + +def _load_failure_exception(suite: unittest.TestSuite): + """The exception behind a deferred import failure, or None. + + ``loadTestsFromName`` wraps a module that fails to import in a + ``unittest.loader._FailedTest`` placeholder carrying the original exception + in ``_exception``. Walk the suite for one and return that exception, so the + caller can classify the failure instead of running a placeholder that errors + anonymously. Private-API tolerant: if the internals ever change, return None + and let the suite run (the pre-taxonomy behaviour).""" + try: + from unittest.loader import _FailedTest + except Exception: + return None + + def _walk(test): + if isinstance(test, unittest.TestSuite): + for child in test: + found = _walk(child) + if found is not None: + return found + return None + if isinstance(test, _FailedTest): + return getattr(test, "_exception", None) or ImportError( + "module failed to load" + ) + return None + + return _walk(suite) def _bootstrap_gi() -> None: @@ -136,19 +189,27 @@ def _run_worker(modname: str, root: str = ".") -> int: addon_dir = os.path.join(root, addon) if addon_dir not in sys.path: sys.path.append(addon_dir) - # Probe the import EXPLICITLY first. loadTestsFromName does not raise on a - # module-level ImportError/SyntaxError — since Python 3.5 it swallows the - # error into a _FailedTest placeholder that only errors when run, so the - # failure would reach the parent as an anonymous `broke=1` with its shape - # lost. Importing here surfaces the real exception so it can be classified - # (dependency-shaped vs a code bug). + # Load via unittest (NOT a bare import_module probe): an addon whose + # top-level module shares the addon's directory name (e.g. + # CalculateEstimatedDates/CalculateEstimatedDates.py) is shadowed by + # addon_dir being on sys.path, which breaks a dotted import_module but not + # unittest's own resolution. A load failure surfaces two ways depending on + # the Python version and error kind — loadTestsFromName may RAISE it, or + # defer it into a _FailedTest placeholder that errors only when run (so the + # parent would otherwise see an anonymous `broke=1` with the shape lost). + # Handle both, and classify the real exception (dependency-shaped vs a code + # bug) either way. try: - importlib.import_module(modname) - except Exception as exc: # import-time failure + suite = unittest.defaultTestLoader.loadTestsFromName(modname) + except Exception as exc: # raised import-time failure kind = "dep" if _dep_shaped(exc) else "other" print(f"{_LOADERROR} kind={kind} {exc!r}", flush=True) return 0 - suite = unittest.defaultTestLoader.loadTestsFromName(modname) + load_exc = _load_failure_exception(suite) # deferred (_FailedTest) failure + if load_exc is not None: + kind = "dep" if _dep_shaped(load_exc) else "other" + print(f"{_LOADERROR} kind={kind} {load_exc!r}", flush=True) + return 0 result = unittest.TextTestRunner(verbosity=2).run(suite) broke = len(result.failures) + len(result.errors) print( From c206d1f9a10c68133bdb19a8100faf6ce62c7c94 Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Tue, 21 Jul 2026 21:33:45 +0200 Subject: [PATCH 44/47] ci-scripts: fix same-named-addon-module shadowing; honour module-level SkipTest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two real defects the fork CI run surfaced, both pre-existing (the original runner failed identically) but newly hit by the post-merge addon set: 1. Same-name shadowing. The worker appends the addon's directory to sys.path so its tests can import sibling modules by bare name (WebSearch's 'from models import …'). But most addons also ship /.py, and once / is on sys.path that regular module wins the bare name over the namespace-package directory — so loading the dotted test name .tests.test_x died with "module '' has no attribute 'tests'". 11 real addons failed this way (CalculateEstimatedDates, WebSearch, libaccess, …). Import the addon package from the repo root FIRST, pinning it in sys.modules, before the addon dir joins sys.path; the dotted load then resolves and the bare sibling imports still work. Reproduced and killed with a synthetic same-named-module fixture. 2. A module-level 'raise SkipTest(...)' — an addon's own "needs a display / PyGObject" guard — was classified as a code bug and failed the run. It is an explicit opt-out and is now honoured as a skip on every platform, regardless of declared-dep satisfiability. The load classifier gained a third outcome (skip | dep | other) and now reads the terminal exception out of unittest's wrapped-ImportError message, so a wrapped SkipTest/SyntaxError is classified by what actually happened. --- .github/scripts/run_addon_tests.py | 55 +++++++++++++++++++++++++---- tests/test_run_addon_tests_paths.py | 39 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/.github/scripts/run_addon_tests.py b/.github/scripts/run_addon_tests.py index 163917e02..494cef851 100644 --- a/.github/scripts/run_addon_tests.py +++ b/.github/scripts/run_addon_tests.py @@ -28,7 +28,9 @@ lives in ``addon_system_deps.py``). A module that fails to LOAD is excused as a platform skip only when the failure is dependency-shaped (ImportError, or gi's absent-typelib ValueError); a SyntaxError or a bug in the addon's own - import-time code is a real defect and always FAILS, on every platform. + import-time code is a real defect and always FAILS, on every platform. A + module that raises ``SkipTest`` at import is opting out explicitly and is + always honoured as a skip. Usage:: @@ -46,6 +48,7 @@ from __future__ import annotations import argparse +import importlib import os import re import signal @@ -85,14 +88,28 @@ def _module_timeout() -> int: def _terminal_exc_name(text: str) -> str | None: """The exception class name on the last ``Type: message`` line of a - traceback string (``ModuleNotFoundError``, ``SyntaxError``, …), or None.""" + traceback string (``ModuleNotFoundError``, ``SyntaxError``, ``SkipTest``…), + or None. Scans from the end, so the first match is the terminal line.""" for line in reversed(text.strip().splitlines()): - m = re.match(r"^([A-Za-z_][\w.]*(?:Error|Exception|Warning)):", line.strip()) + m = re.match(r"^([A-Za-z_][\w.]*)\s*:", line.strip()) if m: return m.group(1).rsplit(".", 1)[-1] return None +def _is_skip_request(exc: BaseException) -> bool: + """Whether a module opted out at import time via ``raise SkipTest(...)``. + + A module-level SkipTest is an explicit, deliberate opt-out (the addon's own + guard for "this needs a display / PyGObject / a backend I don't have"), so it + is honoured as a skip on every platform — never a failure, and never + dependent on the addon's declared system deps.""" + if isinstance(exc, unittest.SkipTest): + return True + # unittest's loader may wrap it (see _dep_shaped) — check the terminal type. + return isinstance(exc, ImportError) and _terminal_exc_name(str(exc)) == "SkipTest" + + def _dep_shaped(exc: BaseException) -> bool: """Whether a module load failure is a missing-dependency shape. @@ -121,6 +138,13 @@ def _dep_shaped(exc: BaseException) -> bool: return False +def _load_kind(exc: BaseException) -> str: + """Classify a module load failure: skip | dep | other.""" + if _is_skip_request(exc): + return "skip" + return "dep" if _dep_shaped(exc) else "other" + + def _load_failure_exception(suite: unittest.TestSuite): """The exception behind a deferred import failure, or None. @@ -187,6 +211,20 @@ def _run_worker(modname: str, root: str = ".") -> int: # working too. addon = modname.split(".", 1)[0] addon_dir = os.path.join(root, addon) + # Resolve the addon PACKAGE (its directory, from the repo root) before the + # addon dir joins sys.path. Many addons ship a top-level module named after + # the addon itself (/.py); the moment / is on sys.path + # that regular module wins the bare name over the namespace package, + # and the dotted test name .tests.test_x then dies with "module + # '' has no attribute 'tests'". Importing the package first pins it in + # sys.modules so the dotted load resolves, while the addon dir added below + # still serves the tests' bare sibling imports (e.g. `from models import …`). + try: + importlib.import_module(addon) + except Exception: + # Not importable as a package (single-file addon, or its __init__ needs + # deps): leave it; the dotted load below reports the real failure. + pass if addon_dir not in sys.path: sys.path.append(addon_dir) # Load via unittest (NOT a bare import_module probe): an addon whose @@ -202,12 +240,12 @@ def _run_worker(modname: str, root: str = ".") -> int: try: suite = unittest.defaultTestLoader.loadTestsFromName(modname) except Exception as exc: # raised import-time failure - kind = "dep" if _dep_shaped(exc) else "other" + kind = _load_kind(exc) print(f"{_LOADERROR} kind={kind} {exc!r}", flush=True) return 0 load_exc = _load_failure_exception(suite) # deferred (_FailedTest) failure if load_exc is not None: - kind = "dep" if _dep_shaped(load_exc) else "other" + kind = _load_kind(load_exc) print(f"{_LOADERROR} kind={kind} {load_exc!r}", flush=True) return 0 result = unittest.TextTestRunner(verbosity=2).run(suite) @@ -276,7 +314,12 @@ def _classify(modname: str, platform: str, root: str) -> tuple[bool, str]: if result_line.startswith(_LOADERROR): kind_m = re.search(r"\bkind=(\w+)", result_line) - dep_shaped = bool(kind_m) and kind_m.group(1) == "dep" + kind = kind_m.group(1) if kind_m else "" + if kind == "skip": + # The module raised SkipTest at import: an explicit opt-out, honoured + # on every platform regardless of declared-dep satisfiability. + return False, f" skip {modname} — module opted out (SkipTest at import)" + dep_shaped = kind == "dep" if not dep_shaped: # A non-dependency load failure (SyntaxError, a bug in the addon's # import-time code) is a real defect on every platform — never diff --git a/tests/test_run_addon_tests_paths.py b/tests/test_run_addon_tests_paths.py index b57593d59..fc79800f9 100644 --- a/tests/test_run_addon_tests_paths.py +++ b/tests/test_run_addon_tests_paths.py @@ -228,6 +228,45 @@ def test_non_integer_timeout_env_is_tolerated(self) -> None: self.assertEqual(result.returncode, 0, out) self.assertIn("ignoring non-integer", result.stderr) + def test_same_named_addon_module_does_not_shadow_package(self) -> None: + # Regression: many addons ship /.py. Once / is on + # sys.path (for the tests' bare sibling imports) that regular module wins + # the bare name over the namespace-package directory, and the dotted test + # name .tests.test_x died with "module '' has no attribute + # 'tests'" — 11 real addons failed this way on CI. + self._write("ShadowAddon/shadowaddon.gpr.py", 'register(GRAMPLET, id="s")\n') + self._write("ShadowAddon/ShadowAddon.py", "MAIN = 'addon main module'\n") + self._write("ShadowAddon/sibling.py", "SIB = 5\n") + self._write("ShadowAddon/tests/__init__.py", "") + self._write( + "ShadowAddon/tests/test_x.py", + "import unittest\n" + "from sibling import SIB\n" # bare sibling import needs addon dir + "\n" + "class T(unittest.TestCase):\n" + " def test_sibling(self):\n" + " self.assertEqual(SIB, 5)\n", + ) + result = self._run("ShadowAddon.tests.test_x", platform="apt") + out = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, out) + self.assertIn("ok ShadowAddon.tests.test_x", result.stdout, out) + + def test_module_level_skiptest_is_honored(self) -> None: + # A module that raises SkipTest at import is explicitly opting out (the + # addon's own "needs a display / PyGObject" guard) — honour it as a skip + # on every platform, never a failure. + self._write("SkipAddon/skipaddon.gpr.py", 'register(GRAMPLET, id="s")\n') + self._write("SkipAddon/tests/__init__.py", "") + self._write( + "SkipAddon/tests/test_x.py", + "import unittest\nraise unittest.SkipTest('no display here')\n", + ) + result = self._run("SkipAddon.tests.test_x", platform="apt") + out = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, out) + self.assertIn("opted out", out) + @unittest.skipUnless(os.name == "posix", "process-group kill is POSIX-only") def test_timeout_reaps_grandchild_holding_stdout(self) -> None: # A test that spawns a long-lived child inheriting the worker's stdout From 5bed7845b682f996c08374328c801e3ffdfea9fe Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Tue, 21 Jul 2026 23:49:38 +0200 Subject: [PATCH 45/47] ci(image): install PyICU so gramps' webreport module is importable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The integration lane reported TimePedigreeHTML as a failed-to-load addon (NameError: name 'localAlphabeticIndex' is not defined). The defect is not in the addon: gramps 6.0's gramps/plugins/webreport/common.py sets HAVE_ICU = False try: from icu import Locale ... except ImportError: try: from PyICU import Locale ... except ImportError: pass # <- nothing imported at all and later does 'else: AlphabeticIndex = localAlphabeticIndex' — a name only bound on the inner fallback paths. With NEITHER icu nor PyICU installed the module raises NameError at import, so every addon importing it fails to load. The CI image had no PyICU (the logs were full of 'ICU not loaded ... Localization will be impaired'), so CI was reporting an addon defect that does not exist for real users, who normally have PyICU. Install PyICU (and libicu-dev to build it). This fixes the false addon failure, silences the localization warnings, and makes the image represent a realistic Gramps environment. The underlying gramps guard is still wrong and worth reporting upstream. --- .github/docker/gramps-ci/Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index 9cc3d7c81..8f4fcbc13 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -67,6 +67,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ pkg-config \ python3-dev \ libcairo2-dev \ + libicu-dev \ intltool \ gettext \ git \ @@ -82,7 +83,18 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # ruff is PINNED: an unpinned ruff would change the lint verdict on rebuild (a # new rule in the E9/F63/F7/F82 selection flips a lane red — or silently stops # flagging — with no repo change). Bump deliberately, not by rebuild drift. -RUN pip install --no-cache-dir PyGObject pycairo orjson ruff==0.15.22 +# +# PyICU is REQUIRED, not optional. Gramps degrades localization without it +# ("ICU not loaded ... Localization will be impaired"), and worse, gramps 6.0's +# gramps/plugins/webreport/common.py cannot even be IMPORTED without it: when +# neither `icu` nor `PyICU` resolves, its outer `except ImportError: pass` +# leaves `localAlphabeticIndex` unbound and the module-level +# `AlphabeticIndex = localAlphabeticIndex` raises NameError. Every addon that +# imports that module (e.g. TimePedigreeHTML) then fails to load, so CI without +# PyICU reports an addon defect that does not exist for real users. Installing +# it also makes the image match a normal Gramps environment. (Built from source +# against libicu-dev above.) +RUN pip install --no-cache-dir PyGObject pycairo orjson PyICU ruff==0.15.22 # Install gramps: PyPI first, SHA-pinned git clone as fallback. RUN /bin/bash -eo pipefail <<'BASH' From bc498e9c9c3b10208c6f29d5749a3b4be0e1418a Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Tue, 21 Jul 2026 23:51:49 +0200 Subject: [PATCH 46/47] =?UTF-8?q?ci(image):=20add=20g++=20=E2=80=94=20PyIC?= =?UTF-8?q?U=20builds=20a=20C++=20extension?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyICU's wheel build needs a C++ compiler; the image carried only gcc, so the previous commit's PyICU install failed with "command 'g++' failed: No such file or directory". --- .github/docker/gramps-ci/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/docker/gramps-ci/Dockerfile b/.github/docker/gramps-ci/Dockerfile index 8f4fcbc13..95284d427 100644 --- a/.github/docker/gramps-ci/Dockerfile +++ b/.github/docker/gramps-ci/Dockerfile @@ -64,6 +64,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gir1.2-atk-1.0 \ gir1.2-gexiv2-0.10 \ gcc \ + g++ \ pkg-config \ python3-dev \ libcairo2-dev \ From b8504a28799a8a443627e5f137eb4d88f5c52aaf Mon Sep 17 00:00:00 2001 From: "Eduard R." Date: Wed, 22 Jul 2026 00:20:06 +0200 Subject: [PATCH 47/47] ci-scripts: don't call a class-level skip 'zero tests collected' The zero-collected-tests gate misfired on a module whose setUpClass raises SkipTest: unittest then reports tests=0 with the skips recorded, which is a SKIPPED module, not an empty one. RepositoriesReport's integration test (setUpClass skips with 'example.gramps not found') was failed as 'module loaded but collected zero tests'. Only claim zero-collected when tests==0 AND skipped==0; otherwise fall through to the all-skipped rule, which now compares skipped>=ran so the ran==0 / skipped>0 class-level case lands there and is judged on dep satisfiability like any other all-skip. --- .github/scripts/run_addon_tests.py | 17 +++++++++++------ tests/test_run_addon_tests_paths.py | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/scripts/run_addon_tests.py b/.github/scripts/run_addon_tests.py index 494cef851..f096ae5ae 100644 --- a/.github/scripts/run_addon_tests.py +++ b/.github/scripts/run_addon_tests.py @@ -345,22 +345,27 @@ def _classify(modname: str, platform: str, root: str) -> tuple[bool, str]: if broke: return True, f" FAIL {modname} — {broke} failed/errored" - if ran == 0: - # The module loaded but collected no tests (a class not subclassing + if ran == 0 and skipped == 0: + # The module loaded but collected NOTHING (a class not subclassing # TestCase, a mistyped method name, a refactor that broke collection). - # unittest exits 0 on this, so it would read as green — fail it. + # unittest exits 0 on this, so it would read as green — fail it. Note + # the skipped==0 guard: a class-level skip (setUpClass raising SkipTest) + # reports tests=0 with skips recorded, and that is a skipped module, not + # an empty one — it falls through to the all-skipped rule below. return True, ( f" FAIL {modname} — module loaded but collected zero tests " "(empty or misnamed test module reads as green)" ) - if skipped == ran: + if skipped >= ran: + # Everything skipped — including the ran==0/skipped>0 class-level case. + total = max(ran, skipped) if satisfiable: return True, ( - f" FAIL {modname} — all {ran} tests skipped " + f" FAIL {modname} — all {total} test(s) skipped " f"(degraded coverage; deps ARE available on {platform})" ) return False, ( - f" skip {modname} — all {ran} skipped, expected " + f" skip {modname} — all {total} skipped, expected " f"(addon system deps unavailable on {platform})" ) if skipped: diff --git a/tests/test_run_addon_tests_paths.py b/tests/test_run_addon_tests_paths.py index fc79800f9..3b2666bb3 100644 --- a/tests/test_run_addon_tests_paths.py +++ b/tests/test_run_addon_tests_paths.py @@ -252,6 +252,28 @@ def test_same_named_addon_module_does_not_shadow_package(self) -> None: self.assertEqual(result.returncode, 0, out) self.assertIn("ok ShadowAddon.tests.test_x", result.stdout, out) + def test_class_level_skip_is_not_reported_as_zero_tests(self) -> None: + # setUpClass raising SkipTest yields tests=0 with skips recorded. That is + # a skipped module, not an empty one — the zero-collected rule must not + # claim "collected zero tests" (it did, on RepositoriesReport in CI). + # The addon declares an unsatisfiable-on-conda dep so the all-skipped + # rule treats it as an expected platform skip. + self._write_gi_addon( + "GiClassSkip", + "import unittest\n" + "class T(unittest.TestCase):\n" + " @classmethod\n" + " def setUpClass(cls):\n" + " raise unittest.SkipTest('fixture not available')\n" + " def test_a(self):\n" + " pass\n", + ) + result = self._run("GiClassSkip.tests.test_x", platform="conda") + out = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, out) + self.assertNotIn("zero tests", out) + self.assertIn("skip", result.stdout) + def test_module_level_skiptest_is_honored(self) -> None: # A module that raises SkipTest at import is explicitly opting out (the # addon's own "needs a display / PyGObject" guard) — honour it as a skip