Skip to content

AssociationsTool: stop shadowing the gettext _ with a loop variable #96

AssociationsTool: stop shadowing the gettext _ with a loop variable

AssociationsTool: stop shadowing the gettext _ with a loop variable #96

Workflow file for this run

name: CI
# Branch-neutral workflow: the image tag, make.py argument, and Gramps
# series pin are all derived from the branch ref at runtime, so the same
# file runs unchanged on maintenance/gramps60, maintenance/gramps61, and
# any future maintenance/grampsNN branch.
#
# NOTE for maintainers — first push to a new maintenance branch: the
# corresponding gramps-ci:<suffix> image does not exist in GHCR yet, so
# the container jobs in this workflow will fail at "Initialize containers"
# on the very first run. The companion docker-build.yml workflow fires
# on the same push and builds/pushes the image (~5 min cold). After it
# finishes, re-run the failed CI jobs (Actions tab → this run → "Re-run
# failed jobs") and they will pull the now-existing image. This race
# happens only on initial branch creation — every subsequent push to
# that branch finds the image already in GHCR.
#
# Full operational runbook (GHCR visibility, PyPI-release transitions,
# diagnostic log markers, etc.): .github/CI-MAINTAINER.md
on:
push:
branches: [maintenance/gramps**]
pull_request:
branches: [maintenance/gramps**]
# Least privilege. Every job only reads the repo; the CI image is a public
# GHCR package, so pulling it needs no `packages` scope. This workflow runs
# addon-influenced code (pip package hooks derived from requires_mod, and
# addon import-time code in the load/test lanes), so the ambient token it
# hands that code must not carry write scopes. Presumes the gramps-ci package
# is public (the one-time visibility flip in .github/CI-MAINTAINER.md).
permissions:
contents: read
jobs:
# -----------------------------------------------------------------
# Setup — derive the branch suffix (gramps60 / gramps61 / …) from
# the ref. On push events github.ref_name is the branch being
# pushed; on pull_request events github.base_ref is the target
# branch. Either way the suffix is what follows "maintenance/".
# -----------------------------------------------------------------
setup:
name: Setup
runs-on: ubuntu-latest
outputs:
branch_suffix: ${{ steps.compute.outputs.branch_suffix }}
ci_image: ${{ steps.compute.outputs.ci_image }}
steps:
- id: compute
shell: bash
run: |
ref="${{ github.base_ref || github.ref_name }}"
suffix="${ref#maintenance/}"
case "$suffix" in
gramps[0-9][0-9]) ;;
*) echo "::error::unexpected ref '$ref' (suffix '$suffix')"; exit 1 ;;
esac
echo "branch_suffix=$suffix" >> "$GITHUB_OUTPUT"
# Lowercase the owner/name: GHCR refs must be lowercase, and
# docker-build's metadata-action lowercases what it PUSHES, so the
# pull side must match — otherwise a fork whose owner has uppercase
# letters pulls an invalid (mixed-case) reference and every container
# job fails at init. ${VAR,,} is a bash lowercasing expansion; there
# is no expression-level equivalent in Actions.
repo_lc="${GITHUB_REPOSITORY,,}"
echo "ci_image=ghcr.io/${repo_lc}/gramps-ci:$suffix" >> "$GITHUB_OUTPUT"
# -----------------------------------------------------------------
# Lint (ci container)
# -----------------------------------------------------------------
lint:
name: Lint
needs: setup
runs-on: ubuntu-latest
container:
image: ${{ needs.setup.outputs.ci_image }}
steps:
- uses: actions/checkout@v4
- name: Run ruff (syntax and import errors only)
# Skip addon directories whose every register() in .gpr.py sets
# include_in_listing=False — those addons are not built or released
# by make.py, so CI does not gate on their lint state (per Gary
# Griffin's request on PR #820). To re-enable lint gating for an
# addon, set include_in_listing=True on at least one register()
# call in its descriptor (or remove the field — Gramps' default
# is True). Repeated inline rather than centralised so each job
# step stays self-contained.
shell: bash
run: |
source .github/scripts/active_addons.sh
excludes=""
for d in */; do
d="${d%/}"
ls "$d"/*.gpr.py >/dev/null 2>&1 || continue
is_active "$d" || excludes="$excludes --exclude=$d"
done
ruff check --select=E9,F63,F7,F82 --no-fix --exclude='*.gpr.py' $excludes .
- name: Check trailing whitespace in Python files
run: |
# Use PCRE (-P): in BRE/ERE the bracket expression [ \t] is the
# set { space, backslash, 't' } — git grep matches anything ending
# in 't', not just whitespace. -P makes \t a tab.
if git --no-pager grep --color -n --full-name -P '[ \t]+$' -- '*.py'; then
echo "::error::Trailing whitespace found in Python files"
exit 1
fi
# -----------------------------------------------------------------
# Addon structure (bare runner — just bash, no deps needed)
# -----------------------------------------------------------------
addon-structure:
name: Addon Structure
runs-on: ubuntu-latest
# Non-blocking until the four addons missing po/template.pot are fixed
# in a follow-up PR. Flip this off in that PR.
continue-on-error: true
steps:
- uses: actions/checkout@v4
- name: Check all listed addons have po/template.pot
# Skip include_in_listing=False addons (see lint job for rationale).
shell: bash
run: |
source .github/scripts/active_addons.sh
failed=0
for gpr in */*.gpr.py; do
addon_dir="$(dirname "$gpr")"
is_active "$addon_dir" || continue
if [ ! -d "$addon_dir/po" ]; then
echo "::error::$addon_dir is missing po/ directory"
failed=1
elif [ ! -f "$addon_dir/po/template.pot" ]; then
echo "::error::$addon_dir is missing po/template.pot"
failed=1
fi
done
if [ "$failed" -eq 0 ]; then
echo "All listed addons have po/template.pot"
fi
exit $failed
# -----------------------------------------------------------------
# Compile check (ci container)
# -----------------------------------------------------------------
compile-check:
name: Compile Check
needs: setup
runs-on: ubuntu-latest
container:
image: ${{ needs.setup.outputs.ci_image }}
steps:
- uses: actions/checkout@v4
- name: Compile all Python files in listed addons (excluding .gpr.py)
# Skip include_in_listing=False addons (see lint job for rationale).
shell: bash
run: |
source .github/scripts/active_addons.sh
skipped=""
for d in */; do
d="${d%/}"
ls "$d"/*.gpr.py >/dev/null 2>&1 || continue
is_active "$d" || skipped="$skipped $d"
done
failed=0
while IFS= read -r f; do
skip=0
for s in $skipped; do
case "$f" in ./$s/*) skip=1; break;; esac
done
[ "$skip" = 1 ] && continue
if ! python3 -m py_compile "$f" 2>&1; then
failed=1
fi
done < <(find . -name '*.py' ! -name '*.gpr.py' ! -path './.git/*' ! -path '*/__pycache__/*')
exit $failed
# -----------------------------------------------------------------
# Unit tests — Linux (ci container)
# -----------------------------------------------------------------
unit-test-linux:
name: Unit Tests (Linux)
needs: setup
runs-on: ubuntu-latest
container:
image: ${{ needs.setup.outputs.ci_image }}
steps:
- uses: actions/checkout@v4
- name: Install addon system deps (requires_gi / requires_exe / source-built requires_mod)
# System deps (GI typelibs, executables, and the -dev headers / libpq that
# source-built requires_mod link against — pygraphviz→libgraphviz-dev,
# psycopg2/psycopg→libpq-dev) are not pip-installable as named and gramps
# only *checks* requires_mod, so the image cannot bake them generically
# (its build context excludes addons-source). Derive the apt set from every
# .gpr.py via the single-source map and install it — the container runs as
# root. MUST run before "Install addon runtime deps" so the headers are
# present when pip compiles the source-built modules; the compiler toolchain
# those builds need lives in the CI image (no longer purged there).
shell: bash
run: |
pkgs=$(python3 .github/scripts/addon_system_deps.py --platform apt)
if [ -n "$pkgs" ]; then
echo "→ addon system deps (apt): $pkgs"
apt-get update
apt-get install -y --no-install-recommends $pkgs
else
echo "no requires_gi / requires_exe / source-built requires_mod declarations found"
fi
- name: Validate addon system deps are mapped
# Every requires_gi / requires_exe an addon declares must have an entry in
# addon_system_deps.py, and every requires_mod must be classified as either
# wheel-only or source-built (with a system package), so the install list
# never silently drifts from what addons declare — the build-toolchain
# coverage gap returns the moment a new source-built requires_mod is added
# without a mapping. (The system-dep analogue of the requires_mod dep
# gate below.)
shell: bash
run: |
python3 .github/scripts/addon_system_deps.py --unmapped . || {
echo "::error::Addon(s) declare requires_gi/requires_exe/requires_mod with no entry in .github/scripts/addon_system_deps.py — add a mapping row (or, for a wheel-only module, list it in WHEEL_ONLY_MODS)."
exit 1
}
- name: Install addon runtime deps (derived from requires_mod)
# Auto-derive the union of requires_mod across every .gpr.py in
# the repo. Mirrors Gramps' Addon Manager install path
# (gramps/gui/plug/_windows.py __on_install_clicked → req.install →
# gen/utils/requirements.py). Keeps .gpr.py files as the single
# source of truth for addon deps — no parallel list to maintain
# in the image or workflow. Import→distribution names (PIL→Pillow)
# come from the installed gramps' own _IMPORT_TO_PYPI table
# (gen/utils/pypi.py, gramps 6.1+, PR #2308), with a local mirror
# in addon_python_deps.py for lanes on 6.0. Best-effort: a package
# needing exotic
# system deps (pygraphviz → graphviz-dev, psycopg2 → libpq-dev)
# may fail here; the affected addon's tests will skip or fail in
# isolation without blocking the rest.
shell: bash
run: |
addon_mods=$(python3 .github/scripts/addon_python_deps.py --install-list .)
if [ -n "$addon_mods" ]; then
echo "→ addon deps: $addon_mods"
for mod in $addon_mods; do
pip install "$mod" || echo "× $mod failed to install (continuing)"
done
else
echo "no requires_mod declarations found"
fi
- name: Validate requires_mod names against Gramps' dep gate
# Cross-check: every requires_mod entry that pip successfully
# installed in the previous step must also pass Gramps' own dep
# gate — the script delegates to the installed gramps'
# Requirements().check_mod (gramps/gen/utils/requirements.py):
# find_spec on 6.0; find_spec plus a real import on 6.1+ (gramps
# PR #2308), so the gate matches whichever series this lane
# ships. A name that pip-installs but does not import is a
# declaration bug — e.g. requires_mod=["Pillow"] when the
# importable name is "PIL". Pip-installed-ness is probed by the
# mapped distribution name (pip only knows "Pillow", not "PIL").
# A name whose distribution never installed is judged by category:
# a wheel-only module (WHEEL_ONLY_MODS) ships a wheel that installs
# everywhere, so a miss is a provisioning regression and FAILS the
# gate; a source-built module (pygraphviz/psycopg2) can miss on an
# image/system gap and stays an advisory skip.
shell: bash
run: |
python3 .github/scripts/addon_python_deps.py --check-resolves .
- name: Run per-addon unit tests
# Filename convention (all OSes):
# test_*.py — general (any OS)
# test_linux_*.py — Linux-only
# test_windows_*.py — Windows-only
# test_integration_*.py — Linux-only, full-pipeline/DB-backed
# The Linux job runs test_*.py except the Windows-only and
# integration buckets. Integration tests run in their own job.
#
# shell: bash — the container's default shell is /bin/sh
# (dash on python:3.12-slim), which does not support the
# ${var//pattern/repl} and ${var%.py} parameter expansions
# used below.
shell: bash
env:
PYTHONPATH: .
run: |
source .github/scripts/active_addons.sh
modules=""
shopt -s globstar # also match nested-package tests/<subpkg>/test_*.py
# Glob is scoped to */tests/**/ ON PURPOSE. The only root-level test
# module in the tree, DynamicWeb/test_dynamicweb.py, is a nose-era dev
# harness (nose is gone on py3.12; it asserts a USER_PLUGINS install and
# drives Gramps.py from a source checkout) — unrunnable here, and it
# declares no requires_gi, so it would hard-fail if matched. Its
# CI-shaped sibling DynamicWeb/tests/test_dwr_tree_names.py IS matched,
# so DynamicWeb keeps coverage. The runner can load a root-level module,
# so this is a policy choice, not a limitation.
for f in */tests/**/test_*.py; do
[ -f "$f" ] || continue
addon="${f%%/*}"
is_active "$addon" || continue
case "$(basename "$f")" in
test_integration*) continue ;;
test_windows_*) continue ;;
esac
case "$f" in
# Sqlite/tests/test_sqlite.py predates this pipeline: it needs
# GRAMPS_RESOURCES + example/gramps/example.gramps from a gramps
# SOURCE checkout (the pip/conda wheels ship only gramps/ + share,
# not example/) and writes fixed /tmp paths with no cleanup. Un-
# excluding is a follow-up: provide example.gramps in the lane and
# port the test to tempfile.
Sqlite/tests/test_sqlite.py) continue ;;
esac
mod="${f%.py}"
mod="${mod//\//.}"
modules="$modules $mod"
done
if [ -n "$modules" ]; then
echo "Running unit tests:$modules"
# xvfb-run: some addons create a Gtk style context at import and
# need a display (else a hard Gtk-ERROR abort, not a clean skip).
# run_addon_tests.py: pins the GI versions like gramps' launcher
# (so gramps.gui imports load GTK 3, no PyGIWarning) and fails a
# wholly-skipped module unless the addon's deps are unavailable on
# this platform.
xvfb-run -a --server-args="-screen 0 1920x1080x24" \
python3 .github/scripts/run_addon_tests.py --platform apt --root . $modules
else
echo "No per-addon unit test modules found"
fi
# -----------------------------------------------------------------
# Unit tests — Windows (conda-forge: bundles PyGObject + GTK + Gramps)
# -----------------------------------------------------------------
unit-test-windows:
name: Unit Tests (Windows)
needs: setup
runs-on: windows-latest
defaults:
run:
shell: bash -el {0}
steps:
- uses: actions/checkout@v4
- name: Set up Miniforge
uses: conda-incubator/setup-miniconda@v3
with:
miniforge-version: latest
activate-environment: addons-ci
environment-file: .github/environment.yml
use-mamba: true
- name: Verify environment
run: |
mamba info
mamba list | head -30
python -c "import gramps, gi; print('deps OK')"
- name: Report gramps-vs-branch series (Windows lane caveat)
# The Linux lane runs the branch's exact gramps in its CI image
# (.github/docker/gramps-ci/Dockerfile, PyPI-first / git-tip fallback).
# The conda Windows lane installs gramps from PyPI via environment.yml's
# `pip:` block (NOT from conda-forge), pinned `gramps>=6.0,<6.1`. So on a
# maintenance/gramps61 (or later) branch it validates addons against
# 6.0.x, not the branch's series — for two independent reasons: the pin's
# `<6.1` excludes 6.1, and 6.1 is not published on PyPI yet anyway. This
# step surfaces the mismatch honestly; it does NOT fail. Addon tests that
# depend on series-exact gramps behaviour skip themselves on Windows
# (e.g. TMGimporter's real-DB import tests) and run on the Linux lane
# instead. The caveat clears only when BOTH hold: gramps 6.1 is on PyPI,
# AND environment.yml's pin has been bumped on that branch (it does not
# self-heal — see .github/CI-MAINTAINER.md).
run: |
suffix="${{ needs.setup.outputs.branch_suffix }}" # e.g. gramps61
digits="${suffix#gramps}" # e.g. 61
want="${digits:0:1}.${digits:1}" # e.g. 6.1
have="$(python -c 'from gramps.version import major_version; print(major_version)')"
if [ "$want" = "$have" ]; then
echo "conda-forge gramps $have matches branch series $want — addons tested against the branch's gramps"
else
echo "::warning::Windows lane: branch targets gramps $want but conda-forge ships $have; addons here are validated against $have. Full $want coverage is on the Linux lane (its CI image git-builds $want). See step comment for why conda-Windows cannot build $want."
fi
- name: Install addon system deps (requires_gi / requires_exe / source-built requires_mod)
# The conda-forge-available subset of the single-source map. This lane
# provisions its OWN deps and must mirror the apt lane: source-built
# requires_mod that conda-forge ships prebuilt — pygraphviz, psycopg2,
# psycopg — are installed here as full conda-forge packages
# (MOD_BUILD_PACKAGES' conda side), so the later pip step finds them already
# satisfied and the affected addons' suites RUN on Windows instead of
# silently skipping. `mamba install` fails the job if a mapped package
# cannot be resolved, so a provisioning gap aborts honestly — never a silent
# green. The GTK 3 addon GI libs (goocanvas/osm-gps-map/gexiv2) are
# genuinely NOT on conda-forge, so the map keeps them at None and they are
# not installed; addons needing only those skip on Windows by necessity,
# which run_addon_tests tolerates for declared GI deps (--platform conda).
run: |
pkgs=$(python .github/scripts/addon_system_deps.py --platform conda)
if [ -n "$pkgs" ]; then
echo "→ addon system deps (conda): $pkgs"
mamba install -y -c conda-forge $pkgs
else
echo "no conda-available addon system deps to install"
fi
- name: Validate addon system deps are mapped
# Same drift gate as unit-test-linux (see its comment). This lane runs
# it independently and BEFORE its own pip step: unit-test-windows only
# `needs: setup`, not unit-test-linux, so it must not lean on the Linux
# gate — otherwise a novel requires_mod would reach `pip install`
# unguarded here (arbitrary package execution) while Linux blocks it.
# `python` (conda-forge env) to match the surrounding Windows style.
run: |
python .github/scripts/addon_system_deps.py --unmapped . || {
echo "::error::Addon(s) declare requires_gi/requires_exe/requires_mod with no entry in .github/scripts/addon_system_deps.py — add a mapping row (or, for a wheel-only module, list it in WHEEL_ONLY_MODS)."
exit 1
}
- name: Install addon runtime deps (derived from requires_mod)
# See unit-test-linux for rationale. Uses `python` (conda-forge
# env) to match the surrounding Windows job style.
run: |
addon_mods=$(python .github/scripts/addon_python_deps.py --install-list .)
if [ -n "$addon_mods" ]; then
echo "→ addon deps: $addon_mods"
for mod in $addon_mods; do
pip install "$mod" || echo "× $mod failed to install (continuing)"
done
else
echo "no requires_mod declarations found"
fi
- name: Validate requires_mod names against Gramps' dep gate
# See unit-test-linux for rationale. Uses `python` (conda-forge
# env) to match the surrounding Windows job style.
run: |
python .github/scripts/addon_python_deps.py --check-resolves .
- name: Run per-addon unit tests
# See filename-convention note in unit-test-linux. The Windows
# job runs test_*.py except test_linux_* and test_integration_*.
env:
PYTHONPATH: .
run: |
source .github/scripts/active_addons.sh
modules=""
# Glob scoped to */tests/**/ (see unit-test-linux for why DynamicWeb's
# root-level test module is deliberately not matched).
shopt -s globstar # also match nested-package tests/<subpkg>/test_*.py
for f in */tests/**/test_*.py; do
[ -f "$f" ] || continue
addon="${f%%/*}"
is_active "$addon" || continue
case "$(basename "$f")" in
test_integration*) continue ;;
test_linux_*) continue ;;
esac
case "$f" in
# excluded — see unit-test-linux for the rationale (needs a gramps
# source checkout's example.gramps; a follow-up).
Sqlite/tests/test_sqlite.py) continue ;;
esac
mod="${f%.py}"
mod="${mod//\//.}"
modules="$modules $mod"
done
if [ -n "$modules" ]; then
echo "Running unit tests:$modules"
# No xvfb on Windows (GTK renders natively). run_addon_tests pins
# the GI versions and tolerates addons whose GI deps are not on
# conda-forge (they skip here by platform necessity).
python .github/scripts/run_addon_tests.py --platform conda --root . $modules
else
echo "No per-addon unit test modules found"
fi
# -----------------------------------------------------------------
# Integration tests — Gramps (ci container, xvfb available)
# -----------------------------------------------------------------
integration-test:
name: Integration Tests (Gramps)
runs-on: ubuntu-latest
needs: [setup, unit-test-linux]
container:
image: ${{ needs.setup.outputs.ci_image }}
options: --init
steps:
- uses: actions/checkout@v4
- name: Install addon system deps (requires_gi / requires_exe / source-built requires_mod)
# Same as unit-test-linux: derive the apt set from the single-source
# map and install it (container runs as root). The plugin registration
# test subprocess-loads each addon module, so the GI typelibs those
# modules import — and the -dev headers / libpq their source-built
# requires_mod compile/link against — must be present here too. (Mapping is
# drift-guarded in unit-test-linux, which this job needs:, so no duplicate
# gate here.)
shell: bash
run: |
pkgs=$(python3 .github/scripts/addon_system_deps.py --platform apt)
if [ -n "$pkgs" ]; then
echo "→ addon system deps (apt): $pkgs"
apt-get update
apt-get install -y --no-install-recommends $pkgs
fi
- name: Install addon runtime deps (derived from requires_mod)
# See unit-test-linux for rationale. The plugin registration test
# subprocess-loads each addon's module, which imports its
# requires_mod packages.
shell: bash
run: |
addon_mods=$(python3 .github/scripts/addon_python_deps.py --install-list .)
if [ -n "$addon_mods" ]; then
echo "→ addon deps: $addon_mods"
for mod in $addon_mods; do
pip install "$mod" || echo "× $mod failed to install (continuing)"
done
else
echo "no requires_mod declarations found"
fi
- name: Validate requires_mod names against Gramps' dep gate
# See unit-test-linux for rationale.
shell: bash
run: |
python3 .github/scripts/addon_python_deps.py --check-resolves .
- name: Run plugin registration tests
# shell: bash for consistency with the surrounding steps; the
# current command uses no bashisms, but keeps this block safe
# against future edits. Container default is /bin/sh → dash.
#
# gi_bootstrap on PYTHONPATH pins the GI versions (like the gramps GUI
# launcher) for this process AND the addon-module subprocesses this test
# spawns, so gramps.gui imports load GTK 3 without a PyGIWarning.
#
# NOT run under xvfb: this test only *loads* (imports) addon modules in
# subprocesses and tolerates load failures; it does not render. Giving it
# a display made an addon load hang on the (absent) AT-SPI accessibility
# bus until the per-load timeout. Imports that build a Gtk style context
# are exercised under xvfb in the unit/integration test runs instead.
shell: bash
env:
PYTHONPATH: .github/scripts/gi_bootstrap:.
run: |
python3 -m unittest discover -s tests -p "test_*.py" -t . -v
- name: Run per-addon integration tests
# shell: bash — see unit-test-linux for rationale; the
# ${var//pattern/repl} and ${var%.py} expansions below are
# bash-only.
shell: bash
env:
PYTHONPATH: .
run: |
source .github/scripts/active_addons.sh
modules=""
shopt -s globstar # also match nested-package tests/<subpkg>/test_integration*.py
for f in */tests/**/test_integration*.py; do
[ -f "$f" ] || continue
addon="${f%%/*}"
is_active "$addon" || continue
mod="${f%.py}"
mod="${mod//\//.}"
modules="$modules $mod"
done
if [ -n "$modules" ]; then
echo "Running per-addon integration tests:$modules"
xvfb-run -a --server-args="-screen 0 1920x1080x24" \
python3 .github/scripts/run_addon_tests.py --platform apt --root . $modules
else
echo "No per-addon integration test modules found"
fi
# -----------------------------------------------------------------
# Build (ci container)
# -----------------------------------------------------------------
build:
name: Build
needs: setup
runs-on: ubuntu-latest
container:
image: ${{ needs.setup.outputs.ci_image }}
steps:
- uses: actions/checkout@v4
- name: Determine GRAMPSPATH
id: gramps-path
run: |
GPATH=$(python3 -c "import gramps, os; print(os.path.dirname(os.path.dirname(gramps.__file__)))")
echo "path=$GPATH" >> "$GITHUB_OUTPUT"
- name: Build all addons
env:
GRAMPSPATH: ${{ steps.gramps-path.outputs.path }}
run: |
mkdir -p ../download
python3 make.py "${{ needs.setup.outputs.branch_suffix }}" build all