From 04941ea7f0c97c72f528b1bb11d3a94a3133d66c Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 14:48:54 -0700 Subject: [PATCH 01/11] Add GrampsWebApiDb: use a Gramps Web API server as a live database backend Lets Gramps open a gramps-web-api server (e.g. gramps-connect, Gramps Web) as a regular family tree -- read and write, no export/import step. Subclasses the stock SQLite DBAPI backend rather than reimplementing DbReadBase/DbWriteBase, and keeps a local mirror in sync incrementally via the server's transaction-history endpoint; local edits push back through transaction_commit(). Credentials come from a single GRAMPS_WEB_API_KEY env var (a non-expiring refresh token) rather than a login dialog, which also makes the same webapi_client.py usable as a bare SDK outside Gramps. Status UNSTABLE: no conflict handling (writes are last-write-wins by design, not yet), no undo/redo integration, no media sync. Verified end-to-end against a live gramps-web-api server, including live use from Gramps desktop itself, but no automated test suite yet. Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/grampswebapidb.gpr.py | 36 +++ GrampsWebApiDb/grampswebapidb.py | 231 ++++++++++++++++ GrampsWebApiDb/po/template.pot | 32 +++ GrampsWebApiDb/webapi_client.py | 392 +++++++++++++++++++++++++++ 4 files changed, 691 insertions(+) create mode 100644 GrampsWebApiDb/grampswebapidb.gpr.py create mode 100644 GrampsWebApiDb/grampswebapidb.py create mode 100644 GrampsWebApiDb/po/template.pot create mode 100644 GrampsWebApiDb/webapi_client.py diff --git a/GrampsWebApiDb/grampswebapidb.gpr.py b/GrampsWebApiDb/grampswebapidb.gpr.py new file mode 100644 index 000000000..59cdb9bb3 --- /dev/null +++ b/GrampsWebApiDb/grampswebapidb.gpr.py @@ -0,0 +1,36 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Douglas S. Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +register( + DATABASE, + id="grampswebapidb", + status=UNSTABLE, + name=_("GrampsWebApiDb"), + name_accell=_("Gramps _Web API Database"), + description=_( + "Use a Gramps Web API server (e.g. gramps-connect or Gramps Web) " + "as a live database, mirrored locally for speed." + ), + version="0.1.0", + gramps_target_version="6.0", + fname="grampswebapidb.py", + databaseclass="WebApiDB", + authors=["Doug Blank"], + authors_email=["doug.blank@gmail.com"], +) diff --git a/GrampsWebApiDb/grampswebapidb.py b/GrampsWebApiDb/grampswebapidb.py new file mode 100644 index 000000000..3dd29d9c5 --- /dev/null +++ b/GrampsWebApiDb/grampswebapidb.py @@ -0,0 +1,231 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Douglas S. Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Database backend that mirrors a Gramps Web API server locally. + +Design +------ +This subclasses the stock SQLite DBAPI backend rather than DbReadBase / +DbWriteBase directly. DbGeneric (gramps.gen.db.generic) already implements +every get_*_from_handle / iter_* / get_number_of_* method generically on +top of a small Connection-like object (execute/fetchone/fetchall/commit/ +table_exists/...) -- see SQLite in gramps/plugins/db/dbapi/sqlite.py. So +reads only need a local, fast, complete SQLite mirror; nothing above the +Connection layer needs reimplementing. + +The mirror is kept current via GET /api/transactions/history/?after=, +the same per-object transaction log gramps-web-api's own undo system uses +(gramps_webapi/undodb.py's DbUndoSQLWeb.get_transactions()). Confirmed +against a live server: each entry is a *transaction* dict with a nested +"changes" list, each change carrying obj_class ("Person", "Family", ...), +trans_type (TXNADD=0/TXNUPD=1/TXNDEL=2), obj_handle, and -- when the +"new" query param is set -- new_data, a "_class"-tagged dict in the same +shape gramps.gen.lib.json_utils.data_to_object() reconstructs objects +from (it's literally what the server's own object_to_data(obj) produced +when the change was committed). So syncing is: remember the timestamp of +the last transaction applied, ask for everything after it, and for each +change either data_to_object(new_data) + commit_() (add and update +both being upserts, no need to distinguish) or remove_() for a +delete. + +Credentials come from a single environment variable, GRAMPS_WEB_API_KEY +(see webapi_client.py for its "*" shape and +the tradeoffs of using a refresh token here rather than a real scoped +personal-access-token). There is deliberately no per-tree settings.ini and +no login dialog: the same env var also works as a bare SDK credential +(WebApiHandler.from_env()) for scripts that talk to the server directly, +without going through Gramps at all -- one credential, two consumers. + +Write-through (local edits pushed back to the server) hooks +transaction_commit() rather than the individual commit_person/ +commit_family/... methods: DbTxn.__exit__ calls self.db.transaction_commit +(gramps/gen/db/txn.py) exactly once per completed local transaction, and +DbTxn already accumulates every add/update/delete in that transaction via +its own get_recnos()/get_record() -- transaction_to_json() below turns +that into the flat {type, handle, _class, old, new} list POST +/transactions/ expects (confirmed against base.py's own POST /people/ +handler, which builds its response the same way). This must run *before* +super().transaction_commit(), since DBAPI.transaction_commit() clears the +transaction's records as its last step. + +The other place a DbTxn gets used is _sync_from_server() itself, applying +server-pulled changes -- that uses batch=True, and DBAPI._commit_base() +skips trans.add() entirely for batch transactions (see dbapi.py), so +transaction_to_json() naturally sees nothing there and no push happens. +No separate "am I currently syncing" flag is needed to stop synced +changes from being echoed straight back to the server. + +Conflict handling is not implemented: pushes go out with force=1, which +skips the server's old-data-matches check entirely (see +gramps_webapi/api/tasks.py's process_transactions), so this is +last-write-wins by design. If the push itself fails (network error, +non-conflict validation error), the local commit has already happened +and is not rolled back -- the local mirror just drifts from the server +until the next successful push or read sync. Undo/redo integration is +also still out of scope. +""" + +import logging +from urllib.error import HTTPError, URLError + +from gramps.gen.const import GRAMPS_LOCALE as glocale +from gramps.gen.db import DbTxn +from gramps.gen.db.dbconst import ( + CLASS_TO_KEY_MAP, + KEY_TO_CLASS_MAP, + KEY_TO_NAME_MAP, + TXNADD, + TXNDEL, + TXNUPD, +) +from gramps.gen.db.exceptions import DbConnectionError +from gramps.gen.lib.json_utils import data_to_object, remove_object +from gramps.plugins.db.dbapi.sqlite import SQLite + +from webapi_client import WebApiHandler + +_ = glocale.translation.gettext +LOG = logging.getLogger("grampswebapidb") + +#: How many transactions to request per page while syncing. +SYNC_PAGE_SIZE = 100 + +#: Failure modes from WebApiHandler.from_env()/push_transaction(): a +#: malformed/missing key (ValueError), a bad server response shape +#: (KeyError/JSONDecodeError, the latter a ValueError subclass), or the +#: server being unreachable (HTTPError/URLError/OSError -- socket.timeout +#: is an OSError subclass). +_CONNECTION_ERRORS = (ValueError, KeyError, HTTPError, URLError, OSError) + +_TRANS_TYPE_NAME = {TXNADD: "add", TXNUPD: "update", TXNDEL: "delete"} + + +def transaction_to_json(transaction): + """ + Build the flat change-list payload POST /transactions/ expects, from + a just-committed local DbTxn. Ported from GrampsWebSync's + webapihandler.transaction_to_json (same repo, same license, credit + David Straub) instead of imported, for the same no-cross-addon- + dependency reason as webapi_client.py. + """ + out = [] + for recno in transaction.get_recnos(reverse=False): + key, action, handle, old_data, new_data = transaction.get_record(recno) + obj_cls_name = KEY_TO_CLASS_MAP.get(key) + if obj_cls_name is None: + continue # reference-type record, not a primary object + out.append( + { + "type": _TRANS_TYPE_NAME[action], + "handle": handle, + "_class": obj_cls_name, + "old": None if old_data is None else remove_object(old_data), + "new": None if new_data is None else remove_object(new_data), + } + ) + return out + + +class WebApiDB(SQLite): + """ + DBAPI backend whose local SQLite connection is a mirror of a + Gramps Web API server, kept in sync via the server's transaction + history endpoint. + """ + + def requires_login(self): + # Credentials come from GRAMPS_WEB_API_KEY, not a login dialog. + return False + + def _initialize(self, directory, username, password): + try: + self.web_client = WebApiHandler.from_env() + except _CONNECTION_ERRORS as err: + raise DbConnectionError(str(err), directory) from err + + # Local mirror: reuse SQLite's own _initialize for the on-disk + # cache file, then sync from the server on load(). + super()._initialize(directory, username, password) + + def load(self, *args, **kwargs): + super().load(*args, **kwargs) + self._sync_from_server() + + def transaction_commit(self, transaction): + # Must run before super(): it clears the transaction's records. + payload = transaction_to_json(transaction) + super().transaction_commit(transaction) + if payload: + try: + self.web_client.push_transaction(payload) + except _CONNECTION_ERRORS: + LOG.exception( + "Failed to push %d local change(s) to the server; " + "local mirror has drifted from the server until the " + "next successful push or read sync.", + len(payload), + ) + + def _sync_from_server(self): + """ + Pull every transaction after the last-seen timestamp and replay + its changes into the local mirror. Returns the number of changes + applied. + """ + after = self._get_metadata("sync_last_time", default=0) + applied = 0 + page = 1 + while True: + transactions, _total = self.web_client.get_transaction_history( + after=after, page=page, pagesize=SYNC_PAGE_SIZE + ) + if not transactions: + break + with DbTxn("Sync from server", self, batch=True) as trans: + for server_trans in transactions: + for change in server_trans["changes"]: + if self._apply_change(change, trans): + applied += 1 + after = max(after, server_trans["timestamp"]) + if len(transactions) < SYNC_PAGE_SIZE: + break + page += 1 + self._set_metadata("sync_last_time", after) + return applied + + def _apply_change(self, change, trans): + """Replay one server change into the local mirror. Returns True + if it was a recognized primary-object change (as opposed to a + reference-type change, which carries no obj_class we can map).""" + obj_class = change["obj_class"] + key = CLASS_TO_KEY_MAP.get(obj_class) + if key is None: + return False + name = KEY_TO_NAME_MAP[key] + handle = change["obj_handle"] + if change["trans_type"] == TXNDEL: + getattr(self, f"remove_{name}")(handle, trans) + else: + # add and update are both upserts at the DBAPI level, so + # there's no need to treat them differently here. + obj = data_to_object(change["new_data"]) + getattr(self, f"commit_{name}")(obj, trans) + return True diff --git a/GrampsWebApiDb/po/template.pot b/GrampsWebApiDb/po/template.pot new file mode 100644 index 000000000..07e83bed0 --- /dev/null +++ b/GrampsWebApiDb/po/template.pot @@ -0,0 +1,32 @@ +# SOME DESCRIPTIVE TITLE. +# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER +# This file is distributed under the same license as the PACKAGE package. +# FIRST AUTHOR , YEAR. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2026-08-04 14:40-0700\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language-Team: LANGUAGE \n" +"Language: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" + +#: GrampsWebApiDb/grampswebapidb.gpr.py:24 +msgid "GrampsWebApiDb" +msgstr "" + +#: GrampsWebApiDb/grampswebapidb.gpr.py:25 +msgid "Gramps _Web API Database" +msgstr "" + +#: GrampsWebApiDb/grampswebapidb.gpr.py:27 +msgid "" +"Use a Gramps Web API server (e.g. gramps-connect or Gramps Web) as a live " +"database, mirrored locally for speed." +msgstr "" diff --git a/GrampsWebApiDb/webapi_client.py b/GrampsWebApiDb/webapi_client.py new file mode 100644 index 000000000..6d60302b0 --- /dev/null +++ b/GrampsWebApiDb/webapi_client.py @@ -0,0 +1,392 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2021-2024 David Straub +# Copyright (C) 2026 Douglas S. Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Minimal Gramps Web API client: authentication and read access. + +Trimmed from the WebApiHandler class in the GrampsWebSync addon (same +repo, same license) -- credit to David Straub for the original token +fetch/refresh and SSL-context handling. Dropped everything specific to +GrampsWebSync's push-a-local-transaction / XML-export / media-file-sync +job, since WebApiDB only needs auth plus reading the transaction-history +feed for now. Re-add pieces here (rather than importing GrampsWebSync +directly) so this addon has no runtime dependency on another addon being +installed. + +Credentials +----------- +Two ways in: username+password (POST /token/, matches GrampsWebSync), or +a GRAMPS_WEB_API_KEY-shaped string: "*". + +The REFRESH_TOKEN half is a JWT *refresh* token obtained once via +POST /token/ with include_refresh (gramps-web-api's JWT_REFRESH_TOKEN_EXPIRES +is False by default, so it doesn't expire on its own). From then on, +POST /token/refresh/ trades it for fresh short-lived access tokens -- +no username/password re-entry, no server-side change needed. This is +*not* the same as a real scoped/revocable personal access token +(gramps-web-api has that machinery too, but today it's hardcoded to a +single "anniversaries_ics" scope and isn't wired into general request +auth) -- it's a shortcut that works today at the cost of not being +independently revocable. '*' is a safe delimiter here: neither a JWT +(base64url segments joined by '.') nor base64url output ever contains it. +""" + +from __future__ import annotations + +import base64 +import json +import logging +import os +import platform +import socket +import time +from tempfile import NamedTemporaryFile +from time import sleep +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +LOG = logging.getLogger("grampswebapidb") + +#: Environment variable read by WebApiHandler.from_env(). +API_KEY_ENV_VAR = "GRAMPS_WEB_API_KEY" + +#: Seconds before a request that has produced nothing is abandoned. Without +#: this, ``urlopen`` waits forever and an unreachable-but-listening server +#: hangs Gramps with no way out. +TIMEOUT = 60 + +#: gramps-web-api rate-limits /token/ and /token/refresh/ to 1/second (no +#: Retry-After header is sent on 429); this is how long to back off before +#: the one retry attempt. Found by live testing: minting a key and then +#: immediately constructing another WebApiHandler in the same second +#: reliably 429s otherwise. +RATE_LIMIT_BACKOFF = 1.1 + + +def create_macos_ssl_context(): + """Create an SSL context using macOS system certificates.""" + import ssl + import subprocess + + ctx = ssl.create_default_context() + macos_ca_certs = subprocess.run( + [ + "security", + "find-certificate", + "-a", + "-p", + "/System/Library/Keychains/SystemRootCertificates.keychain", + ], + stdout=subprocess.PIPE, + ).stdout + + with NamedTemporaryFile("w+b") as tmp_file: + tmp_file.write(macos_ca_certs) + ctx.load_verify_locations(tmp_file.name) + + return ctx + + +def decode_jwt_payload(jwt: str) -> dict[str, Any]: + """Decode and return the payload from a JWT.""" + payload_part = jwt.split(".")[1] + padding = len(payload_part) % 4 + if padding > 0: + payload_part += "=" * (4 - padding) + decoded_bytes = base64.urlsafe_b64decode(payload_part) + decoded_str = decoded_bytes.decode("utf-8") + return json.loads(decoded_str) + + +def parse_api_key(api_key: str) -> tuple[str, str]: + """Split a GRAMPS_WEB_API_KEY value into ``(refresh_token, url)``.""" + try: + token, encoded_url = api_key.split("*", 1) + except ValueError as exc: + raise ValueError( + "Malformed GRAMPS_WEB_API_KEY: expected '*'" + ) from exc + padding = "=" * (-len(encoded_url) % 4) + try: + url = base64.urlsafe_b64decode(encoded_url + padding).decode("utf-8") + except (ValueError, UnicodeDecodeError) as exc: + raise ValueError("Malformed GRAMPS_WEB_API_KEY: bad URL encoding") from exc + if not token or not url: + raise ValueError("Malformed GRAMPS_WEB_API_KEY: empty token or URL") + return token, url + + +def make_api_key(refresh_token: str, url: str) -> str: + """Build a GRAMPS_WEB_API_KEY value from a refresh token and URL.""" + encoded_url = base64.urlsafe_b64encode(url.encode("utf-8")).decode("ascii") + return f"{refresh_token}*{encoded_url.rstrip('=')}" + + +class WebApiHandler: + """Web API connection handler: token auth plus authenticated GET.""" + + def __init__( + self, + url: str, + username: str | None = None, + password: str | None = None, + refresh_token: str | None = None, + ) -> None: + """ + Initialize given a server URL, plus either a username+password or + a non-expiring refresh token (exactly one of the two is expected). + """ + self.url = url.rstrip("/") + self.username = username + self.password = password + self._refresh_token = refresh_token + self._access_token: str | None = None + self._ctx = ( + create_macos_ssl_context() if platform.system() == "Darwin" else None + ) + self._authenticate() + + @classmethod + def from_api_key(cls, api_key: str) -> "WebApiHandler": + """Build a handler from a GRAMPS_WEB_API_KEY-shaped string.""" + token, url = parse_api_key(api_key) + return cls(url, refresh_token=token) + + @classmethod + def from_env(cls, env_var: str = API_KEY_ENV_VAR) -> "WebApiHandler": + """ + Build a handler from an environment variable holding a + GRAMPS_WEB_API_KEY-shaped string. This is the SDK entry point: + ``client = WebApiHandler.from_env()``. + """ + api_key = os.environ.get(env_var) + if not api_key: + raise ValueError(f"{env_var} is not set") + return cls.from_api_key(api_key) + + @classmethod + def mint_api_key(cls, url: str, username: str, password: str) -> str: + """ + One-time username+password login that returns a GRAMPS_WEB_API_KEY + value for all future non-interactive use. This is the client-side + half of what a future "Generate SDK Key" UI button would automate + server-side; until that exists, this is how a key gets created at + all. + """ + handler = cls(url, username=username, password=password) + if not handler._refresh_token: + raise ValueError("Server did not return a refresh token") + return make_api_key(handler._refresh_token, handler.url) + + def _open(self, req: Request): + """Open ``req`` with this handler's SSL context and timeout.""" + return urlopen(req, context=self._ctx, timeout=TIMEOUT) + + @property + def access_token(self) -> str: + """Get the access token. Cached after first call unless refresh needed.""" + if not self._access_token: + self._authenticate() + remaining_time = self.get_access_token_remaining_time() + if remaining_time is not None and remaining_time < 60: + self._authenticate() + assert self._access_token # for type checker + return self._access_token + + def get_access_token_remaining_time(self) -> int | None: + """Get the remaining time of the access token in seconds.""" + if self._access_token is None: + return None + payload = decode_jwt_payload(self._access_token) + if "exp" not in payload: + return None + expires = payload["exp"] + now = time.time() + return int(expires - now) + + def _authenticate(self) -> None: + """Get a fresh access token, via whichever credential we hold.""" + if self._refresh_token: + self._refresh_access_token() + else: + self.fetch_token() + + def fetch_token(self, retry_on_rate_limit: bool = True) -> None: + """Fetch and store an access token via username+password.""" + LOG.debug("Fetching an access token from the server") + data = json.dumps({"username": self.username, "password": self.password}) + req = Request( + f"{self.url}/token/", + data=data.encode(), + headers={"Content-Type": "application/json", "User-Agent": "GrampsWebApiDb"}, + ) + try: + with self._open(req) as res: + res_json = json.load(res) + except HTTPError as exc: + if exc.code == 429 and retry_on_rate_limit: + sleep(RATE_LIMIT_BACKOFF) + return self.fetch_token(retry_on_rate_limit=False) + if "/api" not in self.url: + self.url = f"{self.url}/api" + return self.fetch_token(retry_on_rate_limit=retry_on_rate_limit) + raise + except (UnicodeDecodeError, json.JSONDecodeError): + if "/api" not in self.url: + self.url = f"{self.url}/api" + return self.fetch_token(retry_on_rate_limit=retry_on_rate_limit) + raise + self._access_token = res_json["access_token"] + # /token/ with username+password always includes a refresh token + # (TokenResource.post() calls get_tokens(..., include_refresh=True)). + if "refresh_token" in res_json: + self._refresh_token = res_json["refresh_token"] + + def _refresh_access_token(self, retry_on_rate_limit: bool = True) -> None: + """Trade the stored refresh token for a new access token.""" + LOG.debug("Refreshing access token from stored refresh token") + req = Request( + f"{self.url}/token/refresh/", + method="POST", + headers={ + "Authorization": f"Bearer {self._refresh_token}", + "User-Agent": "GrampsWebApiDb", + }, + ) + try: + with self._open(req) as res: + res_json = json.load(res) + except HTTPError as exc: + if exc.code == 429 and retry_on_rate_limit: + sleep(RATE_LIMIT_BACKOFF) + return self._refresh_access_token(retry_on_rate_limit=False) + if "/api" not in self.url: + self.url = f"{self.url}/api" + return self._refresh_access_token(retry_on_rate_limit=retry_on_rate_limit) + raise + except (UnicodeDecodeError, json.JSONDecodeError): + if "/api" not in self.url: + self.url = f"{self.url}/api" + return self._refresh_access_token(retry_on_rate_limit=retry_on_rate_limit) + raise + self._access_token = res_json["access_token"] + + def get_permissions(self) -> set[str]: + """Get the permissions of the current user.""" + return decode_jwt_payload(self.access_token).get("permissions", set()) + + def _get_json(self, url: str, retry: bool = True) -> tuple[Any, dict]: + """GET ``url`` with the bearer token and return ``(body, headers)``.""" + req = Request( + url, + headers={ + "Authorization": f"Bearer {self.access_token}", + "User-Agent": "GrampsWebApiDb", + }, + ) + try: + with self._open(req) as res: + return json.load(res), dict(res.headers) + except HTTPError as exc: + if exc.code == 401 and retry: + # in case of 401, retry once with a new token + sleep(RATE_LIMIT_BACKOFF) # avoid immediately re-tripping the rate limit + self._authenticate() + return self._get_json(url, retry=False) + if exc.code == 429 and retry: + sleep(RATE_LIMIT_BACKOFF) + return self._get_json(url, retry=False) + raise + except (URLError, socket.timeout): + if retry: + sleep(1) + return self._get_json(url, retry=False) + raise + + def get_transaction_history( + self, after: float = 0, page: int = 1, pagesize: int = 100 + ) -> tuple[list[dict[str, Any]], int]: + """ + Fetch one page of the server's transaction history committed + after ``after`` (a Unix timestamp), ascending by transaction id, + including the post-change raw object data. + + :returns: ``(transactions, total_count)``. ``total_count`` comes + from the ``X-Total-Count`` response header, so the caller can + tell whether more pages remain. + """ + params = { + "after": after, + "new": "1", + "sort": "id", + "page": page, + "pagesize": pagesize, + } + url = f"{self.url}/transactions/history/?{urlencode(params)}" + body, headers = self._get_json(url) + total_count = int(headers.get("X-Total-Count", len(body))) + return body, total_count + + def push_transaction( + self, payload: list[dict[str, Any]], retry: bool = True + ) -> None: + """ + POST a batch of local changes to /transactions/. Uses force=1: + without it, the server compares each item's "old" snapshot + against its own current data and rejects the whole batch on any + mismatch (POST /transactions/?force=... semantics, see + gramps_webapi/api/tasks.py's process_transactions -> old_unchanged + check) -- real conflict detection/resolution is out of scope for + now (see grampswebapidb.py), so this is last-write-wins by design, + not an oversight. + """ + if not payload: + return + data = json.dumps(payload).encode() + req = Request( + f"{self.url}/transactions/?force=1", + data=data, + method="POST", + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {self.access_token}", + "User-Agent": "GrampsWebApiDb", + }, + ) + try: + with self._open(req) as res: + res.read() + except HTTPError as exc: + if exc.code == 401 and retry: + sleep(RATE_LIMIT_BACKOFF) + self._authenticate() + return self.push_transaction(payload, retry=False) + if exc.code == 429 and retry: + sleep(RATE_LIMIT_BACKOFF) + return self.push_transaction(payload, retry=False) + raise + except (URLError, socket.timeout): + if retry: + sleep(RATE_LIMIT_BACKOFF) + return self.push_transaction(payload, retry=False) + raise From 22d8429d5b7f55030c3955781997398e38c26866 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 15:28:15 -0700 Subject: [PATCH 02/11] GrampsWebApiDb: add README documenting the refresh-token credential tradeoff Explains that GRAMPS_WEB_API_KEY carries a standard, non-expiring refresh token from the server's normal login flow rather than a scoped/revocable personal access token, so a leaked key is as damaging as a leaked password until it's changed. Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/README.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 GrampsWebApiDb/README.md diff --git a/GrampsWebApiDb/README.md b/GrampsWebApiDb/README.md new file mode 100644 index 000000000..9989425b5 --- /dev/null +++ b/GrampsWebApiDb/README.md @@ -0,0 +1,38 @@ +GrampsWebApiDb is a Gramps database backend that uses a Gramps Web API +server (e.g. gramps-connect or Gramps Web) as a live database, mirrored +locally in SQLite for speed. Reads are served from the local mirror, which +is kept current via the server's transaction-history feed; local edits are +pushed back to the server as they're committed. + +## Credentials + +The addon takes a single credential, via the `GRAMPS_WEB_API_KEY` +environment variable, shaped `*`. There is +deliberately no login dialog and no per-tree settings.ini. Use +`WebApiHandler.mint_api_key(url, username, password)` (see +`webapi_client.py`) once to turn a username/password into this key. + +**Security tradeoff:** the token embedded in `GRAMPS_WEB_API_KEY` is a +standard JWT *refresh* token obtained from the server's normal `/token/` +login endpoint — the same endpoint and flow the official web client uses, +not an undocumented or exploited access path. gramps-web-api leaves refresh +tokens non-expiring by default, so this key is a long-lived, general-purpose +credential carrying the full permissions of the account that minted it. It +is *not* the same as a real scoped, independently revocable personal access +token (gramps-web-api has that machinery, but it isn't generally wired into +request auth yet). Practically, that means: + +* A leaked `GRAMPS_WEB_API_KEY` is as damaging as a leaked password — it + grants full account access until the underlying password is changed. + There is no "revoke this key" action independent of that. +* Treat it accordingly: don't commit it, don't log it, and store it the + same way you'd store a password. + +This is a documented engineering tradeoff, made because the properly-scoped +alternative isn't available server-side today — not a vulnerability in +gramps-web-api or a loophole being exploited. + +## See also + +* `grampswebapidb.py` for the sync/write-through design (module docstring). +* `webapi_client.py` for the token fetch/refresh implementation. From 8a329dc35bb1921ab7170b1890dacce041d665db Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 15:29:25 -0700 Subject: [PATCH 03/11] note about gramps versions --- GrampsWebApiDb/grampswebapidb.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/GrampsWebApiDb/grampswebapidb.py b/GrampsWebApiDb/grampswebapidb.py index 3dd29d9c5..4f570bf9c 100644 --- a/GrampsWebApiDb/grampswebapidb.py +++ b/GrampsWebApiDb/grampswebapidb.py @@ -46,6 +46,14 @@ both being upserts, no need to distinguish) or remove_() for a delete. +This "_class"-tagged new_data shape is only produced by gramps-web-api +servers running against Gramps >= 6.0; a server still on Gramps 5.2 (e.g. +gramps-web-api itself untouched) serializes objects differently (no +"_class"/"value"/"string" triplet on GrampsType-derived fields), and +data_to_object() raises KeyError on it. Confirmed against a live gramps52 +server: read-only endpoints (auth, /trees/, /people/ counts, etc.) work +fine, but _sync_from_server() cannot deserialize its transaction history. + Credentials come from a single environment variable, GRAMPS_WEB_API_KEY (see webapi_client.py for its "*" shape and the tradeoffs of using a refresh token here rather than a real scoped From be1450a98b612d10dd87de58f14d7602d1ecf79d Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 15:47:54 -0700 Subject: [PATCH 04/11] GrampsWebApiDb: add regression tests Cover webapi_client.WebApiHandler (token codec, JWT decoding, auth flows, 429/401 retry and API-prefix fallback, transaction-history/push request shape) and grampswebapidb.WebApiDB (transaction_to_json, _apply_change, _sync_from_server pagination, transaction_commit ordering and error handling). No real server or SQLite file is needed; urlopen and the DBAPI/SQLite base are stubbed throughout. Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/tests/__init__.py | 0 GrampsWebApiDb/tests/test_grampswebapidb.py | 397 +++++++++++++++ GrampsWebApiDb/tests/test_webapi_client.py | 531 ++++++++++++++++++++ 3 files changed, 928 insertions(+) create mode 100644 GrampsWebApiDb/tests/__init__.py create mode 100644 GrampsWebApiDb/tests/test_grampswebapidb.py create mode 100644 GrampsWebApiDb/tests/test_webapi_client.py diff --git a/GrampsWebApiDb/tests/__init__.py b/GrampsWebApiDb/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/GrampsWebApiDb/tests/test_grampswebapidb.py b/GrampsWebApiDb/tests/test_grampswebapidb.py new file mode 100644 index 000000000..185601530 --- /dev/null +++ b/GrampsWebApiDb/tests/test_grampswebapidb.py @@ -0,0 +1,397 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Douglas S. Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Unit tests for grampswebapidb.WebApiDB: the sync/write-through logic. + +WebApiDB subclasses the stock SQLite DBAPI backend, but these tests never +open a real database file -- SQLite's own commit_*/remove_* methods and +transaction machinery are stubbed out (WebApiDB.__new__() plus per-test +attribute overrides), the same pattern SharedPostgreSQL/tests/ +test_initialize.py uses to test _create_settings() without a real Postgres +connection. This isolates exactly the logic this addon adds: + + - transaction_to_json(): local DbTxn -> flat change-list payload + - _apply_change(): one server change -> a commit_*/remove_* call + - _sync_from_server(): pagination + sync_last_time bookkeeping + - transaction_commit(): push-after-commit, ordering, and error swallowing + +Run with:: + + python3 -m unittest GrampsWebApiDb.tests.test_grampswebapidb -v +""" + +# ------------------------------------------------------------------------- +# +# Standard python modules +# +# ------------------------------------------------------------------------- +import os +import sys +import unittest +from urllib.error import HTTPError +from unittest import mock + +# ------------------------------------------------------------------------- +# +# Make the addon importable the way Gramps loads it: its own directory on +# sys.path (grampswebapidb.py does a bare ``from webapi_client import +# WebApiHandler`` -- see CLAUDE.md Testing conventions). +# +# ------------------------------------------------------------------------- +ADDON_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if ADDON_DIR not in sys.path: + sys.path.insert(0, ADDON_DIR) + +try: + import gramps +except ImportError as _err: + raise unittest.SkipTest("gramps package not available: %s" % _err) + +from gramps.gen.db.dbconst import REFERENCE_KEY, TXNADD, TXNDEL, TXNUPD +from gramps.gen.db.exceptions import DbConnectionError +from gramps.gen.lib import Person +from gramps.gen.lib.json_utils import object_to_data, remove_object + +from GrampsWebApiDb import grampswebapidb +from GrampsWebApiDb.grampswebapidb import WebApiDB, transaction_to_json + + +# ------------------------------------------------------------------------- +# +# Test helpers +# +# ------------------------------------------------------------------------- +def person_data(handle="H1", gramps_id="I0001"): + """A real Person's data-dict, the shape new_data/old_data actually take.""" + person = Person() + person.set_handle(handle) + person.set_gramps_id(gramps_id) + return object_to_data(person) + + +class FakeTransaction: + """Duck-types the bit of DbTxn that transaction_to_json() reads: + get_recnos()/get_record(). Avoids needing a real commitdb/pickle round + trip just to test the flattening logic.""" + + def __init__(self, records): + # records: list of (key, action, handle, old_data, new_data) + self._records = records + + def get_recnos(self, reverse=False): + idx = range(len(self._records)) + return reversed(idx) if reverse else idx + + def get_record(self, recno): + return self._records[recno] + + +def new_instance(): + """A WebApiDB that never touched a real SQLite file or server.""" + return WebApiDB.__new__(WebApiDB) + + +# ------------------------------------------------------------------------- +# +# TestTransactionToJson +# +# ------------------------------------------------------------------------- +class TestTransactionToJson(unittest.TestCase): + def test_add_record_shape(self): + new_data = person_data() + trans = FakeTransaction([(0, TXNADD, "H1", None, new_data)]) # PERSON_KEY + out = transaction_to_json(trans) + self.assertEqual(len(out), 1) + entry = out[0] + self.assertEqual(entry["type"], "add") + self.assertEqual(entry["handle"], "H1") + self.assertEqual(entry["_class"], "Person") + self.assertIsNone(entry["old"]) + self.assertNotIn("_object", entry["new"]) + + def test_update_record_carries_old_and_new(self): + old_data = person_data(gramps_id="I0001") + new_data = person_data(gramps_id="I0002") + trans = FakeTransaction([(0, TXNUPD, "H1", old_data, new_data)]) + out = transaction_to_json(trans) + self.assertEqual(out[0]["type"], "update") + self.assertNotIn("_object", out[0]["old"]) + self.assertNotIn("_object", out[0]["new"]) + + def test_delete_record_has_no_new_data(self): + old_data = person_data() + trans = FakeTransaction([(0, TXNDEL, "H1", old_data, None)]) + out = transaction_to_json(trans) + self.assertEqual(out[0]["type"], "delete") + self.assertIsNone(out[0]["new"]) + self.assertIsNotNone(out[0]["old"]) + + def test_reference_type_record_is_skipped(self): + # REFERENCE_KEY has no entry in KEY_TO_CLASS_MAP -- see dbconst.py. + trans = FakeTransaction([(REFERENCE_KEY, TXNADD, "H1", None, {})]) + self.assertEqual(transaction_to_json(trans), []) + + def test_multiple_records_preserve_order(self): + trans = FakeTransaction( + [ + (0, TXNADD, "H1", None, person_data("H1")), + (0, TXNUPD, "H2", person_data("H2"), person_data("H2", "I0002")), + ] + ) + out = transaction_to_json(trans) + self.assertEqual([e["handle"] for e in out], ["H1", "H2"]) + + +# ------------------------------------------------------------------------- +# +# TestApplyChange +# +# ------------------------------------------------------------------------- +class TestApplyChange(unittest.TestCase): + def setUp(self): + self.db = new_instance() + self.db.commit_person = mock.MagicMock() + self.db.remove_person = mock.MagicMock() + self.trans = object() # opaque; just forwarded + + def test_unrecognized_obj_class_is_ignored(self): + change = {"obj_class": "NotAThing", "trans_type": TXNADD, "obj_handle": "H1"} + applied = self.db._apply_change(change, self.trans) + self.assertFalse(applied) + self.db.commit_person.assert_not_called() + self.db.remove_person.assert_not_called() + + def test_delete_calls_remove(self): + change = {"obj_class": "Person", "trans_type": TXNDEL, "obj_handle": "H1"} + applied = self.db._apply_change(change, self.trans) + self.assertTrue(applied) + self.db.remove_person.assert_called_once_with("H1", self.trans) + self.db.commit_person.assert_not_called() + + def test_add_calls_commit_with_reconstructed_object(self): + new_data = remove_object(person_data("H1", "I0001")) + change = { + "obj_class": "Person", + "trans_type": TXNADD, + "obj_handle": "H1", + "new_data": new_data, + } + applied = self.db._apply_change(change, self.trans) + self.assertTrue(applied) + self.db.commit_person.assert_called_once() + obj, trans = self.db.commit_person.call_args[0] + self.assertIsInstance(obj, Person) + self.assertEqual(obj.get_handle(), "H1") + self.assertIs(trans, self.trans) + + def test_update_is_also_an_upsert(self): + new_data = remove_object(person_data("H1", "I0002")) + change = { + "obj_class": "Person", + "trans_type": TXNUPD, + "obj_handle": "H1", + "new_data": new_data, + } + applied = self.db._apply_change(change, self.trans) + self.assertTrue(applied) + self.db.commit_person.assert_called_once() + self.db.remove_person.assert_not_called() + + +# ------------------------------------------------------------------------- +# +# TestSyncFromServer +# +# ------------------------------------------------------------------------- +class FakeDbTxn: + """Stand-in for gramps.gen.db.DbTxn: a plain context manager, so + _sync_from_server's pagination/bookkeeping can be tested without a + real transaction_begin/transaction_commit or get_undodb().""" + + def __init__(self, msg, grampsdb, batch=False): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + +class TestSyncFromServer(unittest.TestCase): + def setUp(self): + self.db = new_instance() + self.db.web_client = mock.MagicMock() + self.metadata = {} + self.db._get_metadata = lambda key, default=0: self.metadata.get( + key, default + ) + self.db._set_metadata = lambda key, value, use_txn=True: self.metadata.__setitem__( + key, value + ) + self.patcher = mock.patch.object(grampswebapidb, "DbTxn", FakeDbTxn) + self.patcher.start() + self.addCleanup(self.patcher.stop) + + def test_stops_after_short_page(self): + change = {"obj_class": "Person", "trans_type": TXNADD, "obj_handle": "H1"} + self.db.web_client.get_transaction_history.return_value = ( + [{"timestamp": 5.0, "changes": [change]}], + 1, + ) + with mock.patch.object(self.db, "_apply_change", return_value=True) as apply: + applied = self.db._sync_from_server() + self.assertEqual(applied, 1) + apply.assert_called_once_with(change, mock.ANY) + self.db.web_client.get_transaction_history.assert_called_once() + + def test_pagination_continues_on_full_page(self): + full_page = [ + {"timestamp": float(i), "changes": []} + for i in range(grampswebapidb.SYNC_PAGE_SIZE) + ] + short_page = [{"timestamp": 999.0, "changes": []}] + self.db.web_client.get_transaction_history.side_effect = [ + (full_page, len(full_page) + 1), + (short_page, 1), + ] + applied = self.db._sync_from_server() + self.assertEqual(applied, 0) + self.assertEqual(self.db.web_client.get_transaction_history.call_count, 2) + calls = self.db.web_client.get_transaction_history.call_args_list + self.assertEqual(calls[0].kwargs["page"], 1) + self.assertEqual(calls[1].kwargs["page"], 2) + + def test_no_transactions_leaves_sync_time_unchanged(self): + self.metadata["sync_last_time"] = 42.0 + self.db.web_client.get_transaction_history.return_value = ([], 0) + applied = self.db._sync_from_server() + self.assertEqual(applied, 0) + self.assertEqual(self.metadata["sync_last_time"], 42.0) + + def test_sync_last_time_advances_to_max_timestamp_seen(self): + page = [ + {"timestamp": 10.0, "changes": []}, + {"timestamp": 30.0, "changes": []}, + {"timestamp": 20.0, "changes": []}, + ] + self.db.web_client.get_transaction_history.return_value = (page, 3) + self.db._sync_from_server() + self.assertEqual(self.metadata["sync_last_time"], 30.0) + + def test_unrecognized_changes_are_not_counted(self): + page = [ + { + "timestamp": 1.0, + "changes": [ + {"obj_class": "Bogus", "trans_type": TXNADD, "obj_handle": "H1"} + ], + } + ] + self.db.web_client.get_transaction_history.return_value = (page, 1) + applied = self.db._sync_from_server() + self.assertEqual(applied, 0) + + +# ------------------------------------------------------------------------- +# +# TestTransactionCommit +# +# ------------------------------------------------------------------------- +class TestTransactionCommit(unittest.TestCase): + def setUp(self): + self.db = new_instance() + self.db.web_client = mock.MagicMock() + + def test_no_local_changes_does_not_push(self): + trans = FakeTransaction([]) + with mock.patch.object(grampswebapidb.SQLite, "transaction_commit"): + self.db.transaction_commit(trans) + self.db.web_client.push_transaction.assert_not_called() + + def test_local_changes_are_pushed(self): + trans = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + with mock.patch.object(grampswebapidb.SQLite, "transaction_commit"): + self.db.transaction_commit(trans) + self.db.web_client.push_transaction.assert_called_once() + payload = self.db.web_client.push_transaction.call_args[0][0] + self.assertEqual(payload[0]["handle"], "H1") + + def test_payload_built_before_super_clears_records(self): + # The base class's transaction_commit() clears the transaction's + # records as its last step -- see the module docstring's "must run + # before super()" note. Simulate that by having the (mocked) super + # call wipe the fake transaction, and confirm the push still saw + # the pre-clear data. + trans = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + + def clear_records(transaction): + transaction._records = [] + + with mock.patch.object( + grampswebapidb.SQLite, "transaction_commit", side_effect=clear_records + ): + self.db.transaction_commit(trans) + payload = self.db.web_client.push_transaction.call_args[0][0] + self.assertEqual(len(payload), 1) + + def test_push_failure_is_logged_not_raised(self): + trans = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + self.db.web_client.push_transaction.side_effect = HTTPError( + "https://example.com/api/transactions/", 500, "boom", None, None + ) + with mock.patch.object(grampswebapidb.SQLite, "transaction_commit"): + with self.assertLogs(grampswebapidb.LOG, level="ERROR"): + self.db.transaction_commit(trans) # must not raise + + +# ------------------------------------------------------------------------- +# +# TestMisc +# +# ------------------------------------------------------------------------- +class TestMisc(unittest.TestCase): + def test_requires_login_is_false(self): + self.assertFalse(new_instance().requires_login()) + + def test_initialize_wraps_connection_errors(self): + db = new_instance() + with mock.patch.object( + grampswebapidb.WebApiHandler, + "from_env", + side_effect=ValueError("GRAMPS_WEB_API_KEY is not set"), + ): + with self.assertRaises(DbConnectionError): + db._initialize("/tmp/some-tree", None, None) + + def test_initialize_stores_web_client_and_calls_super(self): + db = new_instance() + sentinel_client = mock.MagicMock() + with mock.patch.object( + grampswebapidb.WebApiHandler, "from_env", return_value=sentinel_client + ), mock.patch.object(grampswebapidb.SQLite, "_initialize") as super_init: + db._initialize("/tmp/some-tree", "user", "pw") + self.assertIs(db.web_client, sentinel_client) + super_init.assert_called_once_with("/tmp/some-tree", "user", "pw") + + +if __name__ == "__main__": + unittest.main() diff --git a/GrampsWebApiDb/tests/test_webapi_client.py b/GrampsWebApiDb/tests/test_webapi_client.py new file mode 100644 index 000000000..d42faed9f --- /dev/null +++ b/GrampsWebApiDb/tests/test_webapi_client.py @@ -0,0 +1,531 @@ +# +# Gramps - a GTK+/GNOME based genealogy program +# +# Copyright (C) 2026 Douglas S. Blank +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# + +""" +Unit tests for webapi_client.WebApiHandler and its helper functions. + +No real Gramps Web API server is contacted: urlopen() is patched throughout, +via a small FakeResponse context manager and a queue of canned +responses/exceptions. Covers: + + - the GRAMPS_WEB_API_KEY codec (make_api_key/parse_api_key round trip) + - JWT payload decoding + - username/password vs. refresh-token authentication + - the 429 rate-limit backoff-and-retry-once behavior + - the "no /api prefix yet" fallback retry + - 401 re-authentication on expired access tokens + - transaction_history/push_transaction request shape + +Run with:: + + python3 -m unittest GrampsWebApiDb.tests.test_webapi_client -v +""" + +# ------------------------------------------------------------------------- +# +# Standard python modules +# +# ------------------------------------------------------------------------- +import base64 +import json +import os +import sys +import unittest +from urllib.error import HTTPError, URLError +from unittest import mock + +# ------------------------------------------------------------------------- +# +# Make the addon importable the way Gramps loads it: its own directory on +# sys.path (grampswebapidb.py/webapi_client.py use bare, not package- +# relative, imports of each other -- see CLAUDE.md Testing conventions). +# +# ------------------------------------------------------------------------- +ADDON_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +if ADDON_DIR not in sys.path: + sys.path.insert(0, ADDON_DIR) + +try: + import gramps # noqa: F401 (only to trigger the SkipTest below if absent) +except ImportError as _err: + raise unittest.SkipTest("gramps package not available: %s" % _err) + +from GrampsWebApiDb import webapi_client +from GrampsWebApiDb.webapi_client import ( + WebApiHandler, + decode_jwt_payload, + make_api_key, + parse_api_key, +) + + +# ------------------------------------------------------------------------- +# +# Test helpers +# +# ------------------------------------------------------------------------- +def b64url_json(payload: dict) -> str: + """Base64url-encode a dict, stripped of padding, like a real JWT segment.""" + raw = json.dumps(payload).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def fake_jwt(payload: dict) -> str: + """A JWT-shaped string whose payload segment decodes to ``payload``. + + decode_jwt_payload() only ever looks at segment [1], so the header and + signature segments don't need to be real. + """ + return f"header.{b64url_json(payload)}.signature" + + +def token(tag: str) -> str: + """A distinct, real-JWT-shaped access token for ``tag``. + + Every access token this module hands back (even a canned "AT1"-style + placeholder) is real enough to satisfy decode_jwt_payload(), because + access_token's getter unconditionally checks the token's remaining + lifetime -- see get_access_token_remaining_time(). + """ + return fake_jwt({"tag": tag}) + + +class FakeResponse: + """Stand-in for the object returned by ``urlopen(...).__enter__()``.""" + + def __init__(self, body=None, headers=None): + self._body = json.dumps(body if body is not None else {}).encode() + self.headers = headers or {} + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + +def http_error(code, url="https://example.com/api"): + return HTTPError(url, code, f"HTTP {code}", None, None) + + +class QueuedUrlopen: + """``urlopen`` replacement that returns/raises each queued item in turn, + recording every ``Request`` it was called with.""" + + def __init__(self, items): + self._items = list(items) + self.requests = [] + + def __call__(self, req, context=None, timeout=None): + self.requests.append(req) + item = self._items.pop(0) + if isinstance(item, Exception): + raise item + return item + + +# ------------------------------------------------------------------------- +# +# TestApiKeyCodec +# +# ------------------------------------------------------------------------- +class TestApiKeyCodec(unittest.TestCase): + """make_api_key()/parse_api_key() are inverses, and reject malformed input.""" + + def test_roundtrip(self): + key = make_api_key("refresh-tok-123", "https://example.com/api") + self.assertEqual( + parse_api_key(key), ("refresh-tok-123", "https://example.com/api") + ) + + def test_roundtrip_with_padding_needed(self): + # A URL whose base64url encoding needs '=' padding restored. + url = "https://example.com/api/x" + key = make_api_key("tok", url) + self.assertEqual(parse_api_key(key), ("tok", url)) + + def test_missing_delimiter_is_malformed(self): + with self.assertRaises(ValueError): + parse_api_key("no-delimiter-here") + + def test_bad_url_encoding_is_malformed(self): + with self.assertRaises(ValueError): + parse_api_key("tok*not-valid-base64---") + + def test_empty_token_is_rejected(self): + encoded_url = base64.urlsafe_b64encode(b"https://example.com").decode() + with self.assertRaises(ValueError): + parse_api_key("*" + encoded_url) + + def test_empty_url_is_rejected(self): + encoded_empty = base64.urlsafe_b64encode(b"").decode() + with self.assertRaises(ValueError): + parse_api_key("tok*" + encoded_empty) + + +# ------------------------------------------------------------------------- +# +# TestDecodeJwtPayload +# +# ------------------------------------------------------------------------- +class TestDecodeJwtPayload(unittest.TestCase): + def test_decodes_payload_claims(self): + jwt_str = fake_jwt({"sub": "user1", "exp": 1234}) + self.assertEqual(decode_jwt_payload(jwt_str), {"sub": "user1", "exp": 1234}) + + def test_handles_payload_needing_padding(self): + # Pick a payload whose base64url segment length isn't a multiple of 4, + # to exercise the padding-restoration branch. + jwt_str = fake_jwt({"a": "bit-of-text-to-shift-the-length"}) + payload = decode_jwt_payload(jwt_str) + self.assertEqual(payload["a"], "bit-of-text-to-shift-the-length") + + +# ------------------------------------------------------------------------- +# +# TestAuthentication +# +# ------------------------------------------------------------------------- +class TestAuthentication(unittest.TestCase): + """Constructing a handler authenticates once, via whichever credential + was supplied.""" + + def test_username_password_login(self): + fake = QueuedUrlopen( + [FakeResponse({"access_token": token("AT1"), "refresh_token": "RT1"})] + ) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler( + "https://example.com/api", username="alice", password="secret" + ) + self.assertEqual(handler._access_token, token("AT1")) + self.assertEqual(handler._refresh_token, "RT1") + req = fake.requests[0] + self.assertEqual(req.full_url, "https://example.com/api/token/") + self.assertEqual( + json.loads(req.data), {"username": "alice", "password": "secret"} + ) + + def test_refresh_token_login(self): + fake = QueuedUrlopen([FakeResponse({"access_token": token("AT2")})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT0") + self.assertEqual(handler._access_token, token("AT2")) + # Refresh token is unchanged; the endpoint used is /token/refresh/. + self.assertEqual(handler._refresh_token, "RT0") + req = fake.requests[0] + self.assertEqual(req.full_url, "https://example.com/api/token/refresh/") + self.assertEqual(req.get_header("Authorization"), "Bearer RT0") + + def test_mint_api_key_returns_encoded_key(self): + fake = QueuedUrlopen( + [FakeResponse({"access_token": token("AT1"), "refresh_token": "RT1"})] + ) + with mock.patch.object(webapi_client, "urlopen", fake): + key = WebApiHandler.mint_api_key( + "https://example.com/api", "alice", "secret" + ) + self.assertEqual(parse_api_key(key), ("RT1", "https://example.com/api")) + + def test_mint_api_key_requires_refresh_token_in_response(self): + fake = QueuedUrlopen([FakeResponse({"access_token": token("AT1")})]) + with mock.patch.object(webapi_client, "urlopen", fake): + with self.assertRaises(ValueError): + WebApiHandler.mint_api_key("https://example.com/api", "alice", "pw") + + def test_from_env_missing_var_raises(self): + with mock.patch.dict(os.environ, {}, clear=True): + with self.assertRaises(ValueError): + WebApiHandler.from_env() + + def test_from_env_builds_handler_from_refresh_key(self): + key = make_api_key("RT9", "https://example.com/api") + fake = QueuedUrlopen([FakeResponse({"access_token": token("AT9")})]) + with mock.patch.dict(os.environ, {webapi_client.API_KEY_ENV_VAR: key}): + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler.from_env() + self.assertEqual(handler.url, "https://example.com/api") + self.assertEqual(handler._access_token, token("AT9")) + + +# ------------------------------------------------------------------------- +# +# TestAccessTokenProperty +# +# ------------------------------------------------------------------------- +class TestAccessTokenProperty(unittest.TestCase): + def _handler_with_token(self, exp_offset): + """A handler whose access token expires ``exp_offset`` seconds from now.""" + fake = QueuedUrlopen( + [FakeResponse({"access_token": fake_jwt({"exp": time_now() + exp_offset})})] + ) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + return handler, fake + + def test_remaining_time_none_without_exp_claim(self): + fake = QueuedUrlopen([FakeResponse({"access_token": fake_jwt({})})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + self.assertIsNone(handler.get_access_token_remaining_time()) + + def test_access_token_refreshes_when_near_expiry(self): + handler, fake = self._handler_with_token(exp_offset=30) # < 60s left + fake._items.append(FakeResponse({"access_token": token("FRESH")})) + with mock.patch.object(webapi_client, "urlopen", fake): + refreshed = handler.access_token + self.assertEqual(refreshed, token("FRESH")) + self.assertEqual(len(fake.requests), 2) # initial auth + re-auth + + def test_access_token_reused_when_far_from_expiry(self): + handler, fake = self._handler_with_token(exp_offset=3600) + access_token = handler.access_token + self.assertEqual(len(fake.requests), 1) # no re-auth triggered + self.assertTrue(access_token) + + def test_get_permissions_reads_token_claim(self): + fake = QueuedUrlopen( + [ + FakeResponse( + {"access_token": fake_jwt({"permissions": ["edit", "view"]})} + ) + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + self.assertEqual(handler.get_permissions(), ["edit", "view"]) + + +def time_now(): + import time + + return time.time() + + +# ------------------------------------------------------------------------- +# +# TestRateLimitAndFallbackRetries +# +# ------------------------------------------------------------------------- +class TestRateLimitAndFallbackRetries(unittest.TestCase): + """429 responses back off and retry once; a URL missing '/api' is + retried with it appended.""" + + def test_fetch_token_retries_once_after_429(self): + fake = QueuedUrlopen( + [ + http_error(429), + FakeResponse({"access_token": token("AT"), "refresh_token": "RT"}), + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ) as mock_sleep: + handler = WebApiHandler( + "https://example.com/api", username="alice", password="pw" + ) + self.assertEqual(handler._access_token, token("AT")) + mock_sleep.assert_called_once_with(webapi_client.RATE_LIMIT_BACKOFF) + + def test_refresh_retries_once_after_429(self): + fake = QueuedUrlopen([http_error(429), FakeResponse({"access_token": token("AT")})]) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + self.assertEqual(handler._access_token, token("AT")) + + def test_fetch_token_appends_api_prefix_on_non_rate_limit_error(self): + fake = QueuedUrlopen( + [ + http_error(404, url="https://example.com/token/"), + FakeResponse({"access_token": token("AT"), "refresh_token": "RT"}), + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler( + "https://example.com", username="alice", password="pw" + ) + self.assertEqual(handler.url, "https://example.com/api") + self.assertEqual(fake.requests[1].full_url, "https://example.com/api/token/") + + def test_fetch_token_does_not_re_append_api_prefix(self): + # If the URL already ends in /api, a second failure must propagate + # rather than looping. + fake = QueuedUrlopen([http_error(404), http_error(404)]) + with mock.patch.object(webapi_client, "urlopen", fake): + with self.assertRaises(HTTPError): + WebApiHandler("https://example.com/api", username="a", password="p") + + +# ------------------------------------------------------------------------- +# +# TestGetJsonRetries +# +# ------------------------------------------------------------------------- +class TestGetJsonRetries(unittest.TestCase): + """_get_json() re-authenticates on 401, backs off on 429, and retries + once on a transient network error.""" + + def _authed_handler(self): + fake = QueuedUrlopen([FakeResponse({"access_token": token("AT0")})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + return handler + + def test_401_triggers_reauth_and_one_retry(self): + handler = self._authed_handler() + fake = QueuedUrlopen( + [ + http_error(401), + FakeResponse({"access_token": token("AT1")}), # the re-auth call + FakeResponse({"ok": True}), + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + body, _headers = handler._get_json("https://example.com/api/thing/") + self.assertEqual(body, {"ok": True}) + self.assertEqual(handler._access_token, token("AT1")) + + def test_429_retries_once(self): + handler = self._authed_handler() + fake = QueuedUrlopen([http_error(429), FakeResponse({"ok": True})]) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + body, _headers = handler._get_json("https://example.com/api/thing/") + self.assertEqual(body, {"ok": True}) + + def test_network_error_retries_once(self): + handler = self._authed_handler() + fake = QueuedUrlopen( + [URLError("connection refused"), FakeResponse({"ok": True})] + ) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + body, _headers = handler._get_json("https://example.com/api/thing/") + self.assertEqual(body, {"ok": True}) + + def test_second_failure_propagates(self): + handler = self._authed_handler() + fake = QueuedUrlopen([http_error(500), http_error(500)]) + with mock.patch.object(webapi_client, "urlopen", fake): + with self.assertRaises(HTTPError): + handler._get_json("https://example.com/api/thing/") + + +# ------------------------------------------------------------------------- +# +# TestTransactionHistory +# +# ------------------------------------------------------------------------- +class TestTransactionHistory(unittest.TestCase): + def _authed_handler(self): + fake = QueuedUrlopen([FakeResponse({"access_token": token("AT0")})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + return handler + + def test_request_shape_and_total_count_header(self): + handler = self._authed_handler() + body = [{"id": 1, "timestamp": 10.0, "changes": []}] + fake = QueuedUrlopen([FakeResponse(body, headers={"X-Total-Count": "5"})]) + with mock.patch.object(webapi_client, "urlopen", fake): + transactions, total = handler.get_transaction_history( + after=100, page=2, pagesize=50 + ) + self.assertEqual(transactions, body) + self.assertEqual(total, 5) + url = fake.requests[0].full_url + self.assertIn("after=100", url) + self.assertIn("new=1", url) + self.assertIn("sort=id", url) + self.assertIn("page=2", url) + self.assertIn("pagesize=50", url) + + def test_total_count_falls_back_to_body_length(self): + handler = self._authed_handler() + body = [{"id": 1, "timestamp": 1.0, "changes": []}] * 3 + fake = QueuedUrlopen([FakeResponse(body)]) # no X-Total-Count header + with mock.patch.object(webapi_client, "urlopen", fake): + _transactions, total = handler.get_transaction_history() + self.assertEqual(total, 3) + + +# ------------------------------------------------------------------------- +# +# TestPushTransaction +# +# ------------------------------------------------------------------------- +class TestPushTransaction(unittest.TestCase): + def _authed_handler(self): + fake = QueuedUrlopen([FakeResponse({"access_token": token("AT0")})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + return handler + + def test_empty_payload_sends_no_request(self): + handler = self._authed_handler() + fake = QueuedUrlopen([]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler.push_transaction([]) + self.assertEqual(fake.requests, []) + + def test_non_empty_payload_posts_with_force(self): + handler = self._authed_handler() + payload = [{"type": "add", "handle": "H1", "_class": "Person"}] + fake = QueuedUrlopen([FakeResponse({})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler.push_transaction(payload) + req = fake.requests[0] + self.assertEqual(req.full_url, "https://example.com/api/transactions/?force=1") + self.assertEqual(req.get_method(), "POST") + self.assertEqual(json.loads(req.data), payload) + self.assertEqual(req.get_header("Authorization"), f"Bearer {token('AT0')}") + + def test_401_triggers_reauth_and_retry(self): + handler = self._authed_handler() + fake = QueuedUrlopen( + [http_error(401), FakeResponse({"access_token": token("AT1")}), FakeResponse({})] + ) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + handler.push_transaction([{"type": "add"}]) + self.assertEqual(handler._access_token, token("AT1")) + + def test_429_retries_once(self): + handler = self._authed_handler() + fake = QueuedUrlopen([http_error(429), FakeResponse({})]) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + handler.push_transaction([{"type": "add"}]) + self.assertEqual(len(fake.requests), 2) + + +if __name__ == "__main__": + unittest.main() From c2093914f43ec872dc715096db8fb1e3c0e63584 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 17:28:01 -0700 Subject: [PATCH 05/11] GrampsWebApiDb: detect push conflicts instead of silent last-write-wins Drop force=1 from POST /transactions/ so the server's old-data-mismatch check actually runs. A rejected push now raises WebApiPushConflict (webapi_client.py), which transaction_commit() catches separately from generic connection errors: it logs a distinct warning and resyncs from the server so the local mirror stops showing an edit the server never accepted, rather than drifting silently. webapi_client.py's docstring also now notes it's a hand-synced vendored copy of the standalone gramps-web-api-client package. Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/grampswebapidb.py | 55 +++++++++++------ GrampsWebApiDb/tests/test_grampswebapidb.py | 45 +++++++++++++- GrampsWebApiDb/tests/test_webapi_client.py | 55 ++++++++++++++++- GrampsWebApiDb/webapi_client.py | 65 ++++++++++++++++++--- 4 files changed, 191 insertions(+), 29 deletions(-) diff --git a/GrampsWebApiDb/grampswebapidb.py b/GrampsWebApiDb/grampswebapidb.py index 4f570bf9c..6230450d6 100644 --- a/GrampsWebApiDb/grampswebapidb.py +++ b/GrampsWebApiDb/grampswebapidb.py @@ -81,14 +81,22 @@ No separate "am I currently syncing" flag is needed to stop synced changes from being echoed straight back to the server. -Conflict handling is not implemented: pushes go out with force=1, which -skips the server's old-data-matches check entirely (see -gramps_webapi/api/tasks.py's process_transactions), so this is -last-write-wins by design. If the push itself fails (network error, -non-conflict validation error), the local commit has already happened -and is not rolled back -- the local mirror just drifts from the server -until the next successful push or read sync. Undo/redo integration is -also still out of scope. +Pushes go out without force=1, so the server compares each item's "old" +snapshot against its own current data and rejects the whole batch with +WebApiPushConflict (see webapi_client.push_transaction()) if anything +changed server-side since the local mirror last synced -- a real, if +coarse, optimistic-concurrency check: the whole push either applies or +none of it does, with no indication of which item conflicted. On a +conflict, transaction_commit() below resyncs from the server so the +local mirror stops showing an edit the server never accepted; it does +not retry the push or attempt a merge, so the local edit is simply lost +from the server's perspective (still present, stale, in the local +mirror's edit history) -- real conflict *resolution* (merge, prompt the +user) is still out of scope. If the push fails for a non-conflict reason +(network error, auth failure), the local commit has already happened and +is not rolled back -- the local mirror just drifts from the server until +the next successful push or read sync. Undo/redo integration is also +still out of scope. """ import logging @@ -108,7 +116,7 @@ from gramps.gen.lib.json_utils import data_to_object, remove_object from gramps.plugins.db.dbapi.sqlite import SQLite -from webapi_client import WebApiHandler +from webapi_client import WebApiHandler, WebApiPushConflict _ = glocale.translation.gettext LOG = logging.getLogger("grampswebapidb") @@ -181,16 +189,29 @@ def transaction_commit(self, transaction): # Must run before super(): it clears the transaction's records. payload = transaction_to_json(transaction) super().transaction_commit(transaction) - if payload: + if not payload: + return + try: + self.web_client.push_transaction(payload) + except WebApiPushConflict: + LOG.warning( + "Server rejected %d local change(s): the object(s) changed " + "server-side since the local mirror last synced. The local " + "edit was applied to this mirror but was NOT accepted by " + "the server; resyncing the mirror from the server now.", + len(payload), + ) try: - self.web_client.push_transaction(payload) + self._sync_from_server() except _CONNECTION_ERRORS: - LOG.exception( - "Failed to push %d local change(s) to the server; " - "local mirror has drifted from the server until the " - "next successful push or read sync.", - len(payload), - ) + LOG.exception("Resync after a push conflict also failed.") + except _CONNECTION_ERRORS: + LOG.exception( + "Failed to push %d local change(s) to the server; " + "local mirror has drifted from the server until the " + "next successful push or read sync.", + len(payload), + ) def _sync_from_server(self): """ diff --git a/GrampsWebApiDb/tests/test_grampswebapidb.py b/GrampsWebApiDb/tests/test_grampswebapidb.py index 185601530..eefb2a9da 100644 --- a/GrampsWebApiDb/tests/test_grampswebapidb.py +++ b/GrampsWebApiDb/tests/test_grampswebapidb.py @@ -71,7 +71,17 @@ from gramps.gen.lib.json_utils import object_to_data, remove_object from GrampsWebApiDb import grampswebapidb -from GrampsWebApiDb.grampswebapidb import WebApiDB, transaction_to_json +from GrampsWebApiDb.grampswebapidb import WebApiDB, WebApiPushConflict, transaction_to_json + +# grampswebapidb.py imports webapi_client with a bare `from webapi_client +# import ...` (see CLAUDE.md Testing conventions -- this addon has no +# __init__.py, so Gramps and tests alike add its own directory to +# sys.path). That makes "webapi_client" and "GrampsWebApiDb.webapi_client" +# two distinct sys.modules entries for the same file, so an exception +# class must come from whichever import path the code under test actually +# uses -- grampswebapidb.WebApiPushConflict here, not a fresh +# `from GrampsWebApiDb.webapi_client import WebApiPushConflict`, or +# `except WebApiPushConflict` in transaction_commit() won't match it. # ------------------------------------------------------------------------- @@ -362,6 +372,39 @@ def test_push_failure_is_logged_not_raised(self): with self.assertLogs(grampswebapidb.LOG, level="ERROR"): self.db.transaction_commit(trans) # must not raise + def test_conflict_triggers_resync_not_raise(self): + # A WebApiPushConflict means the server rejected the whole batch + # because something changed server-side since the local mirror's + # snapshot -- the response is to resync from the server, not to + # propagate the exception (the local commit already happened). + trans = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + self.db.web_client.push_transaction.side_effect = WebApiPushConflict( + "Object has changed" + ) + with mock.patch.object( + grampswebapidb.SQLite, "transaction_commit" + ), mock.patch.object(self.db, "_sync_from_server") as resync: + with self.assertLogs(grampswebapidb.LOG, level="WARNING"): + self.db.transaction_commit(trans) # must not raise + resync.assert_called_once_with() + + def test_conflict_resync_failure_is_also_swallowed(self): + trans = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + self.db.web_client.push_transaction.side_effect = WebApiPushConflict( + "Object has changed" + ) + with mock.patch.object( + grampswebapidb.SQLite, "transaction_commit" + ), mock.patch.object( + self.db, + "_sync_from_server", + side_effect=HTTPError( + "https://example.com/api/transactions/history/", 500, "boom", None, None + ), + ): + with self.assertLogs(grampswebapidb.LOG, level="WARNING"): + self.db.transaction_commit(trans) # must not raise + # ------------------------------------------------------------------------- # diff --git a/GrampsWebApiDb/tests/test_webapi_client.py b/GrampsWebApiDb/tests/test_webapi_client.py index d42faed9f..503502c73 100644 --- a/GrampsWebApiDb/tests/test_webapi_client.py +++ b/GrampsWebApiDb/tests/test_webapi_client.py @@ -44,6 +44,7 @@ # # ------------------------------------------------------------------------- import base64 +import io import json import os import sys @@ -70,6 +71,7 @@ from GrampsWebApiDb import webapi_client from GrampsWebApiDb.webapi_client import ( WebApiHandler, + WebApiPushConflict, decode_jwt_payload, make_api_key, parse_api_key, @@ -128,6 +130,13 @@ def http_error(code, url="https://example.com/api"): return HTTPError(url, code, f"HTTP {code}", None, None) +def http_error_with_body(code, body, url="https://example.com/api"): + """An HTTPError whose .read() yields a JSON-encoded body, the way a + real gramps-web-api error response (abort_with_message()) looks.""" + fp = io.BytesIO(json.dumps(body).encode()) + return HTTPError(url, code, f"HTTP {code}", None, fp) + + class QueuedUrlopen: """``urlopen`` replacement that returns/raises each queued item in turn, recording every ``Request`` it was called with.""" @@ -494,14 +503,17 @@ def test_empty_payload_sends_no_request(self): handler.push_transaction([]) self.assertEqual(fake.requests, []) - def test_non_empty_payload_posts_with_force(self): + def test_non_empty_payload_posts_without_force(self): + # No force=1: the server's old-data-mismatch check must run, or + # WebApiPushConflict can never fire -- see push_transaction()'s + # docstring. handler = self._authed_handler() payload = [{"type": "add", "handle": "H1", "_class": "Person"}] fake = QueuedUrlopen([FakeResponse({})]) with mock.patch.object(webapi_client, "urlopen", fake): handler.push_transaction(payload) req = fake.requests[0] - self.assertEqual(req.full_url, "https://example.com/api/transactions/?force=1") + self.assertEqual(req.full_url, "https://example.com/api/transactions/") self.assertEqual(req.get_method(), "POST") self.assertEqual(json.loads(req.data), payload) self.assertEqual(req.get_header("Authorization"), f"Bearer {token('AT0')}") @@ -526,6 +538,45 @@ def test_429_retries_once(self): handler.push_transaction([{"type": "add"}]) self.assertEqual(len(fake.requests), 2) + def test_object_changed_400_raises_push_conflict(self): + handler = self._authed_handler() + fake = QueuedUrlopen( + [ + http_error_with_body( + 400, {"error": {"code": 400, "message": "Object has changed"}} + ) + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake): + with self.assertRaises(WebApiPushConflict): + handler.push_transaction([{"type": "add"}]) + # Not retried -- a conflict isn't transient, so exactly one request. + self.assertEqual(len(fake.requests), 1) + + def test_other_400_reasons_are_not_conflicts(self): + # e.g. a payload item missing a required Gramps ID -- our own bug, + # not a concurrent edit -- must propagate as a plain HTTPError. + handler = self._authed_handler() + fake = QueuedUrlopen( + [ + http_error_with_body( + 400, {"error": {"code": 400, "message": "Gramps ID missing"}} + ) + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake): + with self.assertRaises(HTTPError) as ctx: + handler.push_transaction([{"type": "add"}]) + self.assertNotIsInstance(ctx.exception, WebApiPushConflict) + + def test_400_with_unparseable_body_propagates_as_http_error(self): + handler = self._authed_handler() + fake = QueuedUrlopen([http_error_with_body(400, "not-a-dict-body")]) + with mock.patch.object(webapi_client, "urlopen", fake): + with self.assertRaises(HTTPError) as ctx: + handler.push_transaction([{"type": "add"}]) + self.assertNotIsInstance(ctx.exception, WebApiPushConflict) + if __name__ == "__main__": unittest.main() diff --git a/GrampsWebApiDb/webapi_client.py b/GrampsWebApiDb/webapi_client.py index 6d60302b0..c8fcf55a8 100644 --- a/GrampsWebApiDb/webapi_client.py +++ b/GrampsWebApiDb/webapi_client.py @@ -31,6 +31,17 @@ directly) so this addon has no runtime dependency on another addon being installed. +This file is a vendored copy: the canonical, standalone source is now the +gramps-web-api-client package (not yet published; local checkout at +~/gramps/gramps-web-api-client as of this writing), module +gramps_web_api_client/client.py, class Client -- the same class as +WebApiHandler below, just renamed. It was split out so the client could +be discoverable/pip-installable on its own, independent of the Gramps +addon ecosystem. Gramps addons are self-contained tarballs with no +mechanism to declare a pip dependency, so this copy has to stay vendored +here rather than importing that package directly; sync changes by hand in +both directions. + Credentials ----------- Two ways in: username+password (POST /token/, matches GrampsWebSync), or @@ -82,6 +93,33 @@ #: reliably 429s otherwise. RATE_LIMIT_BACKOFF = 1.1 +#: The exact message gramps_webapi/api/tasks.py's old_unchanged() check +#: raises as ValueError("Object has changed"), which POST /transactions/ +#: (without force=1) surfaces as HTTP 400 {"error": {"message": ...}}. +#: push_transaction() matches on this to tell a real conflict apart from +#: the endpoint's other 400s (malformed payload, missing Gramps ID, ...), +#: which are our own bugs, not conflicts, and should propagate as-is. +_CONFLICT_MESSAGE = "Object has changed" + + +class WebApiPushConflict(Exception): + """A push was rejected because the server-side object changed since + the local mirror's snapshot of it (see push_transaction()).""" + + +def _raise_for_push_conflict(exc: HTTPError) -> None: + """Given a 400 from POST /transactions/, raise WebApiPushConflict if + it's the server's old-data-mismatch check; otherwise re-raise ``exc`` + unchanged (a genuinely different 400, e.g. a malformed payload).""" + try: + body = json.loads(exc.read()) + message = body["error"]["message"] + except (ValueError, KeyError, TypeError): + raise exc + if message == _CONFLICT_MESSAGE: + raise WebApiPushConflict(message) from exc + raise exc + def create_macos_ssl_context(): """Create an SSL context using macOS system certificates.""" @@ -351,20 +389,27 @@ def push_transaction( self, payload: list[dict[str, Any]], retry: bool = True ) -> None: """ - POST a batch of local changes to /transactions/. Uses force=1: - without it, the server compares each item's "old" snapshot - against its own current data and rejects the whole batch on any - mismatch (POST /transactions/?force=... semantics, see - gramps_webapi/api/tasks.py's process_transactions -> old_unchanged - check) -- real conflict detection/resolution is out of scope for - now (see grampswebapidb.py), so this is last-write-wins by design, - not an oversight. + POST a batch of local changes to /transactions/ (no force=1): the + server compares each item's "old" snapshot -- the local mirror's + state of the object *before* the local edit -- against its own + current data, and rejects the whole batch with HTTP 400 + ``{"error": {"message": "Object has changed"}}`` on any mismatch + (see gramps_webapi/api/tasks.py's process_transactions -> + old_unchanged()). That's a real, if coarse, optimistic-concurrency + check: it fires whenever the server-side object was edited (by + anyone) since the local mirror last synced, which is exactly what + a conflict is. Raised here as WebApiPushConflict so the caller + (grampswebapidb.py's transaction_commit) can tell "the server + rejected this because something changed underneath it" apart from + a network/auth failure. Actual merge resolution is still out of + scope -- the caller's response to a conflict is to resync from the + server, not to retry the push. """ if not payload: return data = json.dumps(payload).encode() req = Request( - f"{self.url}/transactions/?force=1", + f"{self.url}/transactions/", data=data, method="POST", headers={ @@ -384,6 +429,8 @@ def push_transaction( if exc.code == 429 and retry: sleep(RATE_LIMIT_BACKOFF) return self.push_transaction(payload, retry=False) + if exc.code == 400: + _raise_for_push_conflict(exc) raise except (URLError, socket.timeout): if retry: From 394ca5d6d32fe26acfe6657d06c76e4c998b4a6b Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 17:28:05 -0700 Subject: [PATCH 06/11] GrampsWebApiDb: point README at the gramps-web-api-client generate-key CLI Documents the standalone package's CLI as the primary way to mint a GRAMPS_WEB_API_KEY, with the addon's own vendored WebApiHandler.mint_api_key() as the equivalent no-extra-dependency fallback. Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/GrampsWebApiDb/README.md b/GrampsWebApiDb/README.md index 9989425b5..372dee8d0 100644 --- a/GrampsWebApiDb/README.md +++ b/GrampsWebApiDb/README.md @@ -8,9 +8,20 @@ pushed back to the server as they're committed. The addon takes a single credential, via the `GRAMPS_WEB_API_KEY` environment variable, shaped `*`. There is -deliberately no login dialog and no per-tree settings.ini. Use +deliberately no login dialog and no per-tree settings.ini. Mint one once +via username/password, either from the command line with the standalone +`gramps-web-api-client` package (not yet published; pip-installable from +its own repo, e.g. `pip install -e path/to/gramps-web-api-client`): + +```bash +export GRAMPS_WEB_API_KEY=$(gramps-web-api-client generate-key --url https://your-server/api --username youruser) +``` + +or from Python, using either that package's `Client.mint_api_key(url, +username, password)` or this addon's own vendored copy, `WebApiHandler.mint_api_key(url, username, password)` (see -`webapi_client.py`) once to turn a username/password into this key. +`webapi_client.py`) — same method, same result, no addon-specific +dependency either way. **Security tradeoff:** the token embedded in `GRAMPS_WEB_API_KEY` is a standard JWT *refresh* token obtained from the server's normal `/token/` @@ -35,4 +46,7 @@ gramps-web-api or a loophole being exploited. ## See also * `grampswebapidb.py` for the sync/write-through design (module docstring). -* `webapi_client.py` for the token fetch/refresh implementation. +* `webapi_client.py` for the token fetch/refresh implementation. This is a + hand-synced vendored copy (see its own docstring) -- the canonical, + standalone source is the `gramps-web-api-client` package, which also + has the `generate-key` CLI referenced above. From 3d7a30bce9dce9bcbb8a72d19b045b51b7d42d37 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 17:51:08 -0700 Subject: [PATCH 07/11] GrampsWebApiDb: update references for gramps-web-api-client -> gramps-api-client rename The standalone client package was renamed (gramps_web_api_client -> gramps_api_client, new checkout at ~/gramps/gramps-api-client). Updated webapi_client.py's docstring and README.md accordingly. Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/README.md | 8 ++++---- GrampsWebApiDb/webapi_client.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/GrampsWebApiDb/README.md b/GrampsWebApiDb/README.md index 372dee8d0..b61ded108 100644 --- a/GrampsWebApiDb/README.md +++ b/GrampsWebApiDb/README.md @@ -10,11 +10,11 @@ The addon takes a single credential, via the `GRAMPS_WEB_API_KEY` environment variable, shaped `*`. There is deliberately no login dialog and no per-tree settings.ini. Mint one once via username/password, either from the command line with the standalone -`gramps-web-api-client` package (not yet published; pip-installable from -its own repo, e.g. `pip install -e path/to/gramps-web-api-client`): +`gramps-api-client` package (not yet published; pip-installable from +its own repo, e.g. `pip install -e path/to/gramps-api-client`): ```bash -export GRAMPS_WEB_API_KEY=$(gramps-web-api-client generate-key --url https://your-server/api --username youruser) +export GRAMPS_WEB_API_KEY=$(gramps-api-client generate-key --url https://your-server/api --username youruser) ``` or from Python, using either that package's `Client.mint_api_key(url, @@ -48,5 +48,5 @@ gramps-web-api or a loophole being exploited. * `grampswebapidb.py` for the sync/write-through design (module docstring). * `webapi_client.py` for the token fetch/refresh implementation. This is a hand-synced vendored copy (see its own docstring) -- the canonical, - standalone source is the `gramps-web-api-client` package, which also + standalone source is the `gramps-api-client` package, which also has the `generate-key` CLI referenced above. diff --git a/GrampsWebApiDb/webapi_client.py b/GrampsWebApiDb/webapi_client.py index c8fcf55a8..ef4189956 100644 --- a/GrampsWebApiDb/webapi_client.py +++ b/GrampsWebApiDb/webapi_client.py @@ -32,9 +32,9 @@ installed. This file is a vendored copy: the canonical, standalone source is now the -gramps-web-api-client package (not yet published; local checkout at -~/gramps/gramps-web-api-client as of this writing), module -gramps_web_api_client/client.py, class Client -- the same class as +gramps-api-client package (not yet published; local checkout at +~/gramps/gramps-api-client as of this writing), module +gramps_api_client/client.py, class Client -- the same class as WebApiHandler below, just renamed. It was split out so the client could be discoverable/pip-installable on its own, independent of the Gramps addon ecosystem. Gramps addons are self-contained tarballs with no From 1eec893d07528b37e00bc47bbad1e75d785ed240 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 17:57:56 -0700 Subject: [PATCH 08/11] GrampsWebApiDb: push undo/redo to the server instead of silently desyncing it Gramps core's DbGenericUndo._undo()/_redo() revert the local mirror via low-level _txn_begin()/undo_data()/_txn_commit() calls that never go through transaction_commit(), so a local Undo/Redo previously left the server unchanged with no error at all -- worse than a push conflict, since nothing was even logged. WebApiDB now overrides undo()/redo(): both grab the relevant DbTxn off DbGenericUndo's queue before delegating to super(), rebuild its payload with the existing transaction_to_json(), and push it. Undo sends it to POST /transactions/?undo=1, where gramps-web-api reverses it itself (reverse_transaction()); redo just re-pushes the original forward payload, same as an ordinary commit. Both share the same conflict-detection/resync path as transaction_commit(), factored out into _push_payload(). Verified end-to-end against a live server: add a person, undo (a fresh mirror sync confirms the server no longer has it), redo (confirms it's back). Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/grampswebapidb.py | 55 +++++++++++-- GrampsWebApiDb/tests/test_grampswebapidb.py | 91 +++++++++++++++++++++ GrampsWebApiDb/tests/test_webapi_client.py | 39 +++++++++ GrampsWebApiDb/webapi_client.py | 25 ++++-- 4 files changed, 199 insertions(+), 11 deletions(-) diff --git a/GrampsWebApiDb/grampswebapidb.py b/GrampsWebApiDb/grampswebapidb.py index 6230450d6..ab829753b 100644 --- a/GrampsWebApiDb/grampswebapidb.py +++ b/GrampsWebApiDb/grampswebapidb.py @@ -87,16 +87,32 @@ changed server-side since the local mirror last synced -- a real, if coarse, optimistic-concurrency check: the whole push either applies or none of it does, with no indication of which item conflicted. On a -conflict, transaction_commit() below resyncs from the server so the -local mirror stops showing an edit the server never accepted; it does -not retry the push or attempt a merge, so the local edit is simply lost +conflict, _push_payload() below resyncs from the server so the local +mirror stops showing an edit the server never accepted; it does not +retry the push or attempt a merge, so the losing edit is simply lost from the server's perspective (still present, stale, in the local mirror's edit history) -- real conflict *resolution* (merge, prompt the user) is still out of scope. If the push fails for a non-conflict reason (network error, auth failure), the local commit has already happened and is not rolled back -- the local mirror just drifts from the server until -the next successful push or read sync. Undo/redo integration is also -still out of scope. +the next successful push or read sync. + +Undo/redo integration hooks undo()/redo() the same way transaction_commit() +hooks commits: Gramps core's own DbGenericUndo._undo()/_redo() +(gramps/gen/db/generic.py) revert the local mirror directly via low-level +_txn_begin()/undo_data()/_txn_commit() calls that never go through +transaction_commit(), so without this override a local Undo/Redo would +silently desync the server -- worse than a push conflict, since nothing +would even be logged. The fix reuses transaction_to_json() on the DbTxn +DbGenericUndo already stores in its undo/redo queues (the same object +transaction_commit() turned into a payload the first time), then pushes +it again: undo() sends it to POST /transactions/?undo=1, where the server +reverses it itself (swaps old/new, add<->delete -- see +gramps_webapi/api/resources/util.py's reverse_transaction()); redo() just +pushes the original forward payload again, no different from a fresh +commit. Both go through the same conflict-detection/resync path as a +normal commit. Gramps' own undo history is in-memory/per-session, not +persisted, so this only ever matters within a single running session. """ import logging @@ -189,10 +205,37 @@ def transaction_commit(self, transaction): # Must run before super(): it clears the transaction's records. payload = transaction_to_json(transaction) super().transaction_commit(transaction) + self._push_payload(payload) + + def undo(self, update_history=True): + # Peek before super(): DbGenericUndo._undo() pops this DbTxn off + # undoq. The DbTxn's own backing data isn't touched by that (it + # just moves queues), so building its JSON payload could happen + # either side of super() -- only grabbing the reference itself + # can't wait. + transaction = self.undodb.undoq[-1] if self.undodb.undo_count else None + result = super().undo(update_history) + if result and transaction is not None: + self._push_payload(transaction_to_json(transaction), undo=True) + return result + + def redo(self, update_history=True): + transaction = self.undodb.redoq[-1] if self.undodb.redo_count else None + result = super().redo(update_history) + if result and transaction is not None: + # Redo is just re-applying the original transaction forward -- + # not a variant of undo=True. See push_transaction()'s docstring. + self._push_payload(transaction_to_json(transaction)) + return result + + def _push_payload(self, payload, undo=False): + """Push a change-list payload to the server, handling a rejected + push (conflict or otherwise) the same way regardless of whether it + came from a plain commit, an undo, or a redo.""" if not payload: return try: - self.web_client.push_transaction(payload) + self.web_client.push_transaction(payload, undo=undo) except WebApiPushConflict: LOG.warning( "Server rejected %d local change(s): the object(s) changed " diff --git a/GrampsWebApiDb/tests/test_grampswebapidb.py b/GrampsWebApiDb/tests/test_grampswebapidb.py index eefb2a9da..98022405a 100644 --- a/GrampsWebApiDb/tests/test_grampswebapidb.py +++ b/GrampsWebApiDb/tests/test_grampswebapidb.py @@ -406,6 +406,97 @@ def test_conflict_resync_failure_is_also_swallowed(self): self.db.transaction_commit(trans) # must not raise +# ------------------------------------------------------------------------- +# +# TestUndoRedo +# +# ------------------------------------------------------------------------- +class TestUndoRedo(unittest.TestCase): + """undo()/redo() peek the relevant DbTxn off DbGenericUndo's queue, + turn it back into a change-list payload, and push it -- undo via + push_transaction(..., undo=True) (server reverses it), redo via a + plain push (same as an ordinary commit). Both delegate to + _push_payload(), whose conflict/error handling is already covered by + TestTransactionCommit, so these just confirm the wiring: the right + payload, the right undo flag, and the peek-before-super ordering.""" + + def setUp(self): + self.db = new_instance() + self.db.undodb = mock.MagicMock() + + def test_undo_pushes_with_undo_flag(self): + txn = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + self.db.undodb.undo_count = 1 + self.db.undodb.undoq = [txn] + self.db.undodb.undo.return_value = True + with mock.patch.object(self.db, "_push_payload") as push: + result = self.db.undo() + self.assertTrue(result) + push.assert_called_once() + payload = push.call_args[0][0] + self.assertEqual(payload[0]["handle"], "H1") + self.assertEqual(push.call_args.kwargs, {"undo": True}) + + def test_redo_pushes_without_undo_flag(self): + txn = FakeTransaction([(0, TXNDEL, "H1", person_data("H1"), None)]) + self.db.undodb.redo_count = 1 + self.db.undodb.redoq = [txn] + self.db.undodb.redo.return_value = True + with mock.patch.object(self.db, "_push_payload") as push: + result = self.db.redo() + self.assertTrue(result) + payload = push.call_args[0][0] + self.assertEqual(payload[0]["handle"], "H1") + self.assertEqual(push.call_args.kwargs, {}) + + def test_no_push_when_nothing_to_undo(self): + self.db.undodb.undo_count = 0 + self.db.undodb.undo.return_value = False + with mock.patch.object(self.db, "_push_payload") as push: + result = self.db.undo() + self.assertFalse(result) + push.assert_not_called() + + def test_no_push_when_nothing_to_redo(self): + self.db.undodb.redo_count = 0 + self.db.undodb.redo.return_value = False + with mock.patch.object(self.db, "_push_payload") as push: + result = self.db.redo() + self.assertFalse(result) + push.assert_not_called() + + def test_undo_not_pushed_if_super_reports_nothing_undone(self): + # undo_count > 0 doesn't guarantee _undo() actually ran (e.g. a + # readonly db -- see DbUndo.undo()); only a truthy result pushes. + txn = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + self.db.undodb.undo_count = 1 + self.db.undodb.undoq = [txn] + self.db.undodb.undo.return_value = False + with mock.patch.object(self.db, "_push_payload") as push: + result = self.db.undo() + self.assertFalse(result) + push.assert_not_called() + + def test_transaction_grabbed_before_super_pops_the_queue(self): + # WebApiDB.undo() must read undoq[-1] before delegating to + # super().undo() (-> DbGenericUndo._undo(), which pops it) -- grab + # it too late and the payload would be built from the wrong (or a + # missing) transaction. + txn = FakeTransaction([(0, TXNADD, "H1", None, person_data("H1"))]) + self.db.undodb.undo_count = 1 + self.db.undodb.undoq = [txn] + + def pop_on_undo(update_history): + self.db.undodb.undoq.pop() + return True + + self.db.undodb.undo.side_effect = pop_on_undo + with mock.patch.object(self.db, "_push_payload") as push: + self.db.undo() + payload = push.call_args[0][0] + self.assertEqual(payload[0]["handle"], "H1") + + # ------------------------------------------------------------------------- # # TestMisc diff --git a/GrampsWebApiDb/tests/test_webapi_client.py b/GrampsWebApiDb/tests/test_webapi_client.py index 503502c73..55835d2d9 100644 --- a/GrampsWebApiDb/tests/test_webapi_client.py +++ b/GrampsWebApiDb/tests/test_webapi_client.py @@ -518,6 +518,45 @@ def test_non_empty_payload_posts_without_force(self): self.assertEqual(json.loads(req.data), payload) self.assertEqual(req.get_header("Authorization"), f"Bearer {token('AT0')}") + def test_undo_appends_query_param(self): + handler = self._authed_handler() + payload = [{"type": "add", "handle": "H1", "_class": "Person"}] + fake = QueuedUrlopen([FakeResponse({})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler.push_transaction(payload, undo=True) + req = fake.requests[0] + self.assertEqual( + req.full_url, "https://example.com/api/transactions/?undo=1" + ) + # The payload itself is the original (forward) one -- the server + # reverses it, not the caller. See push_transaction()'s docstring. + self.assertEqual(json.loads(req.data), payload) + + def test_undo_defaults_to_false(self): + handler = self._authed_handler() + fake = QueuedUrlopen([FakeResponse({})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler.push_transaction([{"type": "add"}]) + self.assertEqual(fake.requests[0].full_url, "https://example.com/api/transactions/") + + def test_undo_flag_survives_401_retry(self): + handler = self._authed_handler() + fake = QueuedUrlopen( + [ + http_error(401), + FakeResponse({"access_token": token("AT1")}), + FakeResponse({}), + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + handler.push_transaction([{"type": "add"}], undo=True) + # requests[0] = failed push, [1] = re-auth, [2] = retried push + self.assertEqual( + fake.requests[2].full_url, "https://example.com/api/transactions/?undo=1" + ) + def test_401_triggers_reauth_and_retry(self): handler = self._authed_handler() fake = QueuedUrlopen( diff --git a/GrampsWebApiDb/webapi_client.py b/GrampsWebApiDb/webapi_client.py index ef4189956..26895f686 100644 --- a/GrampsWebApiDb/webapi_client.py +++ b/GrampsWebApiDb/webapi_client.py @@ -386,7 +386,10 @@ def get_transaction_history( return body, total_count def push_transaction( - self, payload: list[dict[str, Any]], retry: bool = True + self, + payload: list[dict[str, Any]], + retry: bool = True, + undo: bool = False, ) -> None: """ POST a batch of local changes to /transactions/ (no force=1): the @@ -404,12 +407,24 @@ def push_transaction( a network/auth failure. Actual merge resolution is still out of scope -- the caller's response to a conflict is to resync from the server, not to retry the push. + + ``undo=True`` sends the *same* payload a prior push_transaction() + call already sent, with ?undo=1: the server reverses it itself + (swaps old/new, add<->delete -- see + gramps_webapi/api/resources/util.py's reverse_transaction()) before + applying, so this is how grampswebapidb.py implements Undo without + having to compute the inverse payload locally. Redo is *not* a + variant of this -- it's just an ordinary push_transaction() call + with the original (forward) payload again. """ if not payload: return data = json.dumps(payload).encode() + url = f"{self.url}/transactions/" + if undo: + url += "?undo=1" req = Request( - f"{self.url}/transactions/", + url, data=data, method="POST", headers={ @@ -425,15 +440,15 @@ def push_transaction( if exc.code == 401 and retry: sleep(RATE_LIMIT_BACKOFF) self._authenticate() - return self.push_transaction(payload, retry=False) + return self.push_transaction(payload, retry=False, undo=undo) if exc.code == 429 and retry: sleep(RATE_LIMIT_BACKOFF) - return self.push_transaction(payload, retry=False) + return self.push_transaction(payload, retry=False, undo=undo) if exc.code == 400: _raise_for_push_conflict(exc) raise except (URLError, socket.timeout): if retry: sleep(RATE_LIMIT_BACKOFF) - return self.push_transaction(payload, retry=False) + return self.push_transaction(payload, retry=False, undo=undo) raise From 60ae3d11c49012178a22bf1ec5379ed1c5f148dd Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 18:03:12 -0700 Subject: [PATCH 09/11] GrampsWebApiDb: add help_url, MANIFEST, and standard i18n fallback - grampswebapidb.py: use the documented try/except get_addon_translator fallback (glocale.translation directly if the addon has no locale/ translations yet) instead of a bare glocale.translation.gettext. - grampswebapidb.gpr.py: add help_url pointing at the addon's wiki page. - MANIFEST: include README.md in the built .addon.tgz -- it documents the GRAMPS_WEB_API_KEY security tradeoff, not just dev notes. Co-Authored-By: Claude Sonnet 5 --- GrampsWebApiDb/MANIFEST | 1 + GrampsWebApiDb/grampswebapidb.gpr.py | 1 + GrampsWebApiDb/grampswebapidb.py | 6 +++++- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 GrampsWebApiDb/MANIFEST diff --git a/GrampsWebApiDb/MANIFEST b/GrampsWebApiDb/MANIFEST new file mode 100644 index 000000000..1cb06e6ea --- /dev/null +++ b/GrampsWebApiDb/MANIFEST @@ -0,0 +1 @@ +GrampsWebApiDb/README.md diff --git a/GrampsWebApiDb/grampswebapidb.gpr.py b/GrampsWebApiDb/grampswebapidb.gpr.py index 59cdb9bb3..330340a00 100644 --- a/GrampsWebApiDb/grampswebapidb.gpr.py +++ b/GrampsWebApiDb/grampswebapidb.gpr.py @@ -33,4 +33,5 @@ databaseclass="WebApiDB", authors=["Doug Blank"], authors_email=["doug.blank@gmail.com"], + help_url="Addon:GrampsWebApiDb", ) diff --git a/GrampsWebApiDb/grampswebapidb.py b/GrampsWebApiDb/grampswebapidb.py index ab829753b..6d6c39df5 100644 --- a/GrampsWebApiDb/grampswebapidb.py +++ b/GrampsWebApiDb/grampswebapidb.py @@ -134,7 +134,11 @@ from webapi_client import WebApiHandler, WebApiPushConflict -_ = glocale.translation.gettext +try: + _trans = glocale.get_addon_translator(__file__) +except ValueError: + _trans = glocale.translation +_ = _trans.gettext LOG = logging.getLogger("grampswebapidb") #: How many transactions to request per page while syncing. From a2c123b2c516f2ce7f7f57042249e60ea82db01f Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Tue, 4 Aug 2026 19:07:08 -0700 Subject: [PATCH 10/11] GrampsWebApiDb: fall back to a full resync when history has a batch-commit gap A batch=True commit (any bulk import, merge, or tool run through gramps-web-api) leaves an empty-changes marker in the transaction history instead of per-object entries, since DBAPI's commit/remove methods skip the undo-log call for batch transactions. _sync_from_server() had no way to detect this and silently missed everything the batch commit did -- confirmed live: importing example.gramps's 2157 people into a synced tree left the local mirror stuck at its pre-import count indefinitely, no matter how often it resynced. _sync_from_server() now treats an empty-changes transaction as a signal, not a no-op, and falls back to a new _full_resync(): download the server's current Gramps XML export and reimport it (via the same stock ImportXml the batch commit itself used) into a freshly wiped local mirror. --- GrampsWebApiDb/grampswebapidb.py | 78 +++++++++++++ GrampsWebApiDb/tests/test_grampswebapidb.py | 115 ++++++++++++++++++++ GrampsWebApiDb/tests/test_webapi_client.py | 86 +++++++++++++++ GrampsWebApiDb/webapi_client.py | 44 ++++++++ 4 files changed, 323 insertions(+) diff --git a/GrampsWebApiDb/grampswebapidb.py b/GrampsWebApiDb/grampswebapidb.py index 6d6c39df5..cfa504b9a 100644 --- a/GrampsWebApiDb/grampswebapidb.py +++ b/GrampsWebApiDb/grampswebapidb.py @@ -81,6 +81,23 @@ No separate "am I currently syncing" flag is needed to stop synced changes from being echoed straight back to the server. +_sync_from_server() can only replay what the history feed actually +logged, and a batch=True commit -- any bulk import, merge, or tool run +through gramps-web-api, not just a one-off -- logs nothing per-object: +DBAPI's own commit_*/remove_* methods guard their trans.add() undo-log +call with `if not trans.batch`, so a batch transaction leaves behind an +empty-changes marker (a real Transaction row, but with no Change rows) +instead of the usual per-object entries. Confirmed live: bulk-importing +example.gramps produced exactly one such marker, and the 2157 people it +added were otherwise invisible to this addon's sync no matter how often +it resynced, because the transaction history itself never recorded +them. _sync_from_server() treats an empty-changes transaction as a +signal that its history-replay approach cannot describe what happened, +and falls back to _full_resync() -- downloading the server's current +full Gramps XML export and reimporting it into a wiped local mirror, +the only way to recover completeness when the incremental feed has a +blind spot by construction. + Pushes go out without force=1, so the server compares each item's "old" snapshot against its own current data and rejects the whole batch with WebApiPushConflict (see webapi_client.push_transaction()) if anything @@ -116,6 +133,8 @@ """ import logging +import os +from tempfile import NamedTemporaryFile from urllib.error import HTTPError, URLError from gramps.gen.const import GRAMPS_LOCALE as glocale @@ -130,7 +149,9 @@ ) from gramps.gen.db.exceptions import DbConnectionError from gramps.gen.lib.json_utils import data_to_object, remove_object +from gramps.gen.user import User from gramps.plugins.db.dbapi.sqlite import SQLite +from gramps.plugins.importer.importxml import importData from webapi_client import WebApiHandler, WebApiPushConflict @@ -265,9 +286,19 @@ def _sync_from_server(self): Pull every transaction after the last-seen timestamp and replay its changes into the local mirror. Returns the number of changes applied. + + An empty "changes" list on a transaction is not a no-op: it is + what a batch=True commit leaves behind (see the module + docstring's note on trans.batch guards around trans.add()) -- + something happened server-side that this feed cannot describe. + Flagged rather than silently skipped; _full_resync() is the + fallback once the whole page range has been walked (so + sync_last_time still advances past it and any *describable* + changes around it are applied normally either way). """ after = self._get_metadata("sync_last_time", default=0) applied = 0 + needs_full_resync = False page = 1 while True: transactions, _total = self.web_client.get_transaction_history( @@ -277,6 +308,8 @@ def _sync_from_server(self): break with DbTxn("Sync from server", self, batch=True) as trans: for server_trans in transactions: + if not server_trans["changes"]: + needs_full_resync = True for change in server_trans["changes"]: if self._apply_change(change, trans): applied += 1 @@ -285,8 +318,53 @@ def _sync_from_server(self): break page += 1 self._set_metadata("sync_last_time", after) + if needs_full_resync: + self._full_resync() return applied + def _full_resync(self): + """ + Rebuild the local mirror from scratch: download the server's own + current Gramps XML export and reimport it, after clearing every + local primary object first. Called by _sync_from_server() when + the transaction-history feed contains an empty-changes marker -- + by definition there is nothing in that history to replay for + whatever produced it, so the only way to recover is to fetch the + server's current state wholesale, the same way populating a + brand new local mirror already works. + + Deliberately reuses the stock ImportXml importer against a raw + XML export rather than reconstructing objects from the REST + /people/, /families/, ... endpoints: those return a marshalled + display schema (plain ints for GrampsType fields, no "_class" + tag), not the json_utils shape data_to_object() needs. Only the + transaction-history feed's new_data and a raw XML export share + that shape, and the whole point of this method is that the + former can't be trusted here. + + The clear-then-import pair each run inside their own batch=True + DbTxn (ImportXml's own, internally, for the import half -- see + importxml.py), so neither triggers transaction_commit()'s + push-to-server path (transaction_to_json() sees nothing to + push for a batch transaction) -- this is a purely local rebuild, + same as _sync_from_server()'s own transactions. + """ + data = self.web_client.download_export() + with NamedTemporaryFile(suffix=".gramps", delete=False) as tmp_file: + tmp_file.write(data) + tmp_path = tmp_file.name + try: + with DbTxn(_("Clear local mirror before full resync"), self, batch=True) as trans: + for key in set(CLASS_TO_KEY_MAP.values()): + name = KEY_TO_NAME_MAP[key] + handles = list(getattr(self, f"get_{name}_handles")()) + remove = getattr(self, f"remove_{name}") + for handle in handles: + remove(handle, trans) + importData(self, tmp_path, User()) + finally: + os.remove(tmp_path) + def _apply_change(self, change, trans): """Replay one server change into the local mirror. Returns True if it was a recognized primary-object change (as opposed to a diff --git a/GrampsWebApiDb/tests/test_grampswebapidb.py b/GrampsWebApiDb/tests/test_grampswebapidb.py index 98022405a..5fb0f0aff 100644 --- a/GrampsWebApiDb/tests/test_grampswebapidb.py +++ b/GrampsWebApiDb/tests/test_grampswebapidb.py @@ -260,6 +260,12 @@ def setUp(self): self.patcher = mock.patch.object(grampswebapidb, "DbTxn", FakeDbTxn) self.patcher.start() self.addCleanup(self.patcher.stop) + # Every existing test here predates the empty-changes-marker -> + # full-resync fallback (see TestFullResyncTrigger below) and uses + # "changes": [] purely as pagination/timestamp filler, not to + # exercise that fallback -- stub it out so those tests keep + # testing what they always tested. + self.db._full_resync = mock.MagicMock() def test_stops_after_short_page(self): change = {"obj_class": "Person", "trans_type": TXNADD, "obj_handle": "H1"} @@ -321,6 +327,115 @@ def test_unrecognized_changes_are_not_counted(self): self.assertEqual(applied, 0) +# ------------------------------------------------------------------------- +# +# TestFullResyncTrigger +# +# A batch=True commit (any bulk import/merge/tool run through +# gramps-web-api) leaves an empty "changes" list on its transaction +# row -- see the module docstring's note on trans.batch guards around +# trans.add(). _sync_from_server() can't replay what was never logged, +# so it falls back to _full_resync() whenever it sees one. +# +# ------------------------------------------------------------------------- +class TestFullResyncTrigger(unittest.TestCase): + def setUp(self): + self.db = new_instance() + self.db.web_client = mock.MagicMock() + self.metadata = {} + self.db._get_metadata = lambda key, default=0: self.metadata.get( + key, default + ) + self.db._set_metadata = lambda key, value, use_txn=True: self.metadata.__setitem__( + key, value + ) + self.db._full_resync = mock.MagicMock() + self.patcher = mock.patch.object(grampswebapidb, "DbTxn", FakeDbTxn) + self.patcher.start() + self.addCleanup(self.patcher.stop) + + def test_empty_changes_transaction_triggers_full_resync(self): + page = [{"timestamp": 1.0, "changes": []}] + self.db.web_client.get_transaction_history.return_value = (page, 1) + self.db._sync_from_server() + self.db._full_resync.assert_called_once_with() + + def test_normal_transactions_do_not_trigger_full_resync(self): + change = {"obj_class": "Person", "trans_type": TXNADD, "obj_handle": "H1"} + page = [{"timestamp": 1.0, "changes": [change]}] + self.db.web_client.get_transaction_history.return_value = (page, 1) + with mock.patch.object(self.db, "_apply_change", return_value=True): + self.db._sync_from_server() + self.db._full_resync.assert_not_called() + + def test_marker_alongside_real_changes_still_applies_the_real_ones(self): + # A marker transaction doesn't block replaying whatever *is* + # describable elsewhere in the same page -- only the parts the + # history feed genuinely has no record of need the fallback. + change = {"obj_class": "Person", "trans_type": TXNADD, "obj_handle": "H1"} + page = [ + {"timestamp": 1.0, "changes": []}, + {"timestamp": 2.0, "changes": [change]}, + ] + self.db.web_client.get_transaction_history.return_value = (page, 2) + with mock.patch.object(self.db, "_apply_change", return_value=True): + applied = self.db._sync_from_server() + self.assertEqual(applied, 1) + self.db._full_resync.assert_called_once_with() + + def test_marker_still_advances_sync_last_time(self): + page = [{"timestamp": 42.0, "changes": []}] + self.db.web_client.get_transaction_history.return_value = (page, 1) + self.db._sync_from_server() + self.assertEqual(self.metadata["sync_last_time"], 42.0) + + +# ------------------------------------------------------------------------- +# +# TestFullResync +# +# ------------------------------------------------------------------------- +class TestFullResync(unittest.TestCase): + def setUp(self): + self.db = new_instance() + self.db.web_client = mock.MagicMock() + self.db.web_client.download_export.return_value = b"fake gramps xml bytes" + self.patcher = mock.patch.object(grampswebapidb, "DbTxn", FakeDbTxn) + self.patcher.start() + self.addCleanup(self.patcher.stop) + + def test_downloads_export_wipes_and_reimports(self): + # get__handles/remove_ for every primary type, plus + # importData itself, are all faked out -- this test is only + # confirming the wiring (download -> wipe every type -> import + # the downloaded file -> clean up the temp file), not any real + # Gramps object storage. + for key, name in grampswebapidb.KEY_TO_NAME_MAP.items(): + if key not in grampswebapidb.CLASS_TO_KEY_MAP.values(): + continue + setattr(self.db, f"get_{name}_handles", mock.MagicMock(return_value=["H1"])) + setattr(self.db, f"remove_{name}", mock.MagicMock()) + + captured_path = {} + + def fake_import_data(database, filename, user): + captured_path["path"] = filename + self.assertTrue(os.path.exists(filename)) + with open(filename, "rb") as f: + self.assertEqual(f.read(), b"fake gramps xml bytes") + + with mock.patch.object(grampswebapidb, "importData", fake_import_data): + self.db._full_resync() + + self.db.web_client.download_export.assert_called_once_with() + for key, name in grampswebapidb.KEY_TO_NAME_MAP.items(): + if key not in grampswebapidb.CLASS_TO_KEY_MAP.values(): + continue + getattr(self.db, f"remove_{name}").assert_called_once_with("H1", mock.ANY) + # The temp file is cleaned up after import, not left behind. + self.assertFalse(os.path.exists(captured_path["path"])) + + # ------------------------------------------------------------------------- # # TestTransactionCommit diff --git a/GrampsWebApiDb/tests/test_webapi_client.py b/GrampsWebApiDb/tests/test_webapi_client.py index 55835d2d9..1ad2a938b 100644 --- a/GrampsWebApiDb/tests/test_webapi_client.py +++ b/GrampsWebApiDb/tests/test_webapi_client.py @@ -126,6 +126,24 @@ def __exit__(self, *exc_info): return False +class FakeBinaryResponse: + """Like FakeResponse, but for endpoints that return a raw file body + rather than JSON (see download_export()/_get_binary()).""" + + def __init__(self, body: bytes, headers=None): + self._body = body + self.headers = headers or {} + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + def http_error(code, url="https://example.com/api"): return HTTPError(url, code, f"HTTP {code}", None, None) @@ -484,6 +502,74 @@ def test_total_count_falls_back_to_body_length(self): self.assertEqual(total, 3) +# ------------------------------------------------------------------------- +# +# TestDownloadExport +# +# ------------------------------------------------------------------------- +class TestDownloadExport(unittest.TestCase): + """download_export() (grampswebapidb.py's _full_resync() fallback) + hits GET /exporters//file and returns the raw body, + sharing _get_binary()'s 401/429/network retry behavior with + _get_json().""" + + def _authed_handler(self): + fake = QueuedUrlopen([FakeResponse({"access_token": token("AT0")})]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler = WebApiHandler("https://example.com/api", refresh_token="RT") + return handler + + def test_request_url_and_returns_raw_bytes(self): + handler = self._authed_handler() + fake = QueuedUrlopen([FakeBinaryResponse(b"gzip-bytes-here")]) + with mock.patch.object(webapi_client, "urlopen", fake): + data = handler.download_export() + self.assertEqual(data, b"gzip-bytes-here") + self.assertEqual( + fake.requests[0].full_url, "https://example.com/api/exporters/gramps/file" + ) + + def test_extension_is_configurable(self): + handler = self._authed_handler() + fake = QueuedUrlopen([FakeBinaryResponse(b"gedcom-bytes")]) + with mock.patch.object(webapi_client, "urlopen", fake): + handler.download_export(extension="ged") + self.assertEqual( + fake.requests[0].full_url, "https://example.com/api/exporters/ged/file" + ) + + def test_401_triggers_reauth_and_retry(self): + handler = self._authed_handler() + fake = QueuedUrlopen( + [ + http_error(401), + FakeResponse({"access_token": token("AT1")}), # the re-auth call + FakeBinaryResponse(b"data-after-reauth"), + ] + ) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + data = handler.download_export() + self.assertEqual(data, b"data-after-reauth") + + def test_429_retries_once(self): + handler = self._authed_handler() + fake = QueuedUrlopen([http_error(429), FakeBinaryResponse(b"data")]) + with mock.patch.object(webapi_client, "urlopen", fake), mock.patch.object( + webapi_client, "sleep" + ): + data = handler.download_export() + self.assertEqual(data, b"data") + + def test_second_failure_propagates(self): + handler = self._authed_handler() + fake = QueuedUrlopen([http_error(500), http_error(500)]) + with mock.patch.object(webapi_client, "urlopen", fake): + with self.assertRaises(HTTPError): + handler.download_export() + + # ------------------------------------------------------------------------- # # TestPushTransaction diff --git a/GrampsWebApiDb/webapi_client.py b/GrampsWebApiDb/webapi_client.py index 26895f686..cebe9b163 100644 --- a/GrampsWebApiDb/webapi_client.py +++ b/GrampsWebApiDb/webapi_client.py @@ -361,6 +361,50 @@ def _get_json(self, url: str, retry: bool = True) -> tuple[Any, dict]: return self._get_json(url, retry=False) raise + def _get_binary(self, url: str, retry: bool = True) -> bytes: + """GET ``url`` with the bearer token and return the raw response + body, unlike _get_json() -- for endpoints that return a file + rather than a JSON document (see download_export()).""" + req = Request( + url, + headers={ + "Authorization": f"Bearer {self.access_token}", + "User-Agent": "GrampsWebApiDb", + }, + ) + try: + with self._open(req) as res: + return res.read() + except HTTPError as exc: + if exc.code == 401 and retry: + sleep(RATE_LIMIT_BACKOFF) + self._authenticate() + return self._get_binary(url, retry=False) + if exc.code == 429 and retry: + sleep(RATE_LIMIT_BACKOFF) + return self._get_binary(url, retry=False) + raise + except (URLError, socket.timeout): + if retry: + sleep(1) + return self._get_binary(url, retry=False) + raise + + def download_export(self, extension: str = "gramps") -> bytes: + """ + Download a full backup export of the tree from the server -- + by default a gzip-compressed Gramps XML file, the exact on-disk + shape Gramps' own ImportXml importer already reads (confirmed + against a live server: GET /exporters/gramps/file runs + synchronously and streams the file back, no task polling + needed). Used by grampswebapidb.py's WebApiDB._full_resync() to + rebuild the local mirror wholesale when the transaction-history + feed can't describe what changed -- see that method's own doc + comment on why. + """ + url = f"{self.url}/exporters/{extension}/file" + return self._get_binary(url) + def get_transaction_history( self, after: float = 0, page: int = 1, pagesize: int = 100 ) -> tuple[list[dict[str, Any]], int]: From 76f8fdae811aac3a74e0028295d1af4aced68051 Mon Sep 17 00:00:00 2001 From: Doug Blank Date: Wed, 5 Aug 2026 10:48:35 -0700 Subject: [PATCH 11/11] GrampsWebApiDb: poll the server so open trees pick up remote changes live load() now schedules a periodic re-sync (GLib.timeout_add_seconds), matching gramps-connect's own browser-client poll against the same endpoint, so edits made from another client show up here without closing and reopening the tree; close() cancels the pending timeout. Since _sync_from_server()'s replay runs in a batch=True DbTxn, DBAPI's own add/update/delete signals never fire for it -- _emit_change_signals() reproduces them by hand so already-open views refresh the same way they would for a local edit, and _full_resync() now calls request_rebuild() for the same reason on its wipe-and-reimport path. --- GrampsWebApiDb/README.md | 10 +- GrampsWebApiDb/grampswebapidb.py | 116 +++++++++++ GrampsWebApiDb/tests/test_grampswebapidb.py | 201 ++++++++++++++++++++ 3 files changed, 325 insertions(+), 2 deletions(-) diff --git a/GrampsWebApiDb/README.md b/GrampsWebApiDb/README.md index b61ded108..c4dec14da 100644 --- a/GrampsWebApiDb/README.md +++ b/GrampsWebApiDb/README.md @@ -1,8 +1,14 @@ GrampsWebApiDb is a Gramps database backend that uses a Gramps Web API server (e.g. gramps-connect or Gramps Web) as a live database, mirrored locally in SQLite for speed. Reads are served from the local mirror, which -is kept current via the server's transaction-history feed; local edits are -pushed back to the server as they're committed. +is kept current via the server's transaction-history feed -- both at load +time and on an ongoing poll while the tree stays open, so a change made +from another client (the web app, another desktop instance) shows up here +without closing and reopening the tree; local edits are pushed back to the +server as they're committed. Every already-open Gramps view (People, +Families, ...) refreshes itself automatically as synced changes land, the +same as it would for a local edit -- see `grampswebapidb.py`'s module +docstring for how. ## Credentials diff --git a/GrampsWebApiDb/grampswebapidb.py b/GrampsWebApiDb/grampswebapidb.py index cfa504b9a..c1d300d41 100644 --- a/GrampsWebApiDb/grampswebapidb.py +++ b/GrampsWebApiDb/grampswebapidb.py @@ -114,6 +114,45 @@ is not rolled back -- the local mirror just drifts from the server until the next successful push or read sync. +The mirror stays current while the tree is open, not just at load() time: +load() also schedules a GLib.timeout_add_seconds() tick (POLL_INTERVAL_SECONDS) +that re-runs _sync_from_server() for as long as the database stays open -- +the same timestamp-cursor poll gramps-connect's browser client uses against +this same endpoint (see gramps-connect's store/historyPoll.ts), so a change +made from any other client shows up here without closing and reopening the +tree. It runs synchronously on the GTK main thread (like the initial +load()-time sync already did, and like viewmanager.py's own autobackup +timer) rather than on a background thread -- correct but simple, at the +cost of a brief UI pause during each poll's network round trip; moving it +off-thread (GrampsWebSync's GLibTaskRunner is the precedent, not imported +here for the same no-cross-addon-dependency reason as transaction_to_json() +below) is a reasonable future improvement, not attempted here. close() +cancels the pending timeout so a closed database doesn't keep polling. + +_sync_from_server()'s replay runs inside a batch=True DbTxn deliberately +(see the write-through section below for why), but that has a side effect +beyond suppressing trans.add(): DBAPI.transaction_commit() only emits its +person-add/family-update/event-delete/... signals `if not transaction.batch` +(see dbapi.py), so a batch replay is otherwise invisible to every +already-open GTK view -- the local mirror would update on disk with nothing +on screen changing. _emit_change_signals() reproduces just that signal half +by hand, once per synced page, using the exact same +KEY_TO_NAME_MAP[key] + {"add"/"update"/"delete"} signal names DBAPI itself +emits for a normal (non-batch) local edit -- so every view refreshes exactly +the way it already knows how to for a local change, with no new view-side +code needed. Collapsed to one signal per (obj_class, handle) -- the net +effect across everything applied in that page, so e.g. an update +immediately followed by a delete of the same object only fires the delete +signal, not both. + +A _full_resync() (see below) is the one path that doesn't go through +_emit_change_signals(): a full wipe-and-reimport is exactly the "too much +changed to describe incrementally" case DbGeneric's own request_rebuild() +exists for (it emits a single -rebuild signal per object type, +telling every view to reload wholesale rather than replay a specific +add/update/delete) -- so _full_resync() calls that once after a successful +reimport instead. + Undo/redo integration hooks undo()/redo() the same way transaction_commit() hooks commits: Gramps core's own DbGenericUndo._undo()/_redo() (gramps/gen/db/generic.py) revert the local mirror directly via low-level @@ -137,6 +176,8 @@ from tempfile import NamedTemporaryFile from urllib.error import HTTPError, URLError +from gi.repository import GLib + from gramps.gen.const import GRAMPS_LOCALE as glocale from gramps.gen.db import DbTxn from gramps.gen.db.dbconst import ( @@ -165,6 +206,11 @@ #: How many transactions to request per page while syncing. SYNC_PAGE_SIZE = 100 +#: How often (seconds) load() re-polls the server for as long as the +#: database stays open -- see the module docstring's note on why this runs +#: synchronously on the GTK main thread rather than a background timer. +POLL_INTERVAL_SECONDS = 10 + #: Failure modes from WebApiHandler.from_env()/push_transaction(): a #: malformed/missing key (ValueError), a bad server response shape #: (KeyError/JSONDecodeError, the latter a ValueError subclass), or the @@ -174,6 +220,10 @@ _TRANS_TYPE_NAME = {TXNADD: "add", TXNUPD: "update", TXNDEL: "delete"} +#: Same signal-name suffixes DBAPI.transaction_commit() uses (dbapi.py's +#: own `action` dict) -- see _emit_change_signals(). +_TRANS_TYPE_ACTION = {TXNADD: "-add", TXNUPD: "-update", TXNDEL: "-delete"} + def transaction_to_json(transaction): """ @@ -225,6 +275,30 @@ def _initialize(self, directory, username, password): def load(self, *args, **kwargs): super().load(*args, **kwargs) self._sync_from_server() + self._poll_source_id = GLib.timeout_add_seconds( + POLL_INTERVAL_SECONDS, self._poll_tick + ) + + def close(self, *args, **kwargs): + # Stop polling a database that's no longer open -- otherwise the + # next tick would run _sync_from_server() (and touch self.dbapi) + # against a connection that's about to be (or already) closed. + poll_source_id = getattr(self, "_poll_source_id", None) + if poll_source_id is not None: + GLib.source_remove(poll_source_id) + self._poll_source_id = None + super().close(*args, **kwargs) + + def _poll_tick(self): + """GLib.timeout_add_seconds callback -- see the module docstring's + polling section. Must return True (GLib.SOURCE_CONTINUE) to keep + firing; returning a falsy value cancels the timeout, so a network + error is caught and logged here rather than left to propagate.""" + try: + self._sync_from_server() + except _CONNECTION_ERRORS: + LOG.exception("Periodic sync from server failed; will retry.") + return GLib.SOURCE_CONTINUE def transaction_commit(self, transaction): # Must run before super(): it clears the transaction's records. @@ -306,6 +380,9 @@ def _sync_from_server(self): ) if not transactions: break + # (obj_class, handle) -> trans_type, collapsed to the net + # effect within this page -- see _emit_change_signals(). + net_changes = {} with DbTxn("Sync from server", self, batch=True) as trans: for server_trans in transactions: if not server_trans["changes"]: @@ -313,7 +390,11 @@ def _sync_from_server(self): for change in server_trans["changes"]: if self._apply_change(change, trans): applied += 1 + net_changes[ + (change["obj_class"], change["obj_handle"]) + ] = change["trans_type"] after = max(after, server_trans["timestamp"]) + self._emit_change_signals(net_changes) if len(transactions) < SYNC_PAGE_SIZE: break page += 1 @@ -362,6 +443,13 @@ def _full_resync(self): for handle in handles: remove(handle, trans) importData(self, tmp_path, User()) + # importData() runs its own batch=True DbTxn internally, so + # (like _sync_from_server()'s replay) it emits nothing to + # already-open views on its own -- request_rebuild() is the + # "too much changed to describe incrementally" signal DbGeneric + # itself defines for exactly this case (one -rebuild per + # object type, telling every view to reload wholesale). + self.request_rebuild() finally: os.remove(tmp_path) @@ -383,3 +471,31 @@ def _apply_change(self, change, trans): obj = data_to_object(change["new_data"]) getattr(self, f"commit_{name}")(obj, trans) return True + + def _emit_change_signals(self, net_changes): + """Emit the person-add/family-update/event-delete/... signals a + normal (non-batch) local commit would have emitted for these same + changes -- see the module docstring's note on why + _sync_from_server()'s batch=True replay needs this done by hand. + + net_changes: {(obj_class, obj_handle): trans_type}, already + collapsed to the net effect per handle (see _sync_from_server()). + Unrecognized obj_class values (reference-type changes never reach + here in the first place -- see _apply_change()) are skipped the + same way _apply_change() skips them. + + Grouped and emitted in the same order DBAPI.transaction_commit() + uses for a normal commit -- deletes and adds before updates -- so + a view that (for instance) cares about total counts sees them + change before it sees an in-place update to one of the survivors. + """ + by_type = {TXNDEL: {}, TXNADD: {}, TXNUPD: {}} + for (obj_class, handle), trans_type in net_changes.items(): + key = CLASS_TO_KEY_MAP.get(obj_class) + if key is None: + continue + name = KEY_TO_NAME_MAP[key] + by_type[trans_type].setdefault(name, []).append(handle) + for trans_type in (TXNDEL, TXNADD, TXNUPD): + for name, handles in by_type[trans_type].items(): + self.emit(name + _TRANS_TYPE_ACTION[trans_type], (handles,)) diff --git a/GrampsWebApiDb/tests/test_grampswebapidb.py b/GrampsWebApiDb/tests/test_grampswebapidb.py index 5fb0f0aff..2aa675124 100644 --- a/GrampsWebApiDb/tests/test_grampswebapidb.py +++ b/GrampsWebApiDb/tests/test_grampswebapidb.py @@ -226,6 +226,59 @@ def test_update_is_also_an_upsert(self): self.db.remove_person.assert_not_called() +# ------------------------------------------------------------------------- +# +# TestEmitChangeSignals +# +# ------------------------------------------------------------------------- +class TestEmitChangeSignals(unittest.TestCase): + """_emit_change_signals() reproduces the person-add/family-update/... + signals a normal (non-batch) local commit would have emitted -- see + _sync_from_server()'s batch=True DbTxn and the module docstring's note + on why that otherwise leaves already-open views unaware anything + changed.""" + + def setUp(self): + self.db = new_instance() + self.db.emit = mock.MagicMock() + + def emitted(self): + return {call.args[0]: call.args[1][0] for call in self.db.emit.call_args_list} + + def test_add_emits_person_add_with_handle(self): + self.db._emit_change_signals({("Person", "H1"): TXNADD}) + self.assertEqual(self.emitted(), {"person-add": ["H1"]}) + + def test_update_emits_dash_update(self): + self.db._emit_change_signals({("Family", "F1"): TXNUPD}) + self.assertEqual(self.emitted(), {"family-update": ["F1"]}) + + def test_delete_emits_dash_delete(self): + self.db._emit_change_signals({("Event", "E1"): TXNDEL}) + self.assertEqual(self.emitted(), {"event-delete": ["E1"]}) + + def test_unrecognized_obj_class_is_skipped(self): + self.db._emit_change_signals({("NotAThing", "H1"): TXNADD}) + self.db.emit.assert_not_called() + + def test_same_class_and_trans_type_batched_into_one_call(self): + self.db._emit_change_signals( + {("Person", "H1"): TXNUPD, ("Person", "H2"): TXNUPD} + ) + self.db.emit.assert_called_once() + name, (handles,) = self.db.emit.call_args[0] + self.assertEqual(name, "person-update") + self.assertEqual(set(handles), {"H1", "H2"}) + + def test_deletes_and_adds_emitted_before_updates(self): + # Same ordering as DBAPI.transaction_commit()'s own signal loop. + self.db._emit_change_signals( + {("Person", "H1"): TXNUPD, ("Family", "F1"): TXNDEL} + ) + names = [call.args[0] for call in self.db.emit.call_args_list] + self.assertEqual(names, ["family-delete", "person-update"]) + + # ------------------------------------------------------------------------- # # TestSyncFromServer @@ -250,6 +303,12 @@ class TestSyncFromServer(unittest.TestCase): def setUp(self): self.db = new_instance() self.db.web_client = mock.MagicMock() + # _sync_from_server() now emits change signals per page (see + # TestEmitChangeSignals for that logic in isolation) -- emit() + # itself needs Callback.__init__'s instance state, which + # new_instance()'s bare __new__() never runs, so it's stubbed here + # the same way commit_person/remove_person are stubbed elsewhere. + self.db.emit = mock.MagicMock() self.metadata = {} self.db._get_metadata = lambda key, default=0: self.metadata.get( key, default @@ -326,6 +385,51 @@ def test_unrecognized_changes_are_not_counted(self): applied = self.db._sync_from_server() self.assertEqual(applied, 0) + def test_emits_a_signal_per_applied_change(self): + change = {"obj_class": "Person", "trans_type": TXNADD, "obj_handle": "H1"} + self.db.web_client.get_transaction_history.return_value = ( + [{"timestamp": 5.0, "changes": [change]}], + 1, + ) + with mock.patch.object(self.db, "_apply_change", return_value=True): + self.db._sync_from_server() + self.db.emit.assert_called_once_with("person-add", (["H1"],)) + + def test_repeated_changes_to_one_handle_collapse_to_the_last(self): + # Same handle, updated then deleted within one page/poll -- only + # the net (delete) signal should fire, not both. + page = [ + { + "timestamp": 1.0, + "changes": [ + {"obj_class": "Person", "trans_type": TXNUPD, "obj_handle": "H1"} + ], + }, + { + "timestamp": 2.0, + "changes": [ + {"obj_class": "Person", "trans_type": TXNDEL, "obj_handle": "H1"} + ], + }, + ] + self.db.web_client.get_transaction_history.return_value = (page, 2) + with mock.patch.object(self.db, "_apply_change", return_value=True): + self.db._sync_from_server() + self.db.emit.assert_called_once_with("person-delete", (["H1"],)) + + def test_unrecognized_changes_emit_no_signal(self): + page = [ + { + "timestamp": 1.0, + "changes": [ + {"obj_class": "Bogus", "trans_type": TXNADD, "obj_handle": "H1"} + ], + } + ] + self.db.web_client.get_transaction_history.return_value = (page, 1) + self.db._sync_from_server() + self.db.emit.assert_not_called() + # ------------------------------------------------------------------------- # @@ -342,6 +446,7 @@ class TestFullResyncTrigger(unittest.TestCase): def setUp(self): self.db = new_instance() self.db.web_client = mock.MagicMock() + self.db.emit = mock.MagicMock() # see TestSyncFromServer.setUp's note self.metadata = {} self.db._get_metadata = lambda key, default=0: self.metadata.get( key, default @@ -400,6 +505,7 @@ def setUp(self): self.db = new_instance() self.db.web_client = mock.MagicMock() self.db.web_client.download_export.return_value = b"fake gramps xml bytes" + self.db.emit = mock.MagicMock() # see TestSyncFromServer.setUp's note self.patcher = mock.patch.object(grampswebapidb, "DbTxn", FakeDbTxn) self.patcher.start() self.addCleanup(self.patcher.stop) @@ -434,6 +540,32 @@ def fake_import_data(database, filename, user): getattr(self.db, f"remove_{name}").assert_called_once_with("H1", mock.ANY) # The temp file is cleaned up after import, not left behind. self.assertFalse(os.path.exists(captured_path["path"])) + # A successful reimport can't be described as specific add/update/ + # delete signals, so every view is told to reload wholesale instead + # -- see request_rebuild() in gramps.gen.db.generic. + emitted = [call.args[0] for call in self.db.emit.call_args_list] + self.assertIn("person-rebuild", emitted) + self.assertIn("family-rebuild", emitted) + + def test_failed_import_does_not_trigger_rebuild(self): + # request_rebuild() sits after importData() in _full_resync(), not + # in a finally -- a reimport that raised partway through left the + # mirror in an unknown state, which is not something to tell every + # view "reload, this is now correct" about. + for key, name in grampswebapidb.KEY_TO_NAME_MAP.items(): + if key not in grampswebapidb.CLASS_TO_KEY_MAP.values(): + continue + setattr(self.db, f"get_{name}_handles", mock.MagicMock(return_value=[])) + setattr(self.db, f"remove_{name}", mock.MagicMock()) + + def failing_import_data(database, filename, user): + raise RuntimeError("boom") + + with mock.patch.object(grampswebapidb, "importData", failing_import_data): + with self.assertRaises(RuntimeError): + self.db._full_resync() + + self.db.emit.assert_not_called() # ------------------------------------------------------------------------- @@ -642,5 +774,74 @@ def test_initialize_stores_web_client_and_calls_super(self): super_init.assert_called_once_with("/tmp/some-tree", "user", "pw") +# ------------------------------------------------------------------------- +# +# TestPolling +# +# load() schedules a GLib.timeout_add_seconds() tick that re-syncs for as +# long as the database stays open (see the module docstring's polling +# section); close() must cancel it so a closed database doesn't keep +# polling on a connection that's going away. +# +# ------------------------------------------------------------------------- +class TestPolling(unittest.TestCase): + def setUp(self): + self.db = new_instance() + + def test_load_syncs_and_schedules_polling(self): + with mock.patch.object( + grampswebapidb.SQLite, "load" + ) as super_load, mock.patch.object( + self.db, "_sync_from_server" + ) as sync, mock.patch.object( + grampswebapidb.GLib, "timeout_add_seconds", return_value=42 + ) as timeout_add: + self.db.load("some/path") + super_load.assert_called_once_with("some/path") + sync.assert_called_once_with() + timeout_add.assert_called_once_with( + grampswebapidb.POLL_INTERVAL_SECONDS, self.db._poll_tick + ) + self.assertEqual(self.db._poll_source_id, 42) + + def test_close_cancels_pending_poll(self): + self.db._poll_source_id = 42 + with mock.patch.object( + grampswebapidb.SQLite, "close" + ) as super_close, mock.patch.object( + grampswebapidb.GLib, "source_remove" + ) as source_remove: + self.db.close() + source_remove.assert_called_once_with(42) + self.assertIsNone(self.db._poll_source_id) + super_close.assert_called_once_with() + + def test_close_without_a_poll_scheduled_is_a_no_op(self): + # e.g. close() called after a failed load(), before the timeout + # was ever scheduled. + with mock.patch.object( + grampswebapidb.SQLite, "close" + ) as super_close, mock.patch.object( + grampswebapidb.GLib, "source_remove" + ) as source_remove: + self.db.close() + source_remove.assert_not_called() + super_close.assert_called_once_with() + + def test_poll_tick_syncs_and_keeps_repeating(self): + with mock.patch.object(self.db, "_sync_from_server") as sync: + result = self.db._poll_tick() + sync.assert_called_once_with() + self.assertEqual(result, grampswebapidb.GLib.SOURCE_CONTINUE) + + def test_poll_tick_swallows_connection_errors_and_keeps_repeating(self): + with mock.patch.object( + self.db, "_sync_from_server", side_effect=OSError("network down") + ): + with self.assertLogs(grampswebapidb.LOG, level="ERROR"): + result = self.db._poll_tick() + self.assertEqual(result, grampswebapidb.GLib.SOURCE_CONTINUE) + + if __name__ == "__main__": unittest.main()