diff --git a/.env.example b/.env.example index f7d3438..823705b 100644 --- a/.env.example +++ b/.env.example @@ -31,6 +31,14 @@ #KP2BW_UPDATE=1 #KP2BW_INCLUDE_OVERSIZE_SECRETS=0 +# URL handling. +# Match mode for plain URLs migrated into login URIs: +# default (default, leaves match unset = Bitwarden account default) | domain (force base-domain, +# reproduces KeePassXC) | host | startswith | exact | regex | never. +#KP2BW_URI_MATCH=default +# Interpret KeePassXC URL syntax on additional URLs (quoted = exact, * = wildcard); 0 imports them as plain strings. +#KP2BW_INTERPRET_URI_SYNTAX=1 + # Behavior / output. #KP2BW_YES=0 #KP2BW_VERBOSE=0 @@ -47,6 +55,15 @@ # values above 3600 are clamped. #KP2BW_HTTP_TIMEOUT=180 +# Upgrade existing items: re-fold legacy KP2A_URL*/AndroidApp custom fields into login URIs, +# then exit (no migration; no KeePass database needed). For imports made before URL folding. +#KP2BW_MIGRATE_URIS=0 + +# Print a read-only URI collision report and exit (changes nothing): groups login URLs by +# registrable domain, lists the ones with multiple hosts (the entries that all autofill together +# under base-domain match). 'keepass' reads the db, 'bitwarden' reads the live vault. +#KP2BW_REPORT_URIS=bitwarden + # Finalize mode. # When set, kp2bw removes the KP2BW_ID dedup stamp from every migrated item and exits # (no migration; no KeePass database needed). Run once the migration is complete and diff --git a/CHANGELOG.md b/CHANGELOG.md index dc7ae97..81e5c52 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,48 @@ Format based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Unreleased] +### Fixed + +- **`bw serve` is no longer orphaned on teardown (POSIX), which previously hung the process.** On Linux/macOS `bw` is + commonly a node launcher that spawns a worker; teardown signalled only the tracked PID, leaving the worker alive -- it + kept the port and, when kp2bw's stdout was a pipe, held the pipe open so the parent pipeline never reached EOF (a + multi-minute "still running" hang) and accumulated orphaned `bw serve` processes across runs. `bw serve` is now + started in its own session (`start_new_session=True`) and torn down by signalling the whole process group (SIGTERM, + then SIGKILL after a timeout), so the launcher and worker die together. Windows teardown (taskkill /T + port reap) is + unchanged. + +- **An empty environment variable no longer shadows the matching `.env` entry.** `KP2BW_KEEPASS_FILE=""` (or any + empty-string export) used to override the `.env` value -- `load_dotenv(override=False)` treats an empty export as + "set" -- producing a baffling "KeePass database path is required" even when `.env` clearly had it. `_load_dotenv` now + fills any variable that is unset *or empty* from the file, while a real *non-empty* shell variable still wins (the + documented CLI flag > env var > default precedence is preserved for meaningful values). + ### Added +- **`--report-uris keepass|bitwarden` -- a read-only URI collision report** (env `KP2BW_REPORT_URIS`). Groups every + login URL by registrable domain (a curated two-level public-suffix heuristic, so `10bis.co.il` stays whole) and lists + the domains with more than one host -- exactly the logins that all surface together under Bitwarden's base-domain + matching. `keepass` reads the database (previewing post-migration collisions); `bitwarden` reads the live vault + (honouring `-o`/`-c`). It changes nothing -- it just prints, so you can decide which entries to switch to Host match + (or flip your account's default URI match detection). + +- **Additional URLs and Android packages migrate as Bitwarden login URIs, not custom fields** -- a KeePass(XC) entry's + additional URLs (`KP2A_URL`/`KP2A_URL_n`, plus the plainer `URL`/`URL_n` convention) and Android packages + (`AndroidApp`/`AndroidApp_n`, including the no-underscore `AndroidApp1` variant) were copied verbatim into custom + fields, where they were inert. They now become real entries in `login.uris`, so one login autofills across every site + and app it covered in KeePass; free-text URL labels (`API Url`, `Alt. URL`, `Website`, …) are deliberately left as + custom fields. Each URI gets a per-URI match mode reproducing KeePassXC's behaviour: a plain URL → the account default + (`match` left unset -- what Bitwarden itself writes on export; `--uri-match` / `KP2BW_URI_MATCH` overrides, e.g. + `domain` forces base-domain to replicate KeePassXC's host-based matching), a double-quoted URL → exact, and a `*` + wildcard → starts-with (trailing path) or regex. `AndroidApp` becomes an `androidapp://` URI. Non-web schemes + (`keepassxc://`, `cmd://`, `kdbx://`, `file://`) and unresolved `{REF:…}` URLs are dropped rather than left as dead + URIs. `--no-interpret-uri-syntax` (`KP2BW_INTERPRET_URI_SYNTAX`) disables the quote/wildcard interpretation for a + literal import. Bitwarden applies a regex to the whole URL (unlike KeePassXC's separate host/path regexes), so complex + wildcards are emitted as a best-effort whole-URL regex with a warning to review. Items imported before this are + upgraded by a normal re-run (the change is detected and the item updated in place); for users who don't want to + re-import, `kp2bw --migrate-uris` (env `KP2BW_MIGRATE_URIS`) is a Bitwarden-only one-shot pass that re-folds the + legacy fields into URIs on every existing item. Both honour `--uri-match` / `--interpret-uri-syntax` and `-o`/`-c` + scope. - **Configurable per-request HTTP timeout via `KP2BW_HTTP_TIMEOUT`** -- the timeout for a single `bw serve` request is now overridable through the `KP2BW_HTTP_TIMEOUT` environment variable (seconds), so a slow self-hosted server (e.g. Vaultwarden) where an individual item write outlasts the default no longer times out. Non-numeric or non-positive diff --git a/README.md b/README.md index 66713e3..5932ff6 100644 --- a/README.md +++ b/README.md @@ -85,35 +85,62 @@ kp2bw [-h] [-V] [-k PASSWORD] [-K FILE] [-b PASSWORD] [-o ID] [--path-to-name-skip N] [--skip-expired | --no-skip-expired] [--include-recycle-bin | --no-include-recycle-bin] [--metadata | --no-metadata] [--update | --no-update] - [--include-oversize-secrets] [--strip-ids] [-y] [-v] [-d] + [--include-oversize-secrets] [--uri-match MODE] + [--interpret-uri-syntax | --no-interpret-uri-syntax] + [--migrate-uris] [--report-uris SOURCE] [--strip-ids] [-y] [-v] [-d] [FILE] ``` -| Flag | Description | Env var | -| -------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------- | -| `keepass_file` | Path to your KeePass 2.x database | `KP2BW_KEEPASS_FILE` | -| `-k, --keepass-password` | KeePass password (prompted if omitted) | `KP2BW_KEEPASS_PASSWORD` | -| `-K, --keepass-keyfile` | KeePass key file | `KP2BW_KEEPASS_KEYFILE` | -| `-b, --bitwarden-password` | Bitwarden password (prompted if omitted) | `KP2BW_BITWARDEN_PASSWORD` | -| `-o, --bitwarden-org` | Bitwarden Organization ID | `KP2BW_BITWARDEN_ORG` | -| `-c, --bitwarden-collection` | Collection ID, or `auto` to derive from top-level folder names | `KP2BW_BITWARDEN_COLLECTION` | -| `-t, --import-tags` | Only import entries with these tags | `KP2BW_IMPORT_TAGS` (comma-separated) | -| `--path-to-name` / `--no-path-to-name` | Prepend folder path to entry names (default: off) | `KP2BW_PATH_TO_NAME` | -| `--path-to-name-skip` | Skip first N folders in path prefix (default: 1) | `KP2BW_PATH_TO_NAME_SKIP` | -| `--skip-expired` | Skip entries that have expired in KeePass | `KP2BW_SKIP_EXPIRED` | -| `--include-recycle-bin` | Include Recycle Bin entries (excluded by default) | `KP2BW_INCLUDE_RECYCLE_BIN` | -| `--metadata` / `--no-metadata` | Toggle KeePass tags/expiry as a `KP2BW_META` field (default: on) | `KP2BW_MIGRATE_METADATA` | -| `--update` / `--no-update` | Update existing entries changed in KeePass (default: on) | `KP2BW_UPDATE` | -| `--include-oversize-secrets` | Offload over-limit secret fields[^offload] to a `.txt` attachment instead of dropping them (default: off) | `KP2BW_INCLUDE_OVERSIZE_SECRETS` | -| `--strip-ids` | Finalize: remove the `KP2BW_ID` dedup stamp from migrated items, then exit (no migration; no KeePass db) | `KP2BW_STRIP_IDS` | -| `-y, --yes` | Skip the Bitwarden CLI setup confirmation prompt | `KP2BW_YES` | -| `-v, --verbose` | Verbose output | `KP2BW_VERBOSE` | -| `-d, --debug` | Debug output — includes third-party library logs | `KP2BW_DEBUG` | -| `-V, --version` | Print the installed `kp2bw` version and exit | - | +| Flag | Description | Env var | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ------------------------------------- | +| `keepass_file` | Path to your KeePass 2.x database | `KP2BW_KEEPASS_FILE` | +| `-k, --keepass-password` | KeePass password (prompted if omitted) | `KP2BW_KEEPASS_PASSWORD` | +| `-K, --keepass-keyfile` | KeePass key file | `KP2BW_KEEPASS_KEYFILE` | +| `-b, --bitwarden-password` | Bitwarden password (prompted if omitted) | `KP2BW_BITWARDEN_PASSWORD` | +| `-o, --bitwarden-org` | Bitwarden Organization ID | `KP2BW_BITWARDEN_ORG` | +| `-c, --bitwarden-collection` | Collection ID, or `auto` to derive from top-level folder names | `KP2BW_BITWARDEN_COLLECTION` | +| `-t, --import-tags` | Only import entries with these tags | `KP2BW_IMPORT_TAGS` (comma-separated) | +| `--path-to-name` / `--no-path-to-name` | Prepend folder path to entry names (default: off) | `KP2BW_PATH_TO_NAME` | +| `--path-to-name-skip` | Skip first N folders in path prefix (default: 1) | `KP2BW_PATH_TO_NAME_SKIP` | +| `--skip-expired` | Skip entries that have expired in KeePass | `KP2BW_SKIP_EXPIRED` | +| `--include-recycle-bin` | Include Recycle Bin entries (excluded by default) | `KP2BW_INCLUDE_RECYCLE_BIN` | +| `--metadata` / `--no-metadata` | Toggle KeePass tags/expiry as a `KP2BW_META` field (default: on) | `KP2BW_MIGRATE_METADATA` | +| `--update` / `--no-update` | Update existing entries changed in KeePass (default: on) | `KP2BW_UPDATE` | +| `--include-oversize-secrets` | Offload over-limit secret fields[^offload] to a `.txt` attachment instead of dropping them (default: off) | `KP2BW_INCLUDE_OVERSIZE_SECRETS` | +| `--uri-match MODE` | Match mode for plain URLs: `default`(default, account default)/`domain`/`host`/`startswith`/`exact`/`regex`/`never` | `KP2BW_URI_MATCH` | +| `--interpret-uri-syntax` | Honor KeePassXC quote/wildcard URL syntax on additional URLs (default: on; `--no-…` for literal) | `KP2BW_INTERPRET_URI_SYNTAX` | +| `--migrate-uris` | Upgrade existing items: re-fold legacy `KP2A_URL*`/`AndroidApp` fields into login URIs, then exit (no KeePass) | `KP2BW_MIGRATE_URIS` | +| `--report-uris SOURCE` | Print a read-only URI collision report (`keepass` or `bitwarden`) and exit; lists registrable domains with multiple hosts | `KP2BW_REPORT_URIS` | +| `--strip-ids` | Finalize: remove the `KP2BW_ID` dedup stamp from migrated items, then exit (no migration; no KeePass db) | `KP2BW_STRIP_IDS` | +| `-y, --yes` | Skip the Bitwarden CLI setup confirmation prompt | `KP2BW_YES` | +| `-v, --verbose` | Verbose output | `KP2BW_VERBOSE` | +| `-d, --debug` | Debug output — includes third-party library logs | `KP2BW_DEBUG` | +| `-V, --version` | Print the installed `kp2bw` version and exit | - | Configuration precedence is always: CLI flag > environment variable > built-in default. -### Finalizing (`--strip-ids`) +### URL handling + +A KeePass(XC) entry's additional URLs (`KP2A_URL`/`KP2A_URL_n`, plus the plainer `URL`/`URL_n` convention) and Android +packages (`AndroidApp`/`AndroidApp_n`, incl. the no-underscore `AndroidApp1` variant) are migrated as real Bitwarden +**login URIs** — not inert custom fields — so one login autofills across every site and app it covered in KeePass. +Free-text URL labels (`API Url`, `Alt. URL`, `Website`, …) are left as custom fields, since folding those would wrongly +autofill API endpoints and other metadata. Each URI gets a per-URI match mode reproducing KeePassXC's behaviour: a plain +URL → **account default** (`match` unset, what Bitwarden itself writes; pass `--uri-match domain` to force base-domain +and replicate KeePassXC's host-based matching), a double-quoted URL → **exact**, and a `*` wildcard → **starts-with** +(trailing path) or **regex**. Non-web schemes (`keepassxc://`, `cmd://`, `kdbx://`, `file://`) and unresolved `{REF:…}` +URLs are dropped. `--no-interpret-uri-syntax` disables the quote/wildcard interpretation and imports every URL as a +plain string. + +Already imported before this existed? Two ways to upgrade: re-run a normal migration (the change is detected and the +items are updated in place), or — if you don't want to re-import — run `kp2bw --migrate-uris`, a Bitwarden-only one-shot +pass that re-folds the legacy fields into URIs on every existing item. Both honour +`--uri-match`/`--interpret-uri-syntax` and `-o`/`-c`. + +Too many subdomains autofilling together? Under base-domain matching, every login under `*.example.com` surfaces on any +`example.com` subdomain. `kp2bw --report-uris keepass` (or `bitwarden`) prints a read-only collision report — +registrable domains with multiple hosts — so you can see which entries pile up and switch those to **Host** match (or +flip your Bitwarden account's default URI match detection to Host). It changes nothing; it just lists. Every migrated item carries a plain-text `KP2BW_ID` custom field — the KeePass UUID kp2bw uses to match entries on re-runs so nothing duplicates. Once you're satisfied the migration is complete and you're ready to fully adopt diff --git a/src/kp2bw/bw_serve.py b/src/kp2bw/bw_serve.py index 957d27c..1b7b41f 100644 --- a/src/kp2bw/bw_serve.py +++ b/src/kp2bw/bw_serve.py @@ -22,6 +22,7 @@ from ._console import console from .bw_types import BwCollection, BwFolder, BwItemCreate, BwItemResponse from .exceptions import BitwardenClientError +from .uri_mapping import UriMatchValue, remap_item_fields_to_uris logger = logging.getLogger(__name__) @@ -114,6 +115,17 @@ class StripResult(NamedTuple): stripped: int +class MigrateResult(NamedTuple): + """Outcome of a :meth:`BitwardenServeClient.migrate_url_fields_to_uris` pass. + + *scanned* is every in-scope item inspected; *migrated* is the subset that + carried ``KP2A_URL*`` / ``AndroidApp`` fields and was rewritten to URIs. + """ + + scanned: int + migrated: int + + # Actionable message shown when the Bitwarden CLI cannot be located. BW_NOT_FOUND_MSG: str = ( "Bitwarden CLI ('bw') not found on your PATH. Install it and make sure " @@ -333,25 +345,65 @@ def terminate_serve( catches a worker that outlived or re-parented away from its wrapper. """ if process.poll() is None: - if via_shell and os.name == "nt": - # terminate() would reach only the cmd.exe wrapper; take the tree. - _ = subprocess.run( - ["taskkill", "/F", "/T", "/PID", str(process.pid)], - check=False, - capture_output=True, - stdin=subprocess.DEVNULL, - ) - try: - _ = process.wait(timeout=timeout) - except subprocess.TimeoutExpired: - logger.warning("bw serve did not exit after taskkill /T") + if os.name == "nt": + if via_shell: + # terminate() would reach only the cmd.exe wrapper; take the tree. + _ = subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(process.pid)], + check=False, + capture_output=True, + stdin=subprocess.DEVNULL, + ) + try: + _ = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning("bw serve did not exit after taskkill /T") + else: + process.terminate() + try: + _ = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning("bw serve did not exit on SIGTERM, sending SIGKILL") + process.kill() + try: + _ = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning("bw serve did not exit after SIGKILL") else: - process.terminate() + # POSIX: bw is commonly a node launcher that spawns a worker; killing + # only the tracked PID orphans the worker -- it keeps the port and, + # when kp2bw's stdout is a pipe, holds it open so the parent pipeline + # never reaches EOF (a multi-minute "still running" hang). bw serve + # runs in its own session (start_new_session=True), so when the + # process leads its own group we signal the whole group to take the + # launcher and worker down together. + try: + pgid = os.getpgid(process.pid) + except ProcessLookupError: + pgid = None + + def _signal(sig: int) -> None: + # Group-signal ONLY a real group leader (getpgid == pid, what + # start_new_session guarantees); otherwise a single-PID kill, so + # we never signal kp2bw's own group (e.g. a process a caller + # spawned without its own session). + if pgid is not None and pgid == process.pid: + os.killpg(pgid, sig) + else: + os.kill(process.pid, sig) + + try: + _signal(signal.SIGTERM) + except ProcessLookupError: + pass try: _ = process.wait(timeout=timeout) except subprocess.TimeoutExpired: logger.warning("bw serve did not exit on SIGTERM, sending SIGKILL") - process.kill() + try: + _signal(signal.SIGKILL) + except ProcessLookupError: + pass try: _ = process.wait(timeout=timeout) except subprocess.TimeoutExpired: @@ -678,6 +730,12 @@ def _start_serve(self, session: str | None = None) -> None: stderr=subprocess.PIPE, cwd=self._bw_cwd, env=env, + # POSIX: run bw serve in its own session/process group so teardown + # can kill the launcher *and* its node worker together (see + # terminate_serve). Without this an orphaned worker keeps the port + # and, when our stdout is a pipe, holds it open -> the parent + # pipeline hangs. Ignored on Windows (taskkill /T handles the tree). + start_new_session=True, ) except FileNotFoundError as exc: raise BitwardenClientError(BW_NOT_FOUND_MSG) from exc @@ -965,6 +1023,48 @@ def strip_field_from_items(self, field_name: str) -> StripResult: ) return StripResult(scanned=len(items), stripped=stripped) + def migrate_url_fields_to_uris( + self, *, plain_match: UriMatchValue, interpret_syntax: bool + ) -> MigrateResult: + """Re-fold legacy ``KP2A_URL*`` / ``AndroidApp`` custom fields into URIs. + + The Bitwarden-only upgrade pass for users who imported before URL folding + and do not want to re-import: each in-scope login item carrying those + fields is rewritten so they become login URIs (appended to existing ones, + de-duplicated) and the redundant fields are dropped. Scope mirrors a + migration (the configured org/collection, else the personal vault). Items + without such fields are left untouched; only changed items are PUT. + Safe to repeat -- a second pass finds nothing left to migrate. + """ + items = self.list_items( + organization_id=self._org_id, + collection_id=self._collection_id, + ) + migrated = 0 + for item in items: + if item.get("type") != _BW_ITEM_TYPE_LOGIN: + continue + login = item.get("login") + if login is None: + continue + new_fields, new_uris, changed = remap_item_fields_to_uris( + item.get("fields") or [], + login.get("uris") or [], + plain_match=plain_match, + interpret_syntax=interpret_syntax, + ) + if not changed: + continue + item["fields"] = new_fields + login["uris"] = new_uris + item["login"] = login + self.update_item(item["id"], item) + migrated += 1 + logger.info( + f"Migrated URL fields to URIs on {migrated} of {len(items)} scanned items" + ) + return MigrateResult(scanned=len(items), migrated=migrated) + def create_items_batch( self, entries: Mapping[str, tuple[str | None, BwItemCreate]], diff --git a/src/kp2bw/cli.py b/src/kp2bw/cli.py index 3dfcd92..ea00d07 100644 --- a/src/kp2bw/cli.py +++ b/src/kp2bw/cli.py @@ -7,15 +7,23 @@ from pathlib import Path from typing import NoReturn -from dotenv import find_dotenv, load_dotenv +from dotenv import dotenv_values, find_dotenv from rich.logging import RichHandler from rich.markup import escape from . import VERBOSE, __title__, __version__ from ._console import console from .bw_serve import KP2BW_ID_FIELD_NAME, BitwardenServeClient, ensure_bw_available -from .convert import Converter +from .convert import Converter, collect_keepass_uris from .exceptions import BitwardenClientError, ConversionError +from .uri_mapping import ( + UriMatchValue, + collision_groups, + match_value_names, + parse_match_name, + registrable_domain, + uri_host, +) logger = logging.getLogger(__name__) @@ -84,15 +92,22 @@ def _load_dotenv() -> str | None: Returns the path that was loaded, or ``None`` when no ``.env`` is found. ``usecwd=True`` anchors the search at the user's working directory rather than this module's install location, so an installed ``kp2bw`` still picks - up the project ``.env``. ``override`` is left at its default of ``False`` - so a real shell environment variable always wins over a file entry: a - ``.env`` value simply occupies the env tier of the documented - CLI flag > env var > default precedence. + up the project ``.env``. + + A real, *non-empty* shell variable still wins over a file entry (the + documented CLI flag > env var > default precedence). But a variable that is + set to the **empty string** must not shadow a ``.env`` value: that is almost + never intended and produces a baffling "X is required" when the file clearly + has it. So unlike a plain ``load_dotenv(override=False)`` -- which treats an + empty export as "set" and skips it -- this fills any key that is currently + unset *or empty* from the file, leaving non-empty exports untouched. """ dotenv_path = find_dotenv(usecwd=True) if not dotenv_path: return None - _ = load_dotenv(dotenv_path) + for key, value in dotenv_values(dotenv_path).items(): + if value is not None and not os.environ.get(key): + os.environ[key] = value return dotenv_path @@ -229,6 +244,32 @@ def _argparser() -> MyArgParser: action="store_true", default=None, ) + parser.add_argument( + "--uri-match", + dest="uri_match", + metavar="MODE", + choices=match_value_names(), + help=( + "Match mode for plain URLs migrated into login URIs: " + "domain|host|startswith|exact|regex|never|default. 'default' (the " + "default) leaves match unset so Bitwarden uses your account default " + "-- what Bitwarden itself writes; 'domain' forces base-domain to " + "replicate KeePassXC's host-based matching regardless. Quoted-exact " + "and wildcard URLs keep their own modes (env: KP2BW_URI_MATCH)" + ), + default=None, + ) + parser.add_argument( + "--interpret-uri-syntax", + dest="interpret_uri_syntax", + help=( + "Interpret KeePassXC URL syntax on additional URLs — double-quoted as " + "exact, '*' as wildcard (default: on). --no-interpret-uri-syntax " + "imports every URL as a plain string (env: KP2BW_INTERPRET_URI_SYNTAX)" + ), + action=BooleanOptionalAction, + default=None, + ) parser.add_argument( "--strip-ids", dest="strip_ids", @@ -243,6 +284,34 @@ def _argparser() -> MyArgParser: action="store_true", default=None, ) + parser.add_argument( + "--migrate-uris", + dest="migrate_uris", + help=( + "Upgrade existing Bitwarden items in place: re-fold legacy " + "KP2A_URL*/AndroidApp custom fields into login URIs, then exit (no " + "migration, no KeePass database needed). For users who imported " + "before URL folding and don't want to re-import. Honors --uri-match / " + "--interpret-uri-syntax and -o/-c for scope (env: KP2BW_MIGRATE_URIS)" + ), + action="store_true", + default=None, + ) + parser.add_argument( + "--report-uris", + dest="report_uris", + metavar="SOURCE", + choices=("keepass", "bitwarden"), + help=( + "Print a read-only URI collision report and exit (changes nothing): " + "groups login URLs by registrable domain and lists the ones with " + "multiple hosts -- the entries that all surface together under " + "Bitwarden base-domain match. SOURCE is 'keepass' (reads the KeePass " + "db) or 'bitwarden' (reads the live vault). Honors -o/-c for the " + "bitwarden source (env: KP2BW_REPORT_URIS)" + ), + default=None, + ) parser.add_argument( "-y", "--yes", @@ -362,6 +431,122 @@ def _run_strip_ids( ) +def _run_migrate_uris( + *, + bitwarden_password_arg: str | None, + org_id: str | None, + collection_id: str | None, + skip_confirm: bool, + uri_match: UriMatchValue, + interpret_uri_syntax: bool, +) -> None: + """Upgrade existing items: re-fold legacy URL/app custom fields into URIs. + + The Bitwarden-only counterpart of the new-import URL folding, for users who + imported before it and do not want to re-import. No KeePass database is read; + scope follows ``-o``/``-c``. The change is additive and idempotent (the + field's data survives as a URI), so the confirmation is skippable with ``-y``. + Errors surface as an actionable message rather than a traceback. + """ + scope = _describe_scope(org_id, collection_id) + try: + if not skip_confirm: + console.print( + "This re-folds legacy [bold]KP2A_URL*[/bold]/[bold]AndroidApp[/bold] " + f"custom fields into login URIs on items in [bold]{escape(scope)}[/bold] " + "(the field data is preserved as a URI, then the field removed)." + ) + if not _confirm("Migrate URL fields to URIs? [y/n]: "): + console.print("[yellow]Aborted; nothing changed.[/yellow]") + sys.exit(0) + + bw_pw = _read_password( + bitwarden_password_arg, "Please enter your Bitwarden password: " + ) + with BitwardenServeClient( + bw_pw, org_id=org_id, collection_id=collection_id + ) as bw: + result = bw.migrate_url_fields_to_uris( + plain_match=uri_match, interpret_syntax=interpret_uri_syntax + ) + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted.[/yellow]") + sys.exit(130) + except (BitwardenClientError, ConversionError) as exc: + _fail(exc) + + if result.migrated: + console.print( + f"[green]Migrated URL fields to URIs on {result.migrated} of " + f"{result.scanned} item(s).[/green]" + ) + else: + console.print( + f"No items carried legacy URL fields in {escape(scope)} " + f"({result.scanned} scanned); nothing to do." + ) + + +def _print_uri_report(uris: list[str], source: str) -> None: + """Print the read-only URI collision report to stdout (changes nothing). + + Groups hosts by registrable domain and lists the multi-host groups -- the + logins that all surface together under Bitwarden's base-domain matching. + The header goes through the Rich console; the data lines are plain ``print`` + so they copy/paste cleanly. + """ + groups = collision_groups(uris) + bases = {registrable_domain(host) for u in uris if (host := uri_host(u))} + console.print( + f"[bold]URI collision report ({source})[/bold]: {len(uris)} URIs across " + f"{len(bases)} base domain(s); [bold]{len(groups)}[/bold] have multiple " + "hosts (these all surface together under Bitwarden base-domain match -- " + "switch them to Host match to separate them)." + ) + if not groups: + print("# no base-domain collisions") + return + for base, hosts in sorted(groups.items(), key=lambda kv: (-len(kv[1]), kv[0])): + print(f"{base} ({len(hosts)}): " + ", ".join(hosts)) + + +def _run_report_uris_bitwarden( + *, + bitwarden_password_arg: str | None, + org_id: str | None, + collection_id: str | None, +) -> None: + """Read the live Bitwarden vault and print the URI collision report. + + Read-only Bitwarden mode (no KeePass): lists in-scope items, gathers their + login URIs, and reports the base-domain collisions. Scope follows ``-o``/``-c``. + """ + bw_pw = _read_password( + bitwarden_password_arg, "Please enter your Bitwarden password: " + ) + try: + with BitwardenServeClient( + bw_pw, org_id=org_id, collection_id=collection_id + ) as bw: + items = bw.list_items(organization_id=org_id, collection_id=collection_id) + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted.[/yellow]") + sys.exit(130) + except (BitwardenClientError, ConversionError) as exc: + _fail(exc) + + uris: list[str] = [] + for item in items: + login = item.get("login") + if login is None: + continue + for entry in login.get("uris") or []: + value = entry.get("uri") + if value: + uris.append(value) + _print_uri_report(uris, "bitwarden") + + # Third-party loggers whose DEBUG/INFO chatter is valuable in the always-on file # log but noise on the console. The loggers stay at DEBUG (so the file keeps # everything); console handlers attach a ConsoleNoiseFilter to drop their @@ -528,6 +713,21 @@ def main() -> None: strip_ids = _resolve_bool_option( args.strip_ids, "KP2BW_STRIP_IDS", default=False ) + migrate_uris = _resolve_bool_option( + args.migrate_uris, "KP2BW_MIGRATE_URIS", default=False + ) + raw_report = _with_env(args.report_uris, "KP2BW_REPORT_URIS") + report_uris = raw_report.strip().lower() if raw_report else None + if report_uris not in (None, "keepass", "bitwarden"): + raise ValueError( + f"Invalid KP2BW_REPORT_URIS={raw_report!r}; use 'keepass' or 'bitwarden'" + ) + interpret_uri_syntax = _resolve_bool_option( + args.interpret_uri_syntax, "KP2BW_INTERPRET_URI_SYNTAX", default=True + ) + uri_match: UriMatchValue = parse_match_name( + _with_env(args.uri_match, "KP2BW_URI_MATCH") or "default" + ) verbose = _resolve_bool_option(args.verbose, "KP2BW_VERBOSE", default=False) debug = _resolve_bool_option(args.debug, "KP2BW_DEBUG", default=False) except ValueError as exc: @@ -550,9 +750,10 @@ def main() -> None: _argparser().print_help() sys.exit(2) - # --strip-ids is a Bitwarden-only finalize step: no KeePass database is read, - # so the path requirement (and the KeePass password prompt below) is skipped. - if not strip_ids and not args.keepass_file: + # Bitwarden-only modes read no KeePass database, so the path requirement + # (and the KeePass password prompt below) is skipped for them. + bitwarden_only = strip_ids or migrate_uris or report_uris == "bitwarden" + if not bitwarden_only and not args.keepass_file: _ = sys.stderr.write( "ERROR: KeePass database path is required " "(positional FILE or KP2BW_KEEPASS_FILE)\n\n" @@ -578,6 +779,26 @@ def main() -> None: if log_path is not None: logger.info(f"Writing full debug log to {log_path}") + # --report-uris keepass reads only the KeePass database (no Bitwarden, no bw + # CLI), so it short-circuits before the bw availability check. + if report_uris == "keepass": + kp_pw = _read_password( + args.kp_pw, "Please enter your KeePass 2.x db password: " + ) + assert args.keepass_file is not None + try: + uris = collect_keepass_uris( + args.keepass_file, + kp_pw, + args.kp_keyfile, + uri_match=uri_match, + interpret_uri_syntax=interpret_uri_syntax, + ) + except (OSError, ValueError) as exc: + _fail(exc) + _print_uri_report(uris, "keepass") + return + # Verify the Bitwarden CLI is available before prompting for secrets, so the # user isn't asked for passwords only to hit a missing-`bw` failure later. try: @@ -585,6 +806,16 @@ def main() -> None: except BitwardenClientError as exc: _fail(exc) + # --report-uris bitwarden reads the live vault (no KeePass) and prints a + # read-only collision report. + if report_uris == "bitwarden": + _run_report_uris_bitwarden( + bitwarden_password_arg=args.bw_pw, + org_id=args.bw_org, + collection_id=args.bw_coll, + ) + return + # --strip-ids short-circuits migration entirely: it only touches Bitwarden # (remove kp2bw's dedup stamps), so it needs neither the bw-setup spiel nor a # KeePass password. It handles its own confirmation and Bitwarden password. @@ -597,6 +828,18 @@ def main() -> None: ) return + # --migrate-uris is likewise a Bitwarden-only upgrade pass (no KeePass). + if migrate_uris: + _run_migrate_uris( + bitwarden_password_arg=args.bw_pw, + org_id=args.bw_org, + collection_id=args.bw_coll, + skip_confirm=skip_confirm, + uri_match=uri_match, + interpret_uri_syntax=interpret_uri_syntax, + ) + return + # bw confirmation if not skip_confirm: confirm: str | None = None @@ -641,6 +884,8 @@ def main() -> None: migrate_metadata=migrate_metadata, update_existing=update_existing, include_oversize_secrets=include_oversize_secrets, + uri_match=uri_match, + interpret_uri_syntax=interpret_uri_syntax, ) try: failures = c.convert() diff --git a/src/kp2bw/convert.py b/src/kp2bw/convert.py index 7e40b88..8026c19 100644 --- a/src/kp2bw/convert.py +++ b/src/kp2bw/convert.py @@ -31,6 +31,13 @@ ) from .exceptions import BitwardenClientError, ConversionError from .otp import resolve_otp +from .uri_mapping import ( + UriMatchValue, + build_login_uris, + is_android_app_key, + is_url_attribute_key, + url_attribute_index, +) logger = logging.getLogger(__name__) @@ -115,6 +122,65 @@ def _print_summary( ) +def _entry_url_inputs(entry: Entry) -> tuple[str, list[str], list[str]]: + """``(primary url, additional URLs, android packages)`` for a KeePass entry. + + Mirrors the extraction in :meth:`Converter._add_bw_entry_to_entries_dict` + (suffix-ordered for determinism) so callers that need an entry's would-be + ``login.uris`` -- the report and REF merging -- fold the same inputs the + migration does. + """ + url_attrs: list[tuple[int, str]] = [] + app_attrs: list[tuple[int, str]] = [] + for key, value in entry.custom_properties.items(): + if value and is_url_attribute_key(key): + bucket = app_attrs if is_android_app_key(key) else url_attrs + bucket.append((url_attribute_index(key), value)) + return ( + entry.url or "", + [v for _, v in sorted(url_attrs)], + [v for _, v in sorted(app_attrs)], + ) + + +def collect_keepass_uris( + keepass_file_path: str, + keepass_password: str | None, + keepass_keyfile_path: str | None, + *, + uri_match: UriMatchValue = None, + interpret_uri_syntax: bool = True, +) -> list[str]: + """Return the login-URI values migration would write for every entry. + + Read-only helper for the ``--report-uris keepass`` collision report: each + entry's primary URL plus its ``KP2A_URL*`` / ``URL_*`` / ``AndroidApp*`` + attributes are run through the same :func:`build_login_uris` the migration + uses, so the report previews exactly the URIs that would be written -- + including quote/wildcard transforms and dropped non-web schemes -- not the + raw values. + """ + kp = PyKeePass( + filename=keepass_file_path, + password=keepass_password, + keyfile=keepass_keyfile_path, + ) + uris: list[str] = [] + for entry in kp.entries: + primary, additional, android = _entry_url_inputs(entry) + uris.extend( + bw_uri["uri"] + for bw_uri in build_login_uris( + primary_url=primary, + additional_urls=additional, + android_packages=android, + plain_match=uri_match, + interpret_syntax=interpret_uri_syntax, + ) + ) + return uris + + class Converter: _keepass_file_path: str _keepass_password: str | None @@ -130,6 +196,8 @@ class Converter: _migrate_metadata: bool _update_existing: bool _include_oversize_secrets: bool + _uri_match: UriMatchValue + _interpret_uri_syntax: bool _kp_ref_entries: list[Entry] _entries: dict[str, EntryValue] _member_reference_resolving_dict: dict[str, str] @@ -154,6 +222,8 @@ def __init__( migrate_metadata: bool = True, update_existing: bool = True, include_oversize_secrets: bool = False, + uri_match: UriMatchValue = None, + interpret_uri_syntax: bool = True, ) -> None: """Initialise the converter with KeePass source and Bitwarden target settings.""" self._keepass_file_path = keepass_file_path @@ -170,6 +240,8 @@ def __init__( self._migrate_metadata = migrate_metadata self._update_existing = update_existing self._include_oversize_secrets = include_oversize_secrets + self._uri_match = uri_match + self._interpret_uri_syntax = interpret_uri_syntax self._kp_ref_entries = [] self._entries = {} self._ref_entries_by_uuid = {} @@ -236,9 +308,22 @@ def _create_bw_python_object( password: str, custom_properties: dict[str, FieldSpec], fido2_credentials: list[BwFido2Credential] | None = None, + additional_urls: list[str] | None = None, + android_packages: list[str] | None = None, ) -> BwItemCreate: - """Build a Bitwarden item dict from individual entry fields.""" - uris: list[BwUri] = [BwUri(uri=url, match=None)] if url else [] + """Build a Bitwarden item dict from individual entry fields. + + The primary ``url`` plus any KeePass(XC) additional URLs and Android + package ids are folded into ``login.uris`` with per-URI match modes (see + :mod:`kp2bw.uri_mapping`), rather than left as inert custom fields. + """ + uris: list[BwUri] = build_login_uris( + primary_url=url, + additional_urls=additional_urls or [], + android_packages=android_packages or [], + plain_match=self._uri_match, + interpret_syntax=self._interpret_uri_syntax, + ) login: BwItemLogin = BwItemLogin( uris=uris, username=username, @@ -359,10 +444,21 @@ def _add_bw_entry_to_entries_dict( logger.warning(f"{entry.title or '_untitled'}: {warning}") custom_properties: dict[str, FieldSpec] = {} + # KeePass(XC) additional URLs / Android packages are folded into + # login.uris (not custom fields); collected here keyed by their suffix so + # the emitted URI order is deterministic across re-runs. + url_attrs: list[tuple[int, str]] = [] + app_attrs: list[tuple[int, str]] = [] for key, value in custom_props.items(): # Skip passkey attributes and OTP fields folded into login.totp. if key.startswith(KPEX_PASSKEY_PREFIX) or key in otp_result.consumed_keys: continue + # Route URL/app attributes to login.uris instead of custom fields. + if is_url_attribute_key(key): + if value: + bucket = app_attrs if is_android_app_key(key) else url_attrs + bucket.append((url_attribute_index(key), value)) + continue # A value over the item-size limit is offloaded to a .txt # attachment below (mirroring the notes handling); keep it out of the # inline fields entirely so it is not also stored inline, which would @@ -415,6 +511,8 @@ def _add_bw_entry_to_entries_dict( password=entry.password if entry.password else "", custom_properties=custom_properties, fido2_credentials=fido2_credentials, + additional_urls=[v for _, v in sorted(url_attrs)], + android_packages=[v for _, v in sorted(app_attrs)], ) # get attachments to store later on. A value over the inline size limit @@ -692,12 +790,22 @@ def _resolve_single_ref_entry(self, kp_entry: Entry) -> EntryValue | None: break if username_and_password_match and ref_result is not None: - # => add url to bw_item => username / pw identical + # => merge this entry's URLs into the referent (same creds). Fold + # the full set (primary + KP2A_URL*/URL_*/AndroidApp*) the same way + # migration would, and append only URIs not already present. _, _, ref_item, _ = self._unpack_entry(ref_result) - if kp_entry.url: - ref_item["login"]["uris"].append( - BwUri(uri=kp_entry.url, match=None) - ) + existing_uris = ref_item["login"]["uris"] + existing_values = {u.get("uri") for u in existing_uris} + primary, additional, android = _entry_url_inputs(kp_entry) + for bw_uri in build_login_uris( + primary_url=primary, + additional_urls=additional, + android_packages=android, + plain_match=self._uri_match, + interpret_syntax=self._interpret_uri_syntax, + ): + if bw_uri["uri"] not in existing_values: + existing_uris.append(bw_uri) canonical = ref_result else: # => create new bitwarden item diff --git a/src/kp2bw/uri_mapping.py b/src/kp2bw/uri_mapping.py new file mode 100644 index 0000000..6200833 --- /dev/null +++ b/src/kp2bw/uri_mapping.py @@ -0,0 +1,441 @@ +"""Map KeePass(XC) entry URLs onto Bitwarden login URIs with per-URI match modes. + +KeePass2Android / KeePassXC store a primary ``URL`` plus any number of +*additional URLs* in ``KP2A_URL`` / ``KP2A_URL_`` custom attributes, and +Android package ids in ``AndroidApp`` attributes. Historically kp2bw copied +those verbatim into Bitwarden *custom fields*, where they are inert noise. + +This module converts them into the thing Bitwarden actually uses for autofill: +extra entries in ``login.uris``. Each emitted URI carries a ``match`` mode chosen +to reproduce how KeePassXC itself would have matched that URL string: + +* KeePassXC's default matching is **host-based** -- a stored URL matches its base + domain and all subdomains, ignoring path/query/fragment for inclusion. The + faithful Bitwarden equivalent is **base domain (0)**, which is the default for + a plain string. +* KeePassXC honours two inline syntaxes **on additional URLs only**: a + double-quoted string is an *exact* match, and a ``*`` is a wildcard. These map + to **exact (3)** and **starts-with (2)** / **regex (4)** respectively. +* Non-web schemes (``keepassxc://``, ``cmd://``, ``kdbx://``, ``file://``) and + unresolved ``{REF:...}`` placeholders are dropped -- they are not site URLs and + would only leave dead URIs behind. + +Two orthogonal knobs steer the behaviour: + +* ``plain_match`` -- what a no-encoded-intent (plain) string becomes. Defaults to + ``None`` (defer to the user's Bitwarden account default -- what Bitwarden itself + writes on export); pass ``0`` for base-domain to faithfully replicate + KeePassXC's host-based matching regardless of the account default. +* ``interpret_syntax`` -- whether the quote/wildcard conventions are honoured at + all. With it off, every additional URL is treated as a plain string (the + scheme/garbage drops still apply), for a deliberately literal import. + +Important caveat on regex: Bitwarden applies a single regex to the **whole URL** +(unanchored, case-insensitive), whereas KeePassXC regexes host and path +*separately*. The wildcard->regex translation here is therefore a best-effort +whole-URL pattern, not a byte-for-byte copy of KeePassXC's internal regex; +complex wildcards are emitted with a warning so the user can review them. +""" + +import logging +import re +from collections.abc import Iterable +from typing import Literal + +from .bw_types import BwField, BwUri + +logger = logging.getLogger(__name__) + +# Bitwarden URI match-detection modes; ``None`` means "use the account default". +type UriMatchValue = Literal[0, 1, 2, 3, 4, 5] | None + +# Symbolic names accepted by the --uri-match / KP2BW_URI_MATCH knob. ``domain`` +# is the faithful KeePassXC-replication setting; ``default``/``null`` defer to the +# user's Bitwarden account default. +_MATCH_NAMES: dict[str, UriMatchValue] = { + "domain": 0, + "host": 1, + "startswith": 2, + "exact": 3, + "regex": 4, + "never": 5, + "default": None, + "null": None, +} + +# Additional-URL attribute names: the KeePass2Android/KeePassXC ``KP2A_URL`` / +# ``KP2A_URL_`` convention, plus the plainer ``URL`` / ``URL_`` convention +# some entries use. Both are unambiguously "extra autofill URLs"; free-text +# labels like "API Url" or "Alt. URL" are deliberately *not* matched -- folding +# those would wrongly autofill API endpoints and other metadata. +_ADDITIONAL_URL_RE = re.compile(r"^(?:KP2A_URL|URL)(_\d+)?$") +# Android package attribute names. The underscore is optional so both the +# ``AndroidApp_`` and the no-underscore ``AndroidApp`` variants are caught. +_ANDROID_APP_RE = re.compile(r"^AndroidApp(_?\d+)?$") + +# Schemes that are never site URLs and must not become Bitwarden URIs. +_DROP_SCHEMES: tuple[str, ...] = ("keepassxc://", "cmd://", "kdbx://", "file://") +# Unresolved KeePass field-reference placeholder. +_KP_REF_MARKER = "{REF:" +# Characters KeePassXC rejects in a URL (`isUrlValid`); we drop strings carrying them. +_ILLEGAL_URL_CHARS = re.compile(r"[<>^`{|}]") +_ANDROID_APP_SCHEME = "androidapp://" + +# Curated two-level public suffixes for the registrable-domain heuristic behind +# the --report-uris collision report. Not the full Public Suffix List -- just the +# common multi-level TLDs -- so e.g. ``10bis.co.il`` collapses to ``10bis.co.il`` +# rather than ``co.il``. A miss only mis-groups a report line; it never touches +# migration behaviour. +_TWO_LEVEL_SUFFIXES: frozenset[str] = frozenset({ + "co.uk", + "org.uk", + "gov.uk", + "ac.uk", + "me.uk", + "net.uk", + "sch.uk", + "co.il", + "org.il", + "net.il", + "ac.il", + "gov.il", + "co.jp", + "or.jp", + "ne.jp", + "ac.jp", + "go.jp", + "co.kr", + "or.kr", + "co.nz", + "org.nz", + "govt.nz", + "co.za", + "org.za", + "co.in", + "net.in", + "org.in", + "co.id", + "co.th", + "in.th", + "com.au", + "net.au", + "org.au", + "edu.au", + "gov.au", + "com.br", + "net.br", + "com.mx", + "com.tr", + "com.cn", + "net.cn", + "com.sg", + "com.hk", + "com.tw", + "com.ar", + "com.co", + "com.ua", + "com.pl", + "com.ru", + "co.cr", +}) + + +def match_value_names() -> tuple[str, ...]: + """Return the accepted --uri-match names, for help text and arg validation.""" + return tuple(_MATCH_NAMES) + + +def parse_match_name(name: str) -> UriMatchValue: + """Resolve a symbolic match name to its Bitwarden value (or ``None``). + + Raises :class:`ValueError` on an unknown name so the CLI/env layer can report + it with the list of valid options. + """ + key = name.strip().lower() + if key not in _MATCH_NAMES: + valid = ", ".join(_MATCH_NAMES) + raise ValueError(f"Invalid URI match mode {name!r}; choose one of: {valid}") + return _MATCH_NAMES[key] + + +def is_url_attribute_key(key: str) -> bool: + """True for KeePass keys that hold URLs/app ids handled as login URIs. + + Used by the converter to keep these out of the custom-fields section -- they + are folded into ``login.uris`` instead. + """ + return bool(_ADDITIONAL_URL_RE.match(key)) or bool(_ANDROID_APP_RE.match(key)) + + +def is_additional_url_key(key: str) -> bool: + """True for ``KP2A_URL`` / ``KP2A_URL_`` additional-URL attribute names.""" + return bool(_ADDITIONAL_URL_RE.match(key)) + + +def is_android_app_key(key: str) -> bool: + """True for ``AndroidApp`` / ``AndroidApp_`` package attribute names.""" + return bool(_ANDROID_APP_RE.match(key)) + + +def url_attribute_index(key: str) -> int: + """Stable sort index for a URL attribute: bare name first, then by suffix. + + ``KP2A_URL`` -> -1, ``KP2A_URL_2`` -> 2, ``KP2A_URL_10`` -> 10, and the + no-underscore ``AndroidApp1`` -> 1, so the emitted URI order is deterministic + across runs (the dedup content-diff compares the ``uris`` list positionally, + so a stable order avoids spurious "changed" runs). + """ + match = re.search(r"(\d+)$", key) + return int(match.group(1)) if match else -1 + + +def _android_uri(package: str) -> BwUri | None: + """Build an ``androidapp://`` URI from a package id, or ``None`` if empty. + + ``match`` is left unset (account default): Bitwarden matches Android apps by + package id, so a URL match mode does not apply. + """ + pkg = package.strip() + if not pkg: + return None + if not pkg.startswith(_ANDROID_APP_SCHEME): + pkg = f"{_ANDROID_APP_SCHEME}{pkg}" + return BwUri(uri=pkg) + + +def _split_packages(value: str) -> list[str]: + """Split an AndroidApp attribute into package ids (comma/whitespace separated).""" + return [p for p in re.split(r"[\s,]+", value.strip()) if p] + + +def _host_part(url: str) -> str: + """Return the host portion of *url* (between scheme:// and the first '/').""" + after_scheme = url.split("://", 1)[-1] + return after_scheme.split("/", 1)[0] + + +def _is_invalid_wildcard(glob: str) -> bool: + """Reproduce KeePassXC's wildcard validity rejections (the common subset). + + Rejects double/adjacent wildcards, all-wildcard strings, and bare TLD + wildcards like ``*.com`` -- inputs KeePassXC itself refuses, so we never emit + a Bitwarden URI for them. (Public-suffix edge cases such as ``*.co.uk`` are a + documented minor divergence.) + """ + if "**" in glob or "*.*" in glob: + return True + # A string that is only wildcards / separators carries no real target. + if not glob.replace("*", "").replace(".", "").replace("/", "").replace(":", ""): + return True + host = _host_part(glob) + # `*.com` style: a wildcard label in front of a single bare TLD label. + return host.startswith("*.") and "." not in host[2:] + + +def _trailing_path_wildcard_prefix(s: str) -> str | None: + """If *s* is a trailing-path-only wildcard, return the literal prefix. + + e.g. ``https://host/app/*`` -> ``https://host/app/`` for a starts-with (2) + match. Returns ``None`` when the wildcard is in the host or appears interior, + which need the regex path instead. + """ + if s.count("*") != 1 or not s.endswith("*"): + return None + if "*" in _host_part(s): + return None + return s[:-1] + + +def _glob_to_regex(glob: str) -> str: + """Best-effort whole-URL regex for a wildcard URL (Bitwarden match mode 4). + + Bitwarden applies one regex to the entire URL (unanchored, case-insensitive), + unlike KeePassXC's separate host/path regexes, so this expands each ``*`` to + ``.*`` over the regex-escaped literal rather than copying KeePassXC's internal + pattern. Faithful enough for autofill on the intended URLs; emitted with a + warning by the caller for human review. + """ + return ".*".join(re.escape(part) for part in glob.split("*")) + + +def _classify_additional_url( + raw: str, *, plain_match: UriMatchValue, interpret_syntax: bool +) -> BwUri | None: + """Map a single additional-URL string to a Bitwarden URI, or ``None`` to drop. + + Drops (scheme/reference/garbage) apply in every mode; the quote and wildcard + interpretations are skipped when *interpret_syntax* is off, so the string is + emitted as a plain URI instead. + """ + s = raw.strip() + if not s: + return None + # Reasons are logged WITHOUT the URL value: this lands in the always-on debug + # log, which users are encouraged to share for troubleshooting, and entry URLs + # are vault data. + if s.lower().startswith(_DROP_SCHEMES): + logger.debug("Dropping a non-web-scheme URL from URIs") + return None + if _KP_REF_MARKER in s: + logger.debug("Dropping an unresolved field-reference URL from URIs") + return None + if _ILLEGAL_URL_CHARS.search(s): + logger.debug("Dropping a URL with illegal characters from URIs") + return None + + if interpret_syntax: + if len(s) >= 2 and s.startswith('"') and s.endswith('"'): + inner = s[1:-1] + if not inner or "*" in inner: + logger.debug("Dropping an invalid quoted-exact URL") + return None + return BwUri(uri=inner, match=3) + if "*" in s: + if _is_invalid_wildcard(s): + logger.debug("Dropping an invalid wildcard URL") + return None + prefix = _trailing_path_wildcard_prefix(s) + if prefix is not None: + return BwUri(uri=prefix, match=2) + logger.warning( + "A wildcard URL was migrated as a regex match; review your " + "wildcard entries in Bitwarden -- complex wildcard fidelity is " + "best-effort" + ) + return BwUri(uri=_glob_to_regex(s), match=4) + + return BwUri(uri=s, match=plain_match) + + +def build_login_uris( + *, + primary_url: str, + additional_urls: list[str], + android_packages: list[str], + plain_match: UriMatchValue = None, + interpret_syntax: bool = True, +) -> list[BwUri]: + """Build the ordered, de-duplicated ``login.uris`` list for an entry. + + The primary URL is always treated as a plain string (KeePassXC honours the + quote/wildcard syntax only on *additional* URLs), additional URLs go through + :func:`_classify_additional_url`, and each Android package becomes an + ``androidapp://`` URI. Duplicate URI values are collapsed, first occurrence + winning, so the primary URL is never repeated by an identical alias. + """ + uris: list[BwUri] = [] + seen: set[str] = set() + + def _add(uri: BwUri | None) -> None: + if uri is None: + return + if uri["uri"] in seen: + return + seen.add(uri["uri"]) + uris.append(uri) + + primary = primary_url.strip() + if primary: + _add(BwUri(uri=primary, match=plain_match)) + for raw in additional_urls: + _add( + _classify_additional_url( + raw, plain_match=plain_match, interpret_syntax=interpret_syntax + ) + ) + for value in android_packages: + for package in _split_packages(value): + _add(_android_uri(package)) + + return uris + + +def remap_item_fields_to_uris( + fields: list[BwField], + uris: list[BwUri], + *, + plain_match: UriMatchValue = None, + interpret_syntax: bool = True, +) -> tuple[list[BwField], list[BwUri], bool]: + """Re-fold an already-imported item's URL/app *custom fields* into URIs. + + The Bitwarden-only counterpart of the new-import path, for items that predate + it: ``KP2A_URL*`` / ``AndroidApp`` custom fields are removed and converted to + login URIs, appended to the item's existing URIs and de-duplicated by value. + Returns ``(kept_fields, merged_uris, changed)`` -- *changed* is ``False`` when + the item carries no such fields, so the caller can skip an unnecessary PUT. + """ + kept_fields: list[BwField] = [] + url_attrs: list[tuple[int, str]] = [] + app_attrs: list[tuple[int, str]] = [] + for field in fields: + name = field.get("name") or "" + if not is_url_attribute_key(name): + kept_fields.append(field) + continue + value = field.get("value") or "" + if value: + bucket = app_attrs if is_android_app_key(name) else url_attrs + bucket.append((url_attribute_index(name), value)) + + if len(kept_fields) == len(fields): + return fields, uris, False + + derived = build_login_uris( + primary_url="", + additional_urls=[v for _, v in sorted(url_attrs)], + android_packages=[v for _, v in sorted(app_attrs)], + plain_match=plain_match, + interpret_syntax=interpret_syntax, + ) + existing_values = {u["uri"] for u in uris} + merged = list(uris) + [d for d in derived if d["uri"] not in existing_values] + return kept_fields, merged, True + + +def uri_host(uri: str) -> str | None: + """Extract the lowercase host from a URI, or ``None`` when it has none. + + Strips scheme, userinfo, port, and path; skips ``androidapp://`` app URIs and + bare strings without a dot (intranet labels, junk) that carry no domain. + """ + s = uri.strip().lower() + if not s or s.startswith(_ANDROID_APP_SCHEME): + return None + if "://" in s: + s = s.split("://", 1)[1] + host = s.split("/", 1)[0].split("?", 1)[0].split("#", 1)[0] + host = host.split("@")[-1].split(":", 1)[0] + return host if "." in host else None + + +def registrable_domain(host: str) -> str: + """Best-effort registrable domain (eTLD+1) of *host*. + + Uses the curated :data:`_TWO_LEVEL_SUFFIXES` set rather than the full Public + Suffix List, which is good enough for the collision report where a rare miss + only mis-groups one line. + """ + parts = [p for p in host.lower().strip(".").split(".") if p] + if len(parts) < 2: + return host + if ".".join(parts[-2:]) in _TWO_LEVEL_SUFFIXES and len(parts) >= 3: + return ".".join(parts[-3:]) + return ".".join(parts[-2:]) + + +def collision_groups(uris: Iterable[str]) -> dict[str, list[str]]: + """Group URI hosts by registrable domain, keeping only multi-host groups. + + These are the registrable domains under which more than one distinct host is + stored -- i.e. the logins that all surface together under Bitwarden's + base-domain match, and the candidates for switching to Host match. Each value + is the sorted distinct host list. + """ + groups: dict[str, set[str]] = {} + for uri in uris: + host = uri_host(uri) + if host is None: + continue + groups.setdefault(registrable_domain(host), set()).add(host) + return {base: sorted(hosts) for base, hosts in groups.items() if len(hosts) > 1} diff --git a/tests/__snapshots__/vault_after_update.json b/tests/__snapshots__/vault_after_update.json index ebd53af..d3cf065 100644 --- a/tests/__snapshots__/vault_after_update.json +++ b/tests/__snapshots__/vault_after_update.json @@ -189,6 +189,60 @@ "token": "Internet/Long Field", "type": 1 }, + { + "attachments": [], + "favorite": false, + "fields": [], + "folder": "Internet", + "login": { + "fido2Credentials": [], + "password": "multi-pass", + "totp": null, + "uris": [ + { + "match": null, + "uri": "androidapp://com.multi.app" + }, + { + "match": null, + "uri": "androidapp://com.multi.other" + }, + { + "match": 4, + "uri": "https://.*\\.wild\\.example/.*" + }, + { + "match": null, + "uri": "https://alt-one.example" + }, + { + "match": null, + "uri": "https://alt-three.example" + }, + { + "match": null, + "uri": "https://alt-two.example" + }, + { + "match": 3, + "uri": "https://exact.example/login" + }, + { + "match": null, + "uri": "https://multi.example" + }, + { + "match": 2, + "uri": "https://multi.example/area/" + } + ], + "username": "multi-user" + }, + "name": "Multi URL", + "notes": "", + "token": "Internet/Multi URL", + "type": 1 + }, { "attachments": [], "favorite": false, diff --git a/tests/__snapshots__/vault_initial.json b/tests/__snapshots__/vault_initial.json index dfa5cba..197a3e3 100644 --- a/tests/__snapshots__/vault_initial.json +++ b/tests/__snapshots__/vault_initial.json @@ -167,6 +167,60 @@ "token": "Internet/Long Field", "type": 1 }, + { + "attachments": [], + "favorite": false, + "fields": [], + "folder": "Internet", + "login": { + "fido2Credentials": [], + "password": "multi-pass", + "totp": null, + "uris": [ + { + "match": null, + "uri": "androidapp://com.multi.app" + }, + { + "match": null, + "uri": "androidapp://com.multi.other" + }, + { + "match": 4, + "uri": "https://.*\\.wild\\.example/.*" + }, + { + "match": null, + "uri": "https://alt-one.example" + }, + { + "match": null, + "uri": "https://alt-three.example" + }, + { + "match": null, + "uri": "https://alt-two.example" + }, + { + "match": 3, + "uri": "https://exact.example/login" + }, + { + "match": null, + "uri": "https://multi.example" + }, + { + "match": 2, + "uri": "https://multi.example/area/" + } + ], + "username": "multi-user" + }, + "name": "Multi URL", + "notes": "", + "token": "Internet/Multi URL", + "type": 1 + }, { "attachments": [], "favorite": false, diff --git a/tests/bw_serve_teardown_test.py b/tests/bw_serve_teardown_test.py new file mode 100644 index 0000000..7ee666c --- /dev/null +++ b/tests/bw_serve_teardown_test.py @@ -0,0 +1,54 @@ +"""Checks that `terminate_serve` actually ends the bw serve process (no orphan/hang). + +Regression: on POSIX, `bw` is often a node launcher that spawns a worker, and +killing only the tracked PID left the worker orphaned -- it kept the port and, +when kp2bw's stdout was a pipe, held it open so the parent pipeline never reached +EOF (a multi-minute "still running" hang). `bw serve` is now started in its own +session (`start_new_session=True`) and torn down by signalling the whole process +group. This drives `terminate_serve` against a real grouped process and asserts +it is gone afterwards (and that the call returns promptly, i.e. does not hang). +""" + +import os +import subprocess +import sys +import time + +from kp2bw.bw_serve import terminate_serve + + +def assert_terminate_serve_ends_grouped_process() -> None: + """A process started in its own session is dead after `terminate_serve`.""" + if os.name == "nt": + print("skip: POSIX process-group teardown test") + return + + # Mirror _start_serve: a long-lived child in its own session/process group. + proc = subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + stdin=subprocess.DEVNULL, + stderr=subprocess.PIPE, + start_new_session=True, + ) + time.sleep(0.3) + if proc.poll() is not None: + raise AssertionError("test process exited before teardown") + + start = time.monotonic() + terminate_serve(proc, timeout=5.0) + elapsed = time.monotonic() - start + + if proc.poll() is None: + proc.kill() + raise AssertionError("process still alive after terminate_serve") + if elapsed > 10: + raise AssertionError(f"terminate_serve hung ({elapsed:.1f}s)") + + +def main() -> None: + assert_terminate_serve_ends_grouped_process() + print("bw serve teardown test passed") + + +if __name__ == "__main__": + main() diff --git a/tests/cli_env_test.py b/tests/cli_env_test.py index 6cbbb1b..52d6780 100644 --- a/tests/cli_env_test.py +++ b/tests/cli_env_test.py @@ -104,8 +104,49 @@ def assert_dotenv_supplies_keepass_file() -> None: os.environ[key] = value +def assert_empty_env_var_defers_to_dotenv() -> None: + """An empty exported var must not shadow a .env value; a non-empty one wins. + + Regression: `KP2BW_KEEPASS_FILE=""` in the environment used to shadow the + .env entry (load_dotenv override=False treats empty as "set"), producing a + baffling "db required". `_load_dotenv` now fills keys that are unset *or + empty* from the file while leaving non-empty exports untouched. + """ + original_cwd = Path.cwd() + saved = os.environ.get("KP2BW_KEEPASS_FILE") + tmp = tempfile.mkdtemp() + try: + _ = (Path(tmp) / ".env").write_text( + "KP2BW_KEEPASS_FILE=from-dotenv.kdbx\n", encoding="utf-8" + ) + os.chdir(tmp) + + # Empty export -> .env wins. + os.environ["KP2BW_KEEPASS_FILE"] = "" + _ = cli._load_dotenv() + if os.environ.get("KP2BW_KEEPASS_FILE") != "from-dotenv.kdbx": + raise AssertionError( + f"empty export should defer to .env, got " + f"{os.environ.get('KP2BW_KEEPASS_FILE')!r}" + ) + + # Non-empty export -> still wins over .env. + os.environ["KP2BW_KEEPASS_FILE"] = "/real/shell/override.kdbx" + _ = cli._load_dotenv() + if os.environ.get("KP2BW_KEEPASS_FILE") != "/real/shell/override.kdbx": + raise AssertionError("non-empty export must win over .env") + finally: + os.chdir(original_cwd) + if saved is None: + _ = os.environ.pop("KP2BW_KEEPASS_FILE", None) + else: + os.environ["KP2BW_KEEPASS_FILE"] = saved + shutil.rmtree(tmp, ignore_errors=True) + + def main() -> None: assert_dotenv_supplies_keepass_file() + assert_empty_env_var_defers_to_dotenv() print("cli env test passed") diff --git a/tests/convert_ref_resolution_test.py b/tests/convert_ref_resolution_test.py index 7580fbf..ae98ba1 100644 --- a/tests/convert_ref_resolution_test.py +++ b/tests/convert_ref_resolution_test.py @@ -97,6 +97,11 @@ def group(self) -> Group | None: """Return ``None``; the double is not attached to any group.""" return None + @property + def custom_properties(self) -> dict[str, str | None]: + """Return no extra properties; only ``url`` folds into the referent.""" + return {} + class ReferenceResolutionTestConverter(Converter): """Converter wired with stubbed lookups to exercise REF resolution alone.""" diff --git a/tests/e2e_vaultwarden_test.py b/tests/e2e_vaultwarden_test.py index 591851c..7e6c730 100644 --- a/tests/e2e_vaultwarden_test.py +++ b/tests/e2e_vaultwarden_test.py @@ -26,6 +26,8 @@ ) from pykeepass import Entry, PyKeePass, create_database +from kp2bw.uri_mapping import is_url_attribute_key + logger = logging.getLogger("e2e") SENSITIVE_ARG_FLAGS = { @@ -339,6 +341,26 @@ def _create_keepass_snapshot(path: Path, password: str) -> None: # Empty Password: a login with a username but no password. _ = kp.add_entry(kp.root_group, "Empty Password", "lonely-user", "") + # Multi URL: exercises additional-URL -> login.uris folding end to end. The + # KP2A_URL*/URL_*/AndroidApp* custom fields must NOT survive as fields; they + # become login URIs with per-URI match modes -- plain -> account default + # (match unset), double-quoted -> exact, trailing-path wildcard -> + # starts-with, host wildcard -> regex -- plus androidapp:// for packages + # (bare and the no-underscore AndroidApp1 variant). The keepassxc:// value is + # a non-web scheme and is dropped entirely. + multi = kp.add_entry( + internet, "Multi URL", "multi-user", "multi-pass", url="https://multi.example" + ) + multi.set_custom_property("KP2A_URL", "https://alt-one.example") + multi.set_custom_property("KP2A_URL_1", "https://alt-two.example") + multi.set_custom_property("URL_1", "https://alt-three.example") + multi.set_custom_property("AndroidApp", "com.multi.app") + multi.set_custom_property("AndroidApp1", "androidapp://com.multi.other") + multi.set_custom_property("KP2A_URL_2", '"https://exact.example/login"') + multi.set_custom_property("KP2A_URL_3", "https://multi.example/area/*") + multi.set_custom_property("KP2A_URL_4", "https://*.wild.example/*") + multi.set_custom_property("KP2A_URL_5", "keepassxc://by-uuid/dropme") + kp.save() @@ -661,6 +683,37 @@ def _assert_comprehensive_seed(vault: NormVault) -> None: if _field(long_field, "hint")["value"] != "see recovery_codes.txt": raise AssertionError("Short field beside the long one should stay inline") + # Additional URLs / Android packages fold into login.uris with per-URI match + # modes; the KP2A_URL*/URL_*/AndroidApp* custom fields must NOT survive, and + # the keepassxc:// value must be dropped. (Runs unconditionally, so the + # non-golden CI leg validates the fold too.) + multi = _item_by_name(vault, "Multi URL") + multi_uri_list = _login(multi)["uris"] + multi_uri_values = [uri["uri"] for uri in multi_uri_list] + if len(set(multi_uri_values)) != len(multi_uri_values): + raise AssertionError(f"Multi URL has duplicate URIs: {multi_uri_values}") + multi_uris = {uri["uri"]: uri["match"] for uri in multi_uri_list} + expected_multi_uris: dict[str, int | None] = { + "https://multi.example": None, + "https://alt-one.example": None, + "https://alt-two.example": None, + "https://alt-three.example": None, + "androidapp://com.multi.app": None, + "androidapp://com.multi.other": None, + "https://exact.example/login": 3, # quoted -> exact + "https://multi.example/area/": 2, # trailing wildcard -> starts-with + "https://.*\\.wild\\.example/.*": 4, # host wildcard -> regex + } + if multi_uris != expected_multi_uris: + raise AssertionError(f"Multi URL uris/match wrong: {multi_uris}") + leftover = [ + name + for field in multi["fields"] + if is_url_attribute_key(name := field["name"] or "") + ] + if leftover: + raise AssertionError(f"Multi URL still carries legacy URL fields: {leftover}") + def main() -> None: logging.basicConfig( diff --git a/tests/migrate_uris_test.py b/tests/migrate_uris_test.py new file mode 100644 index 0000000..cee6d84 --- /dev/null +++ b/tests/migrate_uris_test.py @@ -0,0 +1,83 @@ +"""Checks the Bitwarden-only URL-field -> URI upgrade pass (`--migrate-uris`). + +`BitwardenServeClient.migrate_url_fields_to_uris` re-folds legacy KP2A_URL*/ +AndroidApp custom fields into login URIs on existing items, skips non-login +items and items without such fields, and only PUTs the ones that change. Driven +with a client double, so no live `bw serve` process is spawned. +""" + +from typing import cast + +from kp2bw.bw_serve import BitwardenServeClient +from kp2bw.bw_types import BwField, BwItemResponse, BwUri +from kp2bw.uri_mapping import is_url_attribute_key + + +def _login(item_id: str, field_names: list[str], uri: str) -> BwItemResponse: + fields = [ + cast(BwField, {"name": n, "value": "https://v.example", "type": 0}) + for n in field_names + ] + return cast( + BwItemResponse, + { + "id": item_id, + "name": item_id, + "type": 1, + "fields": fields, + "login": {"uris": [cast(BwUri, {"uri": uri, "match": 0})]}, + }, + ) + + +class _MigrateClient(BitwardenServeClient): + """Client double serving a fixed item list and recording every PUT.""" + + def __init__(self, items: list[BwItemResponse]) -> None: + self._org_id = None + self._collection_id = None + self._items = items + self.updated_ids: list[str] = [] + + def list_items( + self, + *, + folder_id: str | None = None, + organization_id: str | None = None, + collection_id: str | None = None, + ) -> list[BwItemResponse]: + return self._items + + def update_item(self, item_id: str, item: BwItemResponse) -> None: + self.updated_ids.append(item_id) + fields = [f.get("name") or "" for f in item.get("fields") or []] + if any(is_url_attribute_key(name) for name in fields): + raise AssertionError(f"{item_id} still carries a legacy URL/app field") + + +def assert_only_login_items_with_legacy_fields_migrate() -> None: + legacy = _login("legacy", ["Notes", "KP2A_URL"], "https://legacy.example") + clean = _login("clean", ["Notes"], "https://clean.example") + non_login = cast( + BwItemResponse, + {"id": "note", "name": "note", "type": 2, "fields": [], "login": None}, + ) + client = _MigrateClient([legacy, clean, non_login]) + + result = client.migrate_url_fields_to_uris(plain_match=0, interpret_syntax=True) + + if result.scanned != 3: + raise AssertionError(f"expected 3 scanned, got {result.scanned}") + if result.migrated != 1: + raise AssertionError(f"expected 1 migrated, got {result.migrated}") + if client.updated_ids != ["legacy"]: + raise AssertionError(f"only the legacy item should PUT: {client.updated_ids}") + + +def main() -> None: + assert_only_login_items_with_legacy_fields_migrate() + print("migrate uris test passed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_script_adapters.py b/tests/test_script_adapters.py index 0fb6aaf..5d4534c 100644 --- a/tests/test_script_adapters.py +++ b/tests/test_script_adapters.py @@ -39,10 +39,22 @@ def test_bw_serve_timeout_script() -> None: _run_script_main("bw_serve_timeout_test.py") +def test_bw_serve_teardown_script() -> None: + _run_script_main("bw_serve_teardown_test.py") + + def test_strip_ids_script() -> None: _run_script_main("strip_ids_test.py") +def test_uri_mapping_script() -> None: + _run_script_main("uri_mapping_test.py") + + +def test_migrate_uris_script() -> None: + _run_script_main("migrate_uris_test.py") + + def test_otp_script() -> None: _run_script_main("otp_test.py") diff --git a/tests/uri_mapping_test.py b/tests/uri_mapping_test.py new file mode 100644 index 0000000..01ae35c --- /dev/null +++ b/tests/uri_mapping_test.py @@ -0,0 +1,260 @@ +"""Checks the KeePass(XC) URL -> Bitwarden login-URI mapping (`uri_mapping`). + +Covers the per-URI match table: plain strings -> base domain (the faithful +KeePassXC default), quoted -> exact, trailing-path wildcard -> starts-with, +host/interior wildcard -> regex, non-web schemes / references / garbage dropped, +AndroidApp -> androidapp://, de-duplication, and the two config knobs +(plain-tier match value, interpret-syntax on/off). +""" + +from typing import cast + +from kp2bw.bw_types import BwField, BwUri +from kp2bw.uri_mapping import ( + build_login_uris, + collision_groups, + is_url_attribute_key, + parse_match_name, + registrable_domain, + remap_item_fields_to_uris, + uri_host, +) + + +def _uris(**kwargs: object) -> list[tuple[str, object]]: + """Run build_login_uris and return [(uri, match)] for terse assertions.""" + result = build_login_uris(**kwargs) # pyright: ignore[reportArgumentType] + return [(u["uri"], u.get("match")) for u in result] + + +def assert_match_names_parse() -> None: + """Symbolic names resolve to Bitwarden values; unknown names raise.""" + expected = { + "domain": 0, + "host": 1, + "startswith": 2, + "exact": 3, + "regex": 4, + "never": 5, + "default": None, + "null": None, + } + for name, value in expected.items(): + if parse_match_name(name) != value: + raise AssertionError(f"{name} -> {parse_match_name(name)}, want {value}") + if parse_match_name("DOMAIN") != 0: + raise AssertionError("match name should be case-insensitive") + try: + parse_match_name("nope") + except ValueError: + pass + else: + raise AssertionError("unknown match name should raise ValueError") + + +def assert_url_attribute_keys() -> None: + """URL-like (KP2A_URL*/URL*) and AndroidApp* keys are siphoned; others aren't.""" + for key in ( + "KP2A_URL", + "KP2A_URL_1", + "KP2A_URL_16", + "URL", + "URL_1", + "AndroidApp", + "AndroidApp_2", + "AndroidApp1", # no-underscore variant + ): + if not is_url_attribute_key(key): + raise AssertionError(f"{key} should be a URL attribute") + # Free-text labels and near-misses must stay as custom fields. + for key in ( + "API Url", + "Alt. URL", + "Website", + "KP2A_URLX", + "Notes", + "AndroidApplication", + ): + if is_url_attribute_key(key): + raise AssertionError(f"{key} should NOT be a URL attribute") + + +def assert_plain_urls_default_to_account_default_and_dedup() -> None: + """Plain primary + additional URLs default to match=None (account default), deduped.""" + got = _uris( + primary_url="thuisbezorgd.nl", + additional_urls=[ + "https://takeaway.com", + "https://10bis.co.il", + "thuisbezorgd.nl", + ], + android_packages=["com.takeaway.android"], + ) + expected = [ + ("thuisbezorgd.nl", None), + ("https://takeaway.com", None), + ("https://10bis.co.il", None), + ("androidapp://com.takeaway.android", None), + ] + if got != expected: + raise AssertionError(f"got {got}, want {expected}") + + +def assert_uri_match_domain_forces_base_domain() -> None: + """``plain_match=0`` forces base-domain on plain URLs (the KeePassXC-faithful opt-in).""" + got = _uris( + primary_url="example.com", + additional_urls=["https://alt.example"], + android_packages=[], + plain_match=0, + ) + expected = [("example.com", 0), ("https://alt.example", 0)] + if got != expected: + raise AssertionError(f"got {got}, want {expected}") + + +def assert_special_syntaxes_map_when_interpreting() -> None: + """Quoted -> exact(3); trailing wildcard -> starts-with(2); host wildcard -> regex(4).""" + got = dict( + _uris( + primary_url="", + additional_urls=[ + '"https://exact.example/login"', + "https://host.example/app/*", + "https://*.wild.example/*", + ], + android_packages=[], + ) + ) + if got.get("https://exact.example/login") != 3: + raise AssertionError(f"quoted should be exact(3): {got}") + if got.get("https://host.example/app/") != 2: + raise AssertionError(f"trailing wildcard should be starts-with(2): {got}") + regex_uris = [u for u, m in got.items() if m == 4] + if not regex_uris or "wild" not in regex_uris[0]: + raise AssertionError(f"host wildcard should be regex(4): {got}") + + +def assert_drops() -> None: + """Non-web schemes, references, illegal chars, and invalid wildcards are dropped.""" + got = _uris( + primary_url="", + additional_urls=[ + "keepassxc://by-uuid/abc", + "cmd://run", + "kdbx://x", + "file:///etc/hosts", + "{REF:A@I:1234}", + "https://has space", + "https://**", + "https://*.com", + ], + android_packages=[], + ) + if got: + raise AssertionError(f"all inputs should be dropped, got {got}") + + +def assert_literal_mode_skips_interpretation() -> None: + """interpret_syntax=False keeps quote/wildcard strings as plain URIs verbatim.""" + got = _uris( + primary_url="", + additional_urls=['"https://x/login"', "https://host/*"], + android_packages=[], + interpret_syntax=False, + ) + expected = [('"https://x/login"', None), ("https://host/*", None)] + if got != expected: + raise AssertionError(f"literal mode: got {got}, want {expected}") + + +def assert_plain_match_override() -> None: + """plain_match tunes the plain tier (e.g. None defers to the account default).""" + got = _uris( + primary_url="example.com", + additional_urls=['"https://x/login"'], + android_packages=[], + plain_match=None, + ) + # Plain primary follows the override (None); the quoted form stays exact(3). + if ("example.com", None) not in got: + raise AssertionError(f"plain tier should use override None: {got}") + if ("https://x/login", 3) not in got: + raise AssertionError(f"quoted form should stay exact(3): {got}") + + +def assert_remap_lifts_legacy_fields() -> None: + """remap_item_fields_to_uris drops KP2A_URL*/AndroidApp fields and adds URIs.""" + fields = [ + cast(BwField, {"name": "Notes", "value": "keep", "type": 0}), + cast(BwField, {"name": "KP2A_URL", "value": "https://alt.example", "type": 0}), + cast(BwField, {"name": "AndroidApp", "value": "com.example.app", "type": 0}), + ] + uris = [cast(BwUri, {"uri": "https://primary.example", "match": 0})] + + new_fields, new_uris, changed = remap_item_fields_to_uris(fields, uris) + + if not changed: + raise AssertionError("expected changed=True") + if [f["name"] for f in new_fields] != ["Notes"]: + raise AssertionError(f"legacy fields not dropped: {new_fields}") + new_values = [u["uri"] for u in new_uris] + if new_values != [ + "https://primary.example", + "https://alt.example", + "androidapp://com.example.app", + ]: + raise AssertionError(f"unexpected merged uris: {new_values}") + + +def assert_remap_noop_without_legacy_fields() -> None: + """An item without KP2A_URL*/AndroidApp fields is reported unchanged.""" + fields = [cast(BwField, {"name": "Notes", "value": "keep", "type": 0})] + uris = [cast(BwUri, {"uri": "https://x.example", "match": 0})] + _, _, changed = remap_item_fields_to_uris(fields, uris) + if changed: + raise AssertionError("expected changed=False when no legacy fields present") + + +def assert_collision_report_helpers() -> None: + """uri_host / registrable_domain / collision_groups power --report-uris.""" + # Host extraction strips scheme/port/path; app + bare-label URIs yield None. + if uri_host("https://vault.kajkowalski.nl/login?x=1") != "vault.kajkowalski.nl": + raise AssertionError("host extraction failed") + if uri_host("androidapp://com.x") is not None or uri_host("Windows") is not None: + raise AssertionError("app/bare URIs should have no host") + # Registrable domain honours the two-level public-suffix heuristic. + if registrable_domain("vault.kajkowalski.nl") != "kajkowalski.nl": + raise AssertionError("registrable domain (.nl) failed") + if registrable_domain("app.10bis.co.il") != "10bis.co.il": + raise AssertionError("registrable domain (.co.il) failed") + # Only multi-host registrable domains are reported; singletons are omitted. + groups = collision_groups([ + "https://vault.kajkowalski.nl", + "https://gitea.kajkowalski.nl", + "https://example.com", + "androidapp://com.x", + ]) + if set(groups) != {"kajkowalski.nl"}: + raise AssertionError(f"expected only kajkowalski.nl group, got {groups}") + if groups["kajkowalski.nl"] != ["gitea.kajkowalski.nl", "vault.kajkowalski.nl"]: + raise AssertionError(f"unexpected hosts: {groups}") + + +def main() -> None: + assert_match_names_parse() + assert_url_attribute_keys() + assert_plain_urls_default_to_account_default_and_dedup() + assert_uri_match_domain_forces_base_domain() + assert_special_syntaxes_map_when_interpreting() + assert_drops() + assert_literal_mode_skips_interpretation() + assert_plain_match_override() + assert_remap_lifts_legacy_fields() + assert_remap_noop_without_legacy_fields() + assert_collision_report_helpers() + print("uri mapping test passed") + + +if __name__ == "__main__": + main()