feat(security): support Fernet key rotation via MultiFernet

.env.example

@@ -14,12 +14,22 @@ DATABASE_URL=postgresql+asyncpg://helprs:helprs@db:5432/helprs
 # Generate: python -c "import secrets; print(secrets.token_urlsafe(48))"
 SECRET_KEY=
 
-# Encryption key for stored Claude credentials (Fernet symmetric encryption)
+# Encryption key for stored credentials (Fernet symmetric encryption)
 # Generate: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
 FERNET_KEY=
 
-# Admin panel password (required when ENVIRONMENT=production)
-# In development mode, any password is accepted.
+# Previously-active keys, JSON list, newest first. Values encrypted under them
+# stay readable, so FERNET_KEY can be replaced without a downtime window:
+#   1. set the new key as FERNET_KEY and move the old one here
+#   2. deploy - new writes use the new key, old values still decrypt
+#   3. uv run python -m helprs.scripts.rotate_credentials
+#   4. empty this list and deploy again
+# Leave unset when no rotation is in flight: a retired key is exactly as
+# sensitive as a live one for as long as it appears here.
+# FERNET_KEY_FALLBACKS=["<previous-key>"]
+
+# Admin panel password. Required when ENVIRONMENT=production; when it is unset
+# the panel is not mounted at all, in any environment.
 ADMIN_PASSWORD=
 
 # --- GitHub App --------------------------------------------------------------

CLAUDE.md

@@ -61,6 +61,7 @@ infra/
 - **Auth on all REST routes**: identity and installation routers use `Depends(get_current_user)`, container router uses it too. The webhook handler bypasses REST routes entirely — it calls `create_session()` directly (DB record only, no container start). Container start happens when the authenticated frontend calls the REST endpoint.
 - **No module `__init__` imports**: the four `modules/*/__init__.py` are docstring-only. Re-exporting a router there pulled the whole router graph back through `core.dependencies` (which imports `identity.models`), so `import helprs.core.dependencies` failed on its own and startup depended on `main.py`'s import order. `tests/test_import_graph.py` guards this.
 - **JWT**: PyJWT, not python-jose (unmaintained since 2021, and the source of an unfixable `ecdsa` advisory). `PyJWTError` is the failure type.
+- **Fernet keyset, not a single key**: the crypto helpers take `settings.fernet_keys` (`list[str]`, primary first, then `FERNET_KEY_FALLBACKS`) and go through `MultiFernet` — encrypts with the first, decrypts with any. That is what makes key rotation possible without downtime; `helprs.scripts.rotate_credentials` re-encrypts stored rows so a retired key can actually be dropped. Typed `list[str]` rather than `Sequence[str]` on purpose: `str` satisfies `Sequence[str]`, so a caller passing a bare key would type-check.
 - **Secrets are `SecretStr`**: read them with `.get_secret_value()`. `SecretStr` defines `__len__`, so truthiness checks work unchanged. `repr(Settings())` used to print every credential, and Sentry uploads locals on any unhandled 500.
 - **SSE takes no DB dependency**: FastAPI tears yield-dependencies down only after the streaming body ends, so `Depends(get_db)` — including one behind an auth dependency — pins a pooled connection for the whole stream. The SSE route calls `authenticate_token`/`stream_token` inside a short `get_db_context()` instead.
 - **Production env validation**: `Settings` has a `model_validator` that enforces non-empty secrets when `ENVIRONMENT=production`. Tests use `ENVIRONMENT=test` to skip this.

apps/api/src/helprs/core/config.py

@@ -9,6 +9,18 @@
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
+def _assert_fernet_key(key: SecretStr, name: str) -> None:
+    """Reject anything Fernet could not use, naming the offending setting."""
+    try:
+        Fernet(key.get_secret_value().encode())
+    except (ValueError, InvalidToken) as e:
+        raise ValueError(
+            f"{name} must be a valid Fernet key (32 url-safe base64-encoded bytes). "
+            "Generate with: python -c "
+            "'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'"
+        ) from e
+
+
 class Settings(BaseSettings):
     """Runtime configuration.
 
@@ -28,7 +40,13 @@ class Settings(BaseSettings):
 
     # Security
     SECRET_KEY: SecretStr
+    # The key new credentials are encrypted with.
     FERNET_KEY: SecretStr
+    # Previously-active keys, newest first. Values written under them stay
+    # readable, so a key can be replaced without a downtime window: set the
+    # new key as FERNET_KEY, move the old one here, deploy, re-encrypt with
+    # `python -m helprs.scripts.rotate_credentials`, then empty this list.
+    FERNET_KEY_FALLBACKS: list[SecretStr] = []
     ADMIN_PASSWORD: SecretStr = SecretStr("")
 
     # GitHub App
@@ -93,16 +111,35 @@ def normalize_private_key(cls, v: SecretStr) -> SecretStr:
     @classmethod
     def validate_fernet_key(cls, v: SecretStr) -> SecretStr:
         """Validate that FERNET_KEY is a valid Fernet key (32 url-safe base64 bytes)."""
-        try:
-            Fernet(v.get_secret_value().encode())
-        except (ValueError, InvalidToken) as e:
-            raise ValueError(
-                "FERNET_KEY must be a valid Fernet key (32 url-safe base64-encoded bytes). "
-                "Generate with: python -c "
-                "'from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())'"
-            ) from e
+        _assert_fernet_key(v, "FERNET_KEY")
         return v
 
+    @field_validator("FERNET_KEY_FALLBACKS")
+    @classmethod
+    def validate_fernet_fallbacks(cls, v: list[SecretStr]) -> list[SecretStr]:
+        """A malformed retired key must fail at boot, not at first decrypt.
+
+        Without this the app would start happily and only break when it met a
+        value written under that key -- which is precisely the credential a
+        rotation is trying not to lose.
+        """
+        for index, key in enumerate(v):
+            _assert_fernet_key(key, f"FERNET_KEY_FALLBACKS[{index}]")
+        return v
+
+    @property
+    def fernet_keys(self) -> list[str]:
+        """The ordered keyset: primary first, then retired keys.
+
+        A plain property rather than a ``computed_field`` on purpose -- the
+        latter would put every key in ``model_dump()``, undoing the reason the
+        fields are ``SecretStr`` in the first place.
+        """
+        return [
+            self.FERNET_KEY.get_secret_value(),
+            *(key.get_secret_value() for key in self.FERNET_KEY_FALLBACKS),
+        ]
+
     @model_validator(mode="after")
     def validate_production_secrets(self) -> "Settings":
         """Enforce that critical secrets are set when ENVIRONMENT is production."""

apps/api/src/helprs/core/security.py

@@ -6,19 +6,45 @@
 from datetime import UTC, datetime, timedelta
 
 import jwt
-from cryptography.fernet import Fernet
+from cryptography.fernet import Fernet, MultiFernet
 
 
-def fernet_encrypt(plaintext: str, fernet_key: str) -> str:
-    """Encrypt a string using Fernet symmetric encryption."""
-    f = Fernet(fernet_key.encode())
-    return f.encrypt(plaintext.encode()).decode()
+def _cipher(fernet_keys: list[str]) -> MultiFernet:
+    """Build the cipher for an ordered keyset.
 
+    ``MultiFernet`` encrypts with the FIRST key and decrypts with whichever
+    one matches, which is exactly what rotation needs: put the new key first,
+    keep the retired ones behind it until every stored value has been
+    re-encrypted, then drop them.
+    """
+    if not fernet_keys:
+        raise ValueError("At least one Fernet key is required")
+    return MultiFernet([Fernet(key.encode()) for key in fernet_keys])
+
+
+def fernet_encrypt(plaintext: str, fernet_keys: list[str]) -> str:
+    """Encrypt a string with the primary key of the keyset."""
+    return _cipher(fernet_keys).encrypt(plaintext.encode()).decode()
+
+
+def fernet_decrypt(ciphertext: str, fernet_keys: list[str]) -> str:
+    """Decrypt a value written by any key in the keyset.
 
-def fernet_decrypt(ciphertext: str, fernet_key: str) -> str:
-    """Decrypt a Fernet-encrypted string."""
-    f = Fernet(fernet_key.encode())
-    return f.decrypt(ciphertext.encode()).decode()
+    Raises ``InvalidToken`` when no key matches -- which is also what a
+    tampered ciphertext produces, since Fernet verifies its HMAC before
+    decrypting anything.
+    """
+    return _cipher(fernet_keys).decrypt(ciphertext.encode()).decode()
+
+
+def fernet_rotate(ciphertext: str, fernet_keys: list[str]) -> str:
+    """Re-encrypt an existing value under the primary key.
+
+    Does not need the plaintext: ``MultiFernet.rotate`` decrypts with
+    whichever key matches and re-encrypts with the first. This is what lets a
+    retired key actually be retired instead of carried forever.
+    """
+    return _cipher(fernet_keys).rotate(ciphertext.encode()).decode()
 
 
 def create_app_jwt(app_id: str, private_key: str) -> str:

apps/api/src/helprs/modules/container/router.py

@@ -83,7 +83,7 @@ async def create_container_session(
     if not byok_config:
         raise NotFoundError("No Claude token configured for this installation")
 
-    claude_oauth_token = fernet_decrypt(byok_config.encrypted_api_key, settings.FERNET_KEY.get_secret_value())
+    claude_oauth_token = fernet_decrypt(byok_config.encrypted_api_key, settings.fernet_keys)
 
     # Narrowed to this repo, read-only: the container runs Claude Code over
     # untrusted PR content with network egress.

apps/api/src/helprs/modules/identity/repository.py

@@ -27,3 +27,9 @@ async def add(session: AsyncSession, user: GitHubUser) -> GitHubUser:
     session.add(user)
     await session.flush()
     return user
+
+
+async def list_all(session: AsyncSession) -> list[GitHubUser]:
+    """Every user row. Used by credential rotation, which must miss nobody."""
+    result = await session.execute(select(GitHubUser).order_by(GitHubUser.created_at))
+    return list(result.scalars().all())

apps/api/src/helprs/modules/identity/service.py

@@ -52,7 +52,7 @@ async def sync_user(
     settings: Settings,
 ) -> GitHubUser:
     """Create the user for this GitHub identity, or refresh the stored one."""
-    encrypted_token = fernet_encrypt(access_token, settings.FERNET_KEY.get_secret_value())
+    encrypted_token = fernet_encrypt(access_token, settings.fernet_keys)
     user = await repository.get_by_github_id(session, profile.github_id)
 
     if user is None:
@@ -75,10 +75,10 @@ async def sync_user(
     return user
 
 
-def get_decrypted_github_token(user: GitHubUser, fernet_key: str) -> str:
+def get_decrypted_github_token(user: GitHubUser, fernet_keys: list[str]) -> str:
     """Decrypt a user's stored GitHub access token."""
     try:
-        return fernet_decrypt(user.github_access_token_enc, fernet_key)
+        return fernet_decrypt(user.github_access_token_enc, fernet_keys)
     except InvalidToken as e:
         raise UnauthorizedError("Stored GitHub token is corrupted") from e
 

apps/api/src/helprs/modules/installation/repository.py

@@ -95,3 +95,14 @@ async def add_byok_config(session: AsyncSession, config: BYOKConfig) -> BYOKConf
 async def delete_byok_config(session: AsyncSession, config: BYOKConfig) -> None:
     await session.delete(config)
     await session.flush()
+
+
+async def list_all_byok_configs(session: AsyncSession) -> list[BYOKConfig]:
+    """Every BYOK row, including those of soft-deleted installations.
+
+    Deliberately not filtered by ``_active()``: a retired Fernet key cannot be
+    dropped while any stored ciphertext still needs it, and a soft-deleted
+    installation's row is still stored ciphertext.
+    """
+    result = await session.execute(select(BYOKConfig).order_by(BYOKConfig.created_at))
+    return list(result.scalars().all())

apps/api/src/helprs/modules/installation/router.py

@@ -102,7 +102,7 @@ async def post_byok(
     installation = await _installation_or_404(session, installation_id)
     await verify_admin_permission(user, installation, settings)
 
-    config = await configure_byok(session, installation.id, body.api_key, settings.FERNET_KEY.get_secret_value())
+    config = await configure_byok(session, installation.id, body.api_key, settings.fernet_keys)
     return BYOKConfigResponse.model_validate(config)
 
 

apps/api/src/helprs/modules/installation/service.py

@@ -195,7 +195,7 @@ async def mint_installation_token(
 
 def _user_github_token(user: "GitHubUser", settings: Settings) -> str:
     try:
-        return fernet_decrypt(user.github_access_token_enc, settings.FERNET_KEY.get_secret_value())
+        return fernet_decrypt(user.github_access_token_enc, settings.fernet_keys)
     except InvalidToken as e:
         raise UnauthorizedError("Stored GitHub token is corrupted") from e
 
@@ -295,13 +295,13 @@ async def configure_byok(
     session: AsyncSession,
     installation_id: uuid.UUID,
     api_key: str,
-    fernet_key: str,
+    fernet_keys: list[str],
 ) -> BYOKConfig:
     """Validate a Claude credential, then store it encrypted."""
     if not await anthropic.is_credential_valid(api_key):
         raise BYOKKeyInvalidError("API key validation failed -- check your key and try again")
 
-    encrypted_key = fernet_encrypt(api_key, fernet_key)
+    encrypted_key = fernet_encrypt(api_key, fernet_keys)
     key_hint = f"...{api_key[-4:]}"
     now = datetime.now(UTC)
 
@@ -333,10 +333,10 @@ async def get_byok_config(session: AsyncSession, installation_id: uuid.UUID) ->
     return await repository.get_byok_config(session, installation_id)
 
 
-def decrypt_byok_key(byok_config: BYOKConfig, fernet_key: str) -> str:
+def decrypt_byok_key(byok_config: BYOKConfig, fernet_keys: list[str]) -> str:
     """Decrypt the stored Claude credential."""
     try:
-        return fernet_decrypt(byok_config.encrypted_api_key, fernet_key)
+        return fernet_decrypt(byok_config.encrypted_api_key, fernet_keys)
     except InvalidToken as e:
         raise BYOKKeyInvalidError("Stored API key could not be decrypted") from e
 

apps/api/src/helprs/scripts/__init__.py

@@ -0,0 +1 @@
+"""Operational scripts, run with `python -m helprs.scripts.<name>`."""

apps/api/src/helprs/scripts/rotate_credentials.py

@@ -0,0 +1,108 @@
+"""Re-encrypt every stored credential under the primary Fernet key.
+
+The step that makes key rotation finish. ``FERNET_KEY_FALLBACKS`` keeps old
+ciphertext readable, but a retired key can only actually be dropped once
+nothing is still encrypted with it -- otherwise the fallback list grows
+forever and the "retired" key stays as sensitive as the live one.
+
+    # 1. generate a key and put it in front
+    FERNET_KEY=<new>  FERNET_KEY_FALLBACKS='["<old>"]'
+    # 2. deploy; new writes use <new>, old values still decrypt
+    # 3. rewrite the old values
+    uv run python -m helprs.scripts.rotate_credentials
+    # 4. drop FERNET_KEY_FALLBACKS and deploy again
+
+Safe to re-run: rotating a value already under the primary key just rewrites
+it. Run it against a database you can restore, like any bulk rewrite.
+"""
+
+import asyncio
+import sys
+
+from cryptography.fernet import InvalidToken
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from helprs.core.config import get_settings
+from helprs.core.database import create_engine, create_session_factory
+from helprs.core.security import fernet_rotate
+from helprs.modules.identity import repository as identity_repository
+from helprs.modules.installation import repository as installation_repository
+
+
+class RotationReport:
+    """What happened, per credential kind."""
+
+    def __init__(self) -> None:
+        self.rotated = 0
+        self.failed: list[str] = []
+
+    def record(self, label: str, *, ok: bool) -> None:
+        if ok:
+            self.rotated += 1
+        else:
+            self.failed.append(label)
+
+
+async def _rotate_github_tokens(session: AsyncSession, fernet_keys: list[str], report: RotationReport) -> None:
+    for user in await identity_repository.list_all(session):
+        try:
+            user.github_access_token_enc = fernet_rotate(user.github_access_token_enc, fernet_keys)
+        except InvalidToken:
+            # No key in the set can read it. Reported rather than raised: one
+            # unreadable row must not stop the rest from being rewritten.
+            report.record(f"user {user.id} ({user.github_login})", ok=False)
+        else:
+            report.record(f"user {user.id}", ok=True)
+
+
+async def _rotate_byok_keys(session: AsyncSession, fernet_keys: list[str], report: RotationReport) -> None:
+    for config in await installation_repository.list_all_byok_configs(session):
+        try:
+            config.encrypted_api_key = fernet_rotate(config.encrypted_api_key, fernet_keys)
+        except InvalidToken:
+            report.record(f"byok {config.id} (installation {config.installation_id})", ok=False)
+        else:
+            report.record(f"byok {config.id}", ok=True)
+
+
+async def rotate_all() -> RotationReport:
+    """Rewrite every stored credential with the primary key."""
+    settings = get_settings()
+    report = RotationReport()
+
+    engine = create_engine()
+    try:
+        session_factory = create_session_factory(engine)
+        async with session_factory() as session:
+            await _rotate_github_tokens(session, settings.fernet_keys, report)
+            await _rotate_byok_keys(session, settings.fernet_keys, report)
+            # One transaction: a partial rewrite would leave the operator
+            # unable to tell which key each row is under.
+            await session.commit()
+    finally:
+        await engine.dispose()
+
+    return report
+
+
+def main() -> int:
+    report = asyncio.run(rotate_all())
+    print(f"re-encrypted {report.rotated} credential(s) with the primary key")
+
+    if report.failed:
+        print(f"\n{len(report.failed)} could not be read by any configured key:", file=sys.stderr)
+        for label in report.failed:
+            print(f"  - {label}", file=sys.stderr)
+        print(
+            "\nThe key they were written with is missing from FERNET_KEY_FALLBACKS. "
+            "Add it and re-run, or those credentials have to be re-entered.",
+            file=sys.stderr,
+        )
+        return 1
+
+    print("FERNET_KEY_FALLBACKS can now be emptied.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())