Add Separated-Key Architecture

CHANGELOG.md

@@ -16,11 +16,18 @@ Template:
 
 # Upcoming Release
 
+## New Features
+
+- Add separated-key architecture so that SOPS secrets, including ZFS passphrase, are protected at rest from physical access.
+- Add migration tools for existing hosts: `enable-key-separation`, `install-runtime-key`, and `rotate-boot-key`.
+- Add `runtimeHostKeyPath` and `runtimeHostKeyPub` configuration options for separated-key mode.
+
 ## Breaking Changes
 
 - Remove `hostId` file and directly set the value in the host's `configuration.nix` file.
 - Remove `ssh_port` and `ssh_boot_port` files and directly set the value in the host's `configuration.nix` file.
 - Remove `ip` and `system` files and directly set the value in the host's `flake.nix` file.
+- `gen-new-host` now creates separated-key hosts by default. Use `--single-key` flag for legacy single-key mode (existing single-key hosts remain functional).
 
 ## Fixes
 

docs/normal-operations.md

@@ -79,11 +79,93 @@ All commands are prefixed by the hostname, allowing to handle multiple hosts.
 
    and copy needed config in [flake.nix][].
 
+## Enable Key Separation {#enable-key-separation}
+
+   ::: {.warning}
+   **Security Warning:** Single-key hosts are vulnerable to physical attacks. If someone gains physical access to your server, they can extract the `/boot/host_key` and decrypt all your secrets (passwords, API keys, etc.). Enable key separation to safeguard your user data at rest.
+   :::
+
+   Upgrade existing hosts to separated-key architecture. This separates the boot key from your administrative secrets, protecting SOPS-encrypted data from physical attacks. The runtime key is stored in the encrypted ZFS pool, making it inaccessible until boot unlock. **Note:** New hosts created with `gen-new-host` use separated-key mode by default.
+
+   ```bash
+   # Generate runtime keys & update SOPS config, renaming existing key to myskarabox_boot
+   $ nix run .#myskarabox-enable-key-separation
+   # Re-encrypt secrets so that both keys can decrypt during migration
+   $ nix run .#sops -- updatekeys myskarabox/secrets.yaml
+   # Install runtime key on target
+   $ nix run .#myskarabox-install-runtime-key
+   ```
+
+   Update `myskarabox/configuration.nix` to switch SOPS to runtime key:
+   ```nix
+   sops.age.sshKeyPaths = [
+     "/persist/etc/ssh/ssh_host_ed25519_key"   # Switch from /boot/host_key
+   ];
+   ```
+
+   Update `flake.nix` to enable separated-key mode:
+   ```nix
+   skarabox.hosts.myskarabox = {
+     # ... existing config
+     runtimeHostKeyPub = ./myskarabox/runtime_host_key.pub;
+   };
+   ```
+
+   Deploy the separated-key configuration:
+   ```bash
+   $ nix run .#deploy-rs                       # Switches host to runtime key
+   $ nix run .#myskarabox-gen-knownhosts-file  # Update known_hosts after deployment
+   ```
+
+   After successful deployment, complete the migration:
+   ```bash
+   # Remove the boot key (now aliased as myskarabox_boot) from SOPS
+   $ age_key=$(nix shell nixpkgs#ssh-to-age -c ssh-to-age < myskarabox/host_key.pub)
+   $ nix run .#sops -- -r -i --rm-age "$age_key" myskarabox/secrets.yaml
+
+   # Clean up .sops.yaml by removing the boot key reference and anchor
+   $ sed -i.bak -e '/- \*myskarabox_boot$/d' -e '/&myskarabox_boot/d' .sops.yaml
+
+   # Rotate the boot key to protect against git history attacks
+   $ ssh-keygen -t ed25519 -f myskarabox/host_key -N ""
+   # See warning in rotation section, this is a destructive operation
+   $ nix run .#myskarabox-rotate-boot-key
+   $ nix run .#myskarabox-gen-knownhosts-file
+   ```
+
+   These final steps ensure secrets cannot be decrypted with the old boot key, protecting against both physical attacks and git history attacks.
+
 ## Rotate host key {#rotate-host-key}
 
+   **For single-key hosts (legacy):**
+
    ```bash
    $ ssh-keygen -f ./myskarabox/host_key
    $ nix run .#add-sops-cfg -- -o .sops.yaml alias myskarabox $(ssh-to-age -i ./myskarabox/host_key.pub)
+   $ nix run .#sops -- updatekeys ./myskarabox/secrets.yaml
+   $ nix run .#myskarabox-gen-knownhosts-file
+   $ nix run .#deploy-rs
+   ```
+
+   **For separated-key hosts:**
+
+   Rotate boot key (necessary to protect against git history attack after migration):
+
+   ::: {.warning}
+   **Destructive Operation:** This command securely wipes the boot partition with `dd + TRIM` to make key recovery difficult. The process backs up boot files to tmpfs, wipes the partition, recreates the filesystem, and reinstalls the bootloader. Requires the host to be accessible via runtime key.
+   :::
+
+   ```bash
+   $ ssh-keygen -t ed25519 -f myskarabox/host_key -N ""
+   $ nix run .#myskarabox-rotate-boot-key
+   $ nix run .#myskarabox-gen-knownhosts-file
+   ```
+
+   Rotate runtime key (only if compromised - affects SOPS secrets):
+   ```bash
+   $ ssh-keygen -t ed25519 -N "" -f ./myskarabox/runtime_host_key
+   $ nix run .#add-sops-cfg -- -o .sops.yaml alias myskarabox $(ssh-to-age -i ./myskarabox/runtime_host_key.pub)
+   $ nix run .#sops -- updatekeys ./myskarabox/secrets.yaml
+   $ nix run .#myskarabox-gen-knownhosts-file
    $ nix run .#deploy-rs
-   $ nix run .#baryum-gen-knownhosts-file
    ```

flake.nix

@@ -69,7 +69,7 @@
         };
 
         gen-new-host = import ./lib/gen-new-host.nix {
-          inherit add-sops-cfg pkgs gen-hostId;
+          inherit pkgs add-sops-cfg gen-hostId;
           inherit (pkgs) lib;
         };
 

flakeModules/default.nix

@@ -59,6 +59,18 @@ in
             apply = readAsStr;
             example = lib.literalExpression "./${name}/host_key.pub";
           };
+          runtimeHostKeyPath = mkOption {
+            description = "Path from the top of the repo to the runtime ssh private file (separated-key mode only).";
+            type = types.nullOr types.str;
+            default = "${name}/runtime_host_key";
+          };
+          runtimeHostKeyPub = mkOption {
+            description = "Runtime SSH public file (separated-key mode only).";
+            type = types.nullOr (with types; oneOf [ str path ]);
+            default = null;
+            apply = v: if v == null then null else readAsStr v;
+            example = lib.literalExpression "./${name}/runtime_host_key.pub";
+          };
           ip = mkOption {
             description = ''
               IP or hostname used to ssh into the server.
@@ -340,9 +352,19 @@ in
               ssh_boot_port=${toString hostCfg.skarabox.boot.sshPort}
               host_key_pub="${cfg'.hostKeyPub}"
 
-              gen-knownhosts-file \
-                "$host_key_pub" "$ip" $ssh_port $ssh_boot_port \
-                > ${cfg'.knownHostsPath}
+              {
+                # Check if separated-key mode is configured
+                ${lib.optionalString (cfg'.runtimeHostKeyPub != null) ''
+                  runtime_key_pub="${cfg'.runtimeHostKeyPub}"
+                  gen-knownhosts-file "$host_key_pub" "$ip" $ssh_boot_port
+                  gen-knownhosts-file "$runtime_key_pub" "$ip" $ssh_port
+                ''}
+
+                # Single key mode (backward compatibility)
+                ${lib.optionalString (cfg'.runtimeHostKeyPub == null) ''
+                  gen-knownhosts-file "$host_key_pub" "$ip" $ssh_port $ssh_boot_port
+                ''}
+              } > ${cfg'.knownHostsPath}
             '';
           };
 
@@ -400,7 +422,8 @@ in
                   ++ [ "--ssh-option" "ConnectTimeout=10" ]
                   ++ (lib.optionals (cfg'.sshPrivateKeyPath != null) [ "-i" cfg'.sshPrivateKeyPath ])
                   ++ [ "--disk-encryption-keys" "/tmp/host_key" cfg'.hostKeyPath ]
-                  ++ (lib.flatten (mapAttrsToList (name: path: [ "--disk-encryption-keys" "/tmp/${name}" "\$secret_file_${name}" ]) secrets));
+                  ++ (lib.flatten (mapAttrsToList (name: path: [ "--disk-encryption-keys" "/tmp/${name}" "\$secret_file_${name}" ]) secrets))
+                  ++ (lib.optionals (cfg'.runtimeHostKeyPub != null) [ "--disk-encryption-keys" "/tmp/runtime_host_key" cfg'.runtimeHostKeyPath ]);
 
                 # Convert to a bash array declaration
                 argsString = concatStringsSep " " (map (arg: ''"${arg}"'') extraArgs);
@@ -468,6 +491,22 @@ in
               printf '%s' "$root_passphrase" | boot-ssh -T "$@"
             '';
           };
+
+          enable-key-separation = import ../lib/enable-key-separation.nix {
+            inherit pkgs name;
+            cfg = cfg';
+            add-sops-cfg = import ../lib/add-sops-cfg.nix { inherit pkgs; };
+          };
+
+          install-runtime-key = import ../lib/install-runtime-key.nix {
+            inherit pkgs ssh name;
+            cfg = cfg';
+          };
+
+          rotate-boot-key = import ../lib/rotate-boot-key.nix {
+            inherit pkgs ssh name;
+            cfg = cfg';
+          };
         in {
           "${name}-boot-ssh" = boot-ssh;
           "${name}-sops" = sops;
@@ -478,6 +517,9 @@ in
           "${name}-ssh" = ssh;
           "${name}-get-facter" = get-facter;
           "${name}-unlock" = unlock;
+          "${name}-enable-key-separation" = enable-key-separation;
+          "${name}-install-runtime-key" = install-runtime-key;
+          "${name}-rotate-boot-key" = rotate-boot-key;
         };
     in {
       packages = {