Move secrets + repo seeds to runtime bootstrap

Author: joshp123

.github/workflows/image-build.yml

@@ -44,7 +44,8 @@ jobs:
             nixpkgs#nixos-generators \
             nixpkgs#awscli2 \
             nixpkgs#age \
-            nixpkgs#jq
+            nixpkgs#jq \
+            nixpkgs#zstd
 
       - name: Write agenix image key
         env:
@@ -90,6 +91,16 @@ jobs:
         run: |
           scripts/prepare-repo-seeds.sh repo-seeds
 
+      - name: Upload bootstrap bundle
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          S3_BUCKET: ${{ secrets.S3_BUCKET }}
+          BOOTSTRAP_PREFIX: bootstrap/clawdinator-1
+        run: |
+          scripts/upload-bootstrap.sh
+
       - name: Build image
         run: scripts/build-image.sh
 

AGENTS.md

@@ -63,7 +63,8 @@ Deploy flow (automation-first):
   - Use `nix/hosts/clawdinator-1-image.nix` for image builds.
 - CI is preferred: `.github/workflows/image-build.yml` runs build → S3 upload → AMI import.
 - Resume AMI pipeline work immediately if it stalls; do not use rsync as a workaround. Host edits are allowed but must be committed and baked into a new AMI to persist.
-- CI must provide `CLAWDINATOR_AGE_KEY` (private key) so the image can bake `/etc/agenix/keys/clawdinator.agekey`.
+- CI must provide `CLAWDINATOR_AGE_KEY` to build + upload the runtime bootstrap bundle to S3.
+- Bootstrap bundle location: `s3://${S3_BUCKET}/bootstrap/<instance>/` (secrets + repo seeds).
 - Bootstrap S3 bucket + scoped IAM user + VM Import role with `infra/opentofu/aws` (use homelab-admin creds).
 - Bootstrap AWS instances from the AMI with `infra/opentofu/aws` (set `TF_VAR_ami_id`).
 - Import the image into AWS as an AMI (snapshot import + register image).

README.md

@@ -99,13 +99,13 @@ Image‑based deploy (only path):
 2) Upload the raw image to S3 (private object).
 3) Import into AWS as an AMI (snapshot import + register image).
 4) Launch hosts from the AMI (OpenTofu `infra/opentofu/aws`).
-5) Ensure secrets are encrypted to the baked agenix key and sync them to `/var/lib/clawd/nix-secrets`.
-6) Run `nixos-rebuild switch --flake /var/lib/clawd/repo#clawdinator-1`.
+5) Upload the runtime bootstrap bundle to `s3://<bucket>/bootstrap/<instance>/` (secrets + repo seeds).
+6) Hosts download secrets at boot (`clawdinator-bootstrap.service`) and then run `nixos-rebuild switch --flake /var/lib/clawd/repo#clawdinator-1`.
 
 CI (recommended):
 - GitHub Actions builds the image, uploads to S3, and imports an AMI.
 - See `.github/workflows/image-build.yml` and `scripts/*.sh`.
-- CI must provide `CLAWDINATOR_AGE_KEY` so the image can bake `/etc/agenix/keys/clawdinator.agekey`.
+- CI must provide `CLAWDINATOR_AGE_KEY` to build + upload the runtime bootstrap bundle.
 
 AWS bucket bootstrap:
 - `infra/opentofu/aws` provisions a private S3 bucket + scoped IAM user + VM Import role.

docs/SECRETS.md

@@ -8,7 +8,7 @@ Infrastructure (OpenTofu):
 
 Image pipeline (CI):
 - `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` / `AWS_REGION` / `S3_BUCKET` (required).
-- `CLAWDINATOR_AGE_KEY` (required; private age key baked into the AMI).
+- `CLAWDINATOR_AGE_KEY` (required; used to build the bootstrap bundle uploaded to S3).
 
 Local storage:
 - Keep AWS keys encrypted in `../nix/nix-secrets` for local runs if needed.
@@ -35,11 +35,18 @@ Agenix (local secrets repo):
 - Store encrypted files in `../nix/nix-secrets` (relative to this repo).
 - Sync encrypted secrets to the host at `/var/lib/clawd/nix-secrets`.
 - Decrypt on host with agenix; point NixOS options at `/run/agenix/*`.
-- Image builds bake the agenix identity to `/etc/agenix/keys/clawdinator.agekey`; do not commit this key.
+- Image builds do **not** bake the agenix identity; the age key is injected at runtime via the bootstrap bundle.
 - Required files (minimum): `clawdinator-github-app.pem.age`, `clawdinator-discord-token.age`, `clawdinator-anthropic-api-key.age`.
 - Also required for OpenAI: `clawdinator-openai-api-key-peter-2.age`.
 - CI image pipeline (stored locally, not on hosts): `clawdinator-image-uploader-access-key-id.age`, `clawdinator-image-uploader-secret-access-key.age`, `clawdinator-image-bucket-name.age`, `clawdinator-image-bucket-region.age`.
 
+Bootstrap bundle (runtime injection):
+- CI uploads `secrets.tar.zst` + `repo-seeds.tar.zst` to `s3://${S3_BUCKET}/bootstrap/<instance>/`.
+- `secrets.tar.zst` contains:
+  - `clawdinator.agekey`
+  - `secrets/` directory with `*.age` files.
+- The host downloads + installs these on boot (`clawdinator-bootstrap.service`).
+
 Example NixOS wiring (agenix):
 ```
 { inputs, ... }:

infra/opentofu/aws/README.md

@@ -1,6 +1,6 @@
 # OpenTofu (AWS S3 Image Bucket)
 
-Goal: use the CLAWDINATOR S3 bucket for images, plus create the VM Import role and attach import permissions to the CI IAM user.
+Goal: use the CLAWDINATOR S3 bucket for images + bootstrap artifacts, create the VM Import role, and attach import permissions to the CI IAM user.
 Also provisions EFS for shared memory.
 
 Prereqs:
@@ -34,3 +34,6 @@ CI wiring:
   - `AWS_SECRET_ACCESS_KEY`
   - `AWS_REGION`
   - `S3_BUCKET`
+
+Runtime bootstrap:
+- Instances get an IAM role with read access to `s3://${S3_BUCKET}/bootstrap/*` for secrets + repo seeds.

infra/opentofu/aws/main.tf

@@ -167,6 +167,37 @@ resource "aws_iam_role_policy_attachment" "instance_ssm" {
   policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
 }
 
+data "aws_iam_policy_document" "instance_bootstrap" {
+  statement {
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectAttributes"
+    ]
+    resources = [
+      "${aws_s3_bucket.image_bucket.arn}/bootstrap/*"
+    ]
+  }
+
+  statement {
+    actions = [
+      "s3:GetBucketLocation",
+      "s3:ListBucket"
+    ]
+    resources = [aws_s3_bucket.image_bucket.arn]
+    condition {
+      test     = "StringLike"
+      variable = "s3:prefix"
+      values   = ["bootstrap/*"]
+    }
+  }
+}
+
+resource "aws_iam_role_policy" "instance_bootstrap" {
+  name   = "clawdinator-bootstrap"
+  role   = aws_iam_role.instance.id
+  policy = data.aws_iam_policy_document.instance_bootstrap.json
+}
+
 resource "aws_iam_instance_profile" "instance" {
   name = "clawdinator-instance"
   role = aws_iam_role.instance.name

nix/hosts/clawdinator-1-common.nix

@@ -26,6 +26,8 @@ in
   };
 
   config = {
+    clawdinator.secretsPath = "/var/lib/clawd/nix-secrets";
+
     age.identityPaths = [ "/etc/agenix/keys/clawdinator.agekey" ];
     age.secrets."clawdinator-github-app.pem" = {
       file = "${secretsPath}/clawdinator-github-app.pem.age";
@@ -52,6 +54,16 @@ in
       enable = true;
       instanceName = "CLAWDINATOR-1";
       memoryDir = "/memory";
+      repoSeedSnapshotDir = "/var/lib/clawd/repo-seeds";
+      bootstrap = {
+        enable = true;
+        s3Bucket = "clawdinator-images-eu1-20260107165216";
+        s3Prefix = "bootstrap/clawdinator-1";
+        region = "eu-central-1";
+        secretsDir = "/var/lib/clawd/nix-secrets";
+        repoSeedsDir = "/var/lib/clawd/repo-seeds";
+        ageKeyPath = "/etc/agenix/keys/clawdinator.agekey";
+      };
       memoryEfs = {
         enable = true;
         fileSystemId = "fs-0e7920726c2965a88";

nix/hosts/clawdinator-1-image.nix

@@ -21,42 +21,12 @@
   networking.useDHCP = true;
   services.openssh.enable = true;
   services.openssh.settings.PermitRootLogin = "prohibit-password";
-  assertions = [
-    {
-      assertion = (builtins.getEnv "CLAWDINATOR_AGE_KEY") != "";
-      message = "CLAWDINATOR_AGE_KEY must be set when building the image.";
-    }
-    {
-      assertion = (builtins.getEnv "CLAWDINATOR_SECRETS_DIR") != "";
-      message = "CLAWDINATOR_SECRETS_DIR must point at encrypted age secrets.";
-    }
-    {
-      assertion = (builtins.getEnv "CLAWDINATOR_REPO_SEEDS_DIR") != "";
-      message = "CLAWDINATOR_REPO_SEEDS_DIR must point at preseeded repos.";
-    }
-  ];
-
-  environment.etc."agenix/keys/clawdinator.agekey" = {
-    text = builtins.getEnv "CLAWDINATOR_AGE_KEY";
-    mode = "0400";
-  };
   users.users.root.openssh.authorizedKeys.keys = [
     "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOLItFT3SVm5r7gELrfRRJxh6V2sf/BIx7HKXt6oVWpB"
   ];
 
-  clawdinator.secretsPath = toString (builtins.path {
-    path = builtins.toPath (builtins.getEnv "CLAWDINATOR_SECRETS_DIR");
-    name = "clawdinator-age-secrets";
-  });
-
-  services.clawdinator.repoSeedSnapshotDir =
-    let
-      seedsDir = builtins.getEnv "CLAWDINATOR_REPO_SEEDS_DIR";
-    in
-    if seedsDir == ""
-    then null
-    else builtins.path {
-      path = builtins.toPath seedsDir;
-      name = "clawdinator-repo-seeds";
-    };
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/nixos";
+    fsType = "ext4";
+  };
 }

nix/hosts/clawdinator-1.nix

@@ -21,5 +21,4 @@
 
   networking.firewall.allowedTCPPorts = [ 22 18789 ];
 
-  clawdinator.secretsPath = "/var/lib/clawd/nix-secrets";
 }

nix/modules/clawdinator.nix

@@ -211,11 +211,62 @@ in
     };
 
     repoSeedSnapshotDir = mkOption {
-      type = types.nullOr types.path;
+      type = types.nullOr types.str;
       default = null;
       description = "Optional path to a preseeded repo snapshot (directory of repos). When set, no network cloning happens at boot.";
     };
 
+    bootstrap = {
+      enable = mkEnableOption "Bootstrap secrets + repo seeds from S3";
+
+      s3Bucket = mkOption {
+        type = types.str;
+        description = "S3 bucket holding bootstrap artifacts.";
+      };
+
+      s3Prefix = mkOption {
+        type = types.str;
+        default = "bootstrap/${cfg.instanceName}";
+        description = "S3 prefix for bootstrap artifacts (relative to bucket).";
+      };
+
+      region = mkOption {
+        type = types.str;
+        default = "eu-central-1";
+        description = "AWS region for S3 bootstrap bucket.";
+      };
+
+      secretsArchive = mkOption {
+        type = types.str;
+        default = "secrets.tar.zst";
+        description = "Secrets archive name inside the bootstrap prefix.";
+      };
+
+      repoSeedsArchive = mkOption {
+        type = types.str;
+        default = "repo-seeds.tar.zst";
+        description = "Repo seeds archive name inside the bootstrap prefix.";
+      };
+
+      ageKeyPath = mkOption {
+        type = types.str;
+        default = "/etc/agenix/keys/clawdinator.agekey";
+        description = "Destination path for the agenix identity key.";
+      };
+
+      secretsDir = mkOption {
+        type = types.str;
+        default = "/var/lib/clawd/nix-secrets";
+        description = "Destination directory for encrypted age secrets.";
+      };
+
+      repoSeedsDir = mkOption {
+        type = types.str;
+        default = "/var/lib/clawd/repo-seeds";
+        description = "Destination directory for repo seed snapshots.";
+      };
+    };
+
     workspaceTemplateDir = mkOption {
       type = types.path;
       default = ../../clawdinator/workspace;
@@ -482,9 +533,11 @@ in
       wantedBy = [ "multi-user.target" ];
       after =
         [ "network.target" ]
+        ++ lib.optional cfg.bootstrap.enable "clawdinator-bootstrap.service"
         ++ lib.optional cfg.githubApp.enable "clawdinator-github-app-token.service"
         ++ lib.optional (cfg.repoSeedSnapshotDir != null) "clawdinator-repo-seed.service";
       wants =
+        lib.optional cfg.bootstrap.enable "clawdinator-bootstrap.service"
         lib.optional cfg.githubApp.enable "clawdinator-github-app-token.service"
         ++ lib.optional (cfg.repoSeedSnapshotDir != null) "clawdinator-repo-seed.service";
 
@@ -527,7 +580,10 @@ in
       description = "CLAWDINATOR repo seed (snapshot copy)";
       wantedBy = [ "multi-user.target" ];
       before = [ "clawdinator.service" ];
-      after = [ "local-fs.target" ];
+      after =
+        [ "local-fs.target" ]
+        ++ lib.optional cfg.bootstrap.enable "clawdinator-bootstrap.service";
+      requires = lib.optional cfg.bootstrap.enable "clawdinator-bootstrap.service";
       serviceConfig = {
         Type = "oneshot";
         User = "root";
@@ -536,6 +592,28 @@ in
       script = "${pkgs.bash}/bin/bash ${../../scripts/seed-repos-from-snapshot.sh} ${cfg.repoSeedSnapshotDir} ${repoSeedBaseDir} ${cfg.user} ${cfg.group}";
     };
 
+    systemd.services.clawdinator-bootstrap = lib.mkIf cfg.bootstrap.enable {
+      description = "CLAWDINATOR bootstrap (S3 secrets + repo seeds)";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+      environment = {
+        AWS_REGION = cfg.bootstrap.region;
+        AWS_DEFAULT_REGION = cfg.bootstrap.region;
+      };
+      path = [ pkgs.awscli2 pkgs.coreutils pkgs.gnutar pkgs.zstd ];
+      script = "${pkgs.bash}/bin/bash ${../../scripts/bootstrap-runtime.sh} ${cfg.bootstrap.s3Bucket} ${cfg.bootstrap.s3Prefix} ${cfg.bootstrap.secretsDir} ${cfg.bootstrap.repoSeedsDir} ${cfg.bootstrap.ageKeyPath} ${cfg.bootstrap.secretsArchive} ${cfg.bootstrap.repoSeedsArchive}";
+    };
+
+    systemd.services.agenix = lib.mkIf cfg.bootstrap.enable {
+      requires = [ "clawdinator-bootstrap.service" ];
+      after = [ "clawdinator-bootstrap.service" ];
+    };
+
     systemd.services.clawdinator-efs-stunnel = lib.mkIf cfg.memoryEfs.enable {
       description = "CLAWDINATOR EFS TLS tunnel";
       wantedBy = [ "multi-user.target" ];