Skip to content

Commit e60986d

Browse files
ngalaikongalaiko
committed
don't re-create box on deploy
1 parent be176f4 commit e60986d

11 files changed

Lines changed: 640 additions & 66 deletions

File tree

‎.github/workflows/deploy.yaml‎

Lines changed: 57 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,38 @@
11
name: deploy
22

3+
# Self-deciding deploy:
4+
# * No VM tagged `computer` exists -> CREATE one from the freshly-built image
5+
# (bootstrap / disaster recovery). A fresh VM has no state, so this is the ONE
6+
# path that needs the backup secrets — passed as `exe.dev new --env` so the box
7+
# can restore from B2 on boot. (Brand-new setups still need the manual seeding
8+
# in README: tailscale authkey, pilegram env.)
9+
# * A VM tagged `computer` exists -> ACTIVATE IN PLACE over Tailscale. No VM
10+
# recreate, no node churn, no restore, and NO secrets in CI — RESTIC/B2/pilegram
11+
# already live on the box.
12+
# The next deploy after a create finds the VM and switches it onto the generation.
13+
314
on:
415
workflow_dispatch:
516
inputs:
617
image_tag:
7-
description: image tag to deploy (blank = the built commit's short SHA)
18+
description: image tag to create from (blank = the built commit's short SHA)
819
default: ""
920
workflow_run:
1021
workflows: [build]
1122
types: [completed]
1223
branches: [master]
1324

14-
env:
15-
VM_TAG: computer
16-
1725
concurrency:
1826
group: deploy
1927
cancel-in-progress: false
2028

29+
env:
30+
VM_NAME: computer
31+
VM_TAG: computer
32+
2133
jobs:
2234
deploy:
23-
name: Redeploy
35+
name: Create-or-activate
2436
runs-on: ubuntu-24.04
2537
if: >-
2638
github.event_name == 'workflow_dispatch' ||
@@ -29,19 +41,11 @@ jobs:
2941
- uses: actions/checkout@v5
3042
with:
3143
# the commit that was built (workflow_run), else the dispatched ref.
44+
# depth 1 is enough: create uses `git rev-parse HEAD`, activate builds
45+
# from the working tree — neither needs history.
3246
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
3347

34-
- name: Resolve image tag and VM name
35-
id: resolve
36-
env:
37-
IN_TAG: ${{ github.event.inputs.image_tag }}
38-
run: |
39-
tag="$IN_TAG"
40-
[ -n "$tag" ] || tag="$(git rev-parse --short HEAD)"
41-
echo "tag=$tag" >> "$GITHUB_OUTPUT"
42-
echo "VM_NAME=${VM_TAG}-${tag}" >> "$GITHUB_ENV"
43-
44-
- name: Configure SSH
48+
- name: Configure exe.dev SSH
4549
env:
4650
SSH_KEY: ${{ secrets.SSH_KEY }}
4751
run: |
@@ -55,36 +59,55 @@ jobs:
5559
| grep -q "SHA256:JJOP/lwiBGOMilfONPWZCXUrfK154cnJFXcqlsi6lPo" \
5660
|| { echo "exe.dev host-key fingerprint mismatch"; exit 1; }
5761
58-
- name: Retire older computer VMs
62+
- name: Is there a computer VM?
63+
id: check
5964
run: |
60-
old=$(ssh -i ~/.ssh/id_ed25519 exe.dev ls --l --json \
61-
| jq -r --arg keep "$VM_NAME" --arg tag "$VM_TAG" '
62-
.vms[]
63-
| select((.tags // []) | index($tag))
64-
| (.vm_name // .name)
65-
| select(type == "string" and . != "" and . != $keep)')
66-
for vm in $old; do
67-
echo "retiring $vm"
68-
ssh -i ~/.ssh/id_ed25519 exe.dev rm "$vm" || true
69-
done
65+
n=$(ssh -i ~/.ssh/id_ed25519 exe.dev ls --l --json \
66+
| jq -r --arg tag "$VM_TAG" '[.vms[] | select((.tags // []) | index($tag))] | length')
67+
echo "found $n VM(s) tagged $VM_TAG"
68+
if [ "$n" -gt 0 ]; then echo "exists=true"; else echo "exists=false"; fi >> "$GITHUB_OUTPUT"
7069
71-
- name: Create the new VM
70+
# ---- bootstrap: no VM yet -> create one (backup secrets from CI) ----------
71+
- name: Create the computer VM
72+
if: steps.check.outputs.exists == 'false'
7273
env:
73-
IMAGE_TAG: ${{ steps.resolve.outputs.tag }}
74+
IN_TAG: ${{ github.event.inputs.image_tag }}
7475
RESTIC_PASSWORD: ${{ secrets.RESTIC_PASSWORD }}
7576
B2_ACCOUNT_KEY: ${{ secrets.B2_ACCOUNT_KEY }}
7677
run: |
78+
tag="$IN_TAG"
79+
[ -n "$tag" ] || tag="$(git rev-parse --short HEAD)"
80+
echo "creating $VM_NAME from ghcr.io/ngalaiko/computer.exe:$tag"
7781
ssh -i ~/.ssh/id_ed25519 exe.dev new \
78-
--image="ghcr.io/ngalaiko/computer.exe:$IMAGE_TAG" \
82+
--image="ghcr.io/ngalaiko/computer.exe:$tag" \
7983
--name "$VM_NAME" \
8084
--tag "$VM_TAG" \
8185
--env RESTIC_REPOSITORY=b2:ngalaiko-backups:exedev \
8286
--env RESTIC_PASSWORD="$RESTIC_PASSWORD" \
8387
--env B2_ACCOUNT_ID=0036cfed748e70c0000000002 \
8488
--env B2_ACCOUNT_KEY="$B2_ACCOUNT_KEY"
85-
86-
# 8080 = services.ingress.publicPort, the public root port.
87-
- name: Publish the public ingress port
88-
run: |
89+
# 8080 = services.ingress.publicPort, the public root port.
8990
ssh -i ~/.ssh/id_ed25519 exe.dev share port "$VM_NAME" 8080
9091
ssh -i ~/.ssh/id_ed25519 exe.dev share set-public "$VM_NAME"
92+
93+
# ---- steady state: VM exists -> activate the new generation in place ------
94+
- uses: nixbuild/nix-quick-install-action@v35
95+
if: steps.check.outputs.exists == 'true'
96+
97+
# The box is reachable only over its tailnet. This tailnet OAuth client is
98+
# the only credential the in-place path needs — NOT an app secret. The client
99+
# id is public (inline); only the secret is a repo secret. The policy must let
100+
# tag:ci tailscale-ssh nikita@ the computer node (README):
101+
# "ssh": [{ "action": "accept", "src": ["tag:ci"],
102+
# "dst": ["tag:computer"], "users": ["nikita"] }]
103+
- name: Join tailnet
104+
if: steps.check.outputs.exists == 'true'
105+
uses: tailscale/github-action@v3
106+
with:
107+
oauth-client-id: kt74FE1o8p11CNTRL
108+
oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
109+
tags: tag:ci
110+
111+
- name: Activate in place
112+
if: steps.check.outputs.exists == 'true'
113+
run: nix run .#deploy

‎README.md‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,43 @@
44

55
nix files for:
66
- my mac
7-
- remove [exe.dev](https://exe.dev) machine
7+
- my remote [exe.dev](https://exe.dev) machine
8+
9+
## Deploying
10+
11+
Everyday deploys are **in place** — they update the running VM without recreating
12+
it, so the Tailscale node, the public URL, and all on-disk state are preserved, and
13+
only changed services reload (unchanged ones keep their PIDs):
14+
15+
- **From your Mac:** `nix run .#deploy`. Builds the system generation, ships its
16+
closure to the box over Tailscale, and runs `<gen>/activate switch`. The Mac is
17+
aarch64 and the box is x86_64, so the build is realised in the box's own store.
18+
- **From CI:** `.github/workflows/deploy.yaml` runs after a green `build` on
19+
`master` and is **self-deciding**:
20+
- **no VM tagged `computer`** → it `exe.dev new`s one from the built image
21+
(bootstrap / disaster recovery), passing the backup secrets so the box can
22+
restore from B2 on boot. This is the only path that uses `RESTIC_PASSWORD` /
23+
`B2_ACCOUNT_KEY` (CI secrets).
24+
- **VM exists** → it activates in place (`nix run .#deploy`) with **no app
25+
secrets** — RESTIC/B2/pilegram already live on the box. This path needs only
26+
tailnet access: a Tailscale OAuth client tagged `tag:ci` (client id inline in
27+
`deploy.yaml`, secret in the `TS_OAUTH_SECRET` repo secret) and a policy rule
28+
letting `tag:ci` SSH the computer node:
29+
30+
```jsonc
31+
"ssh": [{ "action": "accept", "src": ["tag:ci"],
32+
"dst": ["tag:computer"], "users": ["nikita"] }]
33+
```
34+
35+
Roll back with `sudo nix-env -p /nix/var/nix/profiles/system --rollback` then
36+
re-run `activate`. `sudo <gen>/activate test` is a dry run that prints the
37+
overlay/reload plan and changes nothing.
38+
39+
**Base changes still take effect only through a create** (s6-overlay / nix / kernel
40+
layer, the init/mount wrapper, `image.env`): they're excluded from in-place
41+
activation, so they land when CI next has to create a VM — or when you `rm` the VM
42+
to force a fresh one. After a create, the next `deploy` re-establishes the latest
43+
generation, and the base `init-wrapper` hands off to it on `exe.dev restart`.
844
945
## After creating a machine
1046

‎flake.nix‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -48,15 +48,13 @@
4848
allSystems = linuxSystems ++ builtins.attrNames darwinToLinux;
4949
linuxOf = system: darwinToLinux.${system} or system;
5050

51-
imageFor =
51+
configFor =
5252
linuxSystem:
53-
let
54-
exedev = import ./modules/exedev {
55-
pkgs = nixpkgs.legacyPackages.${linuxSystem};
56-
specialArgs = { inherit inputs; };
57-
};
58-
in
59-
(exedev.eval ./hosts/exedev).build.image;
53+
(import ./modules/exedev {
54+
pkgs = nixpkgs.legacyPackages.${linuxSystem};
55+
specialArgs = { inherit inputs; };
56+
}).eval
57+
./hosts/exedev;
6058

6159
releaseFor = system: import ./packages/release { pkgs = nixpkgs.legacyPackages.${system}; };
6260
in
@@ -67,15 +65,17 @@
6765
modules = [ ./hosts/macbook ];
6866
};
6967

70-
# `nix build .#exedev` (current system) or `.#packages.<sys>.exedev`.
68+
# `nix build .#exedev` (base image) or `.#system` (the in-place generation);
69+
# `.#packages.<sys>.{exedev,system}` for a specific arch.
7170
packages = lib.genAttrs allSystems (
7271
system:
7372
let
74-
img = imageFor (linuxOf system);
73+
cfg = configFor (linuxOf system);
7574
in
7675
{
77-
exedev = img;
78-
default = img;
76+
exedev = cfg.build.image;
77+
system = cfg.build.system;
78+
default = cfg.build.image;
7979
}
8080
// releaseFor system
8181
);

‎modules/exedev/activate.nix‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
{
2+
pkgs,
3+
config,
4+
lib,
5+
...
6+
}:
7+
let
8+
inherit (lib) mkOption types;
9+
10+
rootfs = config.build.rootfs;
11+
fixups = config.build.activationFixups;
12+
serviceTree = config.s6.build.serviceTree;
13+
oneshots = config.s6.build.oneshots;
14+
reactivate = config.s6.build.reactivate;
15+
16+
# ABI version: init-wrapper and activate share this contract. Bump on a
17+
# breaking change to the gen/ layout.
18+
version = "1";
19+
20+
# top-level rootfs entries activation must never touch: pseudo/runtime/state
21+
# dirs (never overlaid or pruned), the store itself, and the s6-overlay
22+
# runtime (command/package/init) — bumping s6-overlay is a base change that
23+
# goes through an image rebuild + recreate, not a live switch. `init-wrapper`
24+
# is deliberately NOT excluded, so a switch ships the current boot handoff.
25+
excludes = [
26+
"proc"
27+
"sys"
28+
"dev"
29+
"run"
30+
"nix"
31+
"var"
32+
"home"
33+
"root"
34+
"tmp"
35+
"command"
36+
"package"
37+
"init"
38+
];
39+
40+
# managed rel-paths (files + symlinks, dirs implied) minus the excludes.
41+
manifest = pkgs.runCommand "system-manifest" { } ''
42+
cd ${rootfs}
43+
find . -mindepth 1 \( -type f -o -type l \) -printf '%P\n' \
44+
| grep -Ev '^(${lib.concatStringsSep "|" excludes})(/|$)' \
45+
| LC_ALL=C sort > $out
46+
'';
47+
48+
reactivateFile = pkgs.writeText "reactivate" (lib.concatStrings (map (n: n + "\n") reactivate));
49+
50+
activate = pkgs.runCommand "activate" { } ''
51+
substitute ${./activate.sh} $out \
52+
--subst-var-by bash ${pkgs.bashInteractive} \
53+
--subst-var-by coreutils ${pkgs.coreutils} \
54+
--subst-var-by gnugrep ${pkgs.gnugrep} \
55+
--subst-var-by diffutils ${pkgs.diffutils} \
56+
--subst-var-by utilLinux ${pkgs.util-linux} \
57+
--subst-var-by nix ${pkgs.nix} \
58+
--subst-var-by s6 ${config.s6.package}/command
59+
chmod +x $out
60+
${pkgs.bashInteractive}/bin/bash -n $out
61+
'';
62+
63+
# The generation: a self-contained, GC-rootable closure. The symlinks pull
64+
# rootfs/service/oneshots/fixups (and their whole closures) into build.system,
65+
# so `nix copy .#system` ships everything and the system profile roots it all.
66+
system = pkgs.runCommand "computer-system" { } ''
67+
mkdir -p $out
68+
ln -s ${rootfs} $out/rootfs
69+
ln -s ${serviceTree} $out/service
70+
ln -s ${oneshots} $out/oneshots
71+
ln -s ${fixups} $out/fixups
72+
cp ${manifest} $out/manifest
73+
cp ${reactivateFile} $out/reactivate
74+
printf '%s\n' ${version} > $out/version
75+
install -m0755 ${activate} $out/activate
76+
'';
77+
in
78+
{
79+
options.build.system = mkOption {
80+
type = types.package;
81+
readOnly = true;
82+
description = "The in-place-deployable system generation (nix copy + <gen>/activate switch).";
83+
};
84+
85+
config.build.system = system;
86+
}

0 commit comments

Comments
 (0)