Improve onboarding for cloud instances

docs/installation.md

@@ -2,11 +2,11 @@
 # Installation {#installation}
 
 The installation is split in 3 major steps - A, B and C.
-Each individual step may be split it in parallel tracks,
-depending on your setup.
+Each individual step is split in parallel tracks
+and you should follow the one matching your setup.
 
 - Step A: [Repository](#a-repository).
-  - If you don't have an existing repository, follow the [A.1. bootstrapping](#a-repository-bootstrapping)) method.
+  - If you don't have an existing repository, follow the [A.1. bootstrapping](#a-repository-bootstrapping) method.
   - If you do have one, follow the [A.2. Add in Existing Repo](#a-repository-existing-repo) method.
 
 - Step B: [Beacon](#b-beacon). The installation can be done on:
@@ -40,7 +40,13 @@ $ cd myskarabox
 $ nix run github:ibizaman/skarabox?ref=v@VERSION@#init
 ```
 
-The `#init` command accepts arguments, add `-- -h` so see them.
+The `#init` command accepts arguments.
+Use the `-h` argument to print them.
+Don't forget the double dash `--` like so:
+
+```bash
+$ nix run github:ibizaman/skarabox?ref=v@VERSION@#init -- -h
+```
 
 The `#init` command by default asks for the password you want for the admin user
 and will generate all other secrets.
@@ -50,10 +56,9 @@ under the [./myskarabox](@REPO@/template/myskarabox) folder.
 All the files at the root of this new repository
 are common to all hosts.
 
-It will finally ask you to fill out two options: `skarabox.hosts.<name>.ip` and `skarabox.hosts.<name>.system`
-in  [./configuration.nix](@REPO@/template/myskarabox/configuration.nix)
-then afterwards to generate [./known_hosts](@REPO@/template/myskarabox/known_hosts).
-Detailed instructions will be shown in Step B.
+It will finally ask you to fill out options in the main [./flake.nix](@REPO@/template/flake.nix) file
+and in the [./myskarabox/configuration.nix](@REPO@/template/myskarabox/configuration.nix) file.
+Detailed instructions are shown in [Step B](#b-beacon).
 
 ### A. (option 2) Add in Existing Repo {#a-repository-existing-repo}
 
@@ -88,7 +93,7 @@ to a repository using [nix-starter-configs](https://github.com/Misterio77/nix-st
 
    [./myskarabox/configuration.nix]: @REPO@/template/myskarabox/configuration.nix
 
-   Now, pick one of the Step B underneath.
+Continue with [Step B](#b-beacon).
 
 ## B. Beacon {#b-beacon}
 
@@ -106,12 +111,6 @@ skarabox.hosts.<name> = {
 };
 ```
 
-Generate the known hosts file:
-
-```bash
-nix run .#myskarabox-gen-knownhosts-file
-```
-
 Then, start the VM:
 
 ```bash
@@ -229,42 +228,74 @@ Connecting to the beacon will thus depend on the chosen method.
    It is also possible to use a hostname as long as it can resolve to the correct IP,
    for example if you set up your ssh config accordingly.
 
-8. Generate the known host file:
-
-   ```bash
-   $ nix run .#myskarabox-gen-knownhosts-file
-   ```
-
-   Redo this step if any of the ssh port or IP under the flake `skarabox.hosts.<name>` option changes.
-
-9. Open the various files just to see if everything looks good.
+8. Open the various files just to see if everything looks good.
 
 ### B. (option 3) Install on a Cloud Server {#b-beacon-cloud}
 
 No need for the beacon here.
 You just need to start the instance and enable ssh access.
 The easiest is usually to boot in rescue mode.
 
-Retrieve the IP of the server, then update the values:
+Retrieve the IP of the server, then update the values in the `flake.nix` file:
 
 ```nix
 skarabox.hosts.<name> = {
   system = "x86_64-linux";
-  ip = "192.168.1.30";
+  ip = "12.34.56.78";
 }
 ```
 
-Generate the known hosts file:
+Update the `beacon` options to be able to ssh into the instance,
+with the values given by your cloud provider:
 
-```bash
-nix run .#myskarabox-gen-knownhosts-file
+```nix
+skarabox.hosts.<name>.beacon = {
+  username = "<given by cloud provider>";
+  sshPort = "<given by cloud provider>";
+  sshPrivateKeyPath = "<given by cloud provider>";
+  sshPublicKeyPath = "<given by cloud provider>";
+}
 ```
 
 [nixos-anywhere]: https://github.com/nix-community/nixos-anywhere
 
+Test the ssh connection with:
+
+```bash
+$ nix run .#myskarabox-ssh-beacon -- -v
+```
+
 ## C. Run the Installer {#c-installer}
 
-Create a `./myskarabox/facter.json` file containing
+1. Get into NixOS installer
+
+```bash
+$ nix run .#myskarabox-install-on-beacon -- --phases kexec
+```
+
+1. Get the disk layout
+
+```bash
+$ nix run .#myskarabox-ssh-beacon -- fdisk -l
+$ nix run .#myskarabox-ssh-beacon -- lsblk
+```
+
+Set the options `skarabox.disks` in `./myskarabox/configuration.nix` accordingly.
+
+2. Generate the known hosts file:
+
+```bash
+nix run .#myskarabox-gen-knownhosts-file
+```
+
+Redo this step if any of the ssh port or IP under the flake `skarabox.hosts.<name>` option changes.
+
+Note that the host key generated by Skarabox is not used in the beacon
+so it is expected for the steps in this section to not care about it.
+
+3. Generate target host hardware configuration
+
+Generate a `./myskarabox/facter.json` file containing
 the hardware specification of the host (or the VM) with:
 
 ```bash
@@ -289,6 +320,13 @@ Now, run the installation process on the target host:
 $ nix run .#myskarabox-install-on-beacon
 ```
 
+Additional flags can be added like:
+
+```bash
+$ nix run .#myskarabox-install-on-beacon -- -h
+$ nix run .#myskarabox-install-on-beacon -- --debug
+```
+
 The server will reboot into NixOS on its own.
 Upon booting, the root partition will need to be decrypted
 as outlined in the [Normal Operations](./normal-operations.html#decrypt-root) section.

flakeModules/default.nix

@@ -18,10 +18,10 @@ let
       ../modules/beacon.nix
       (modulesPath + "/profiles/minimal.nix")
       {
-        skarabox.username = hostCfg.skarabox.username;
         skarabox.hostname = "${hostCfg.skarabox.hostname}-beacon";
-        skarabox.sshAuthorizedKey = hostCfg.skarabox.sshAuthorizedKey;
         skarabox.staticNetwork = hostCfg.skarabox.staticNetwork;
+        skarabox.username = hostCfg.skarabox.username;
+        skarabox.sshAuthorizedKey = hostCfg.skarabox.sshAuthorizedKey;
         skarabox.sshPort = hostCfg.skarabox.sshPort;
         skarabox.hotspot.ip = lib.mkIf (hostCfg.skarabox.staticNetwork != null) hostCfg.skarabox.staticNetwork.ip;
         boot.initrd.network.udhcpc.enable = hostCfg.skarabox.staticNetwork == null;
@@ -43,7 +43,7 @@ in
     hosts = mkOption {
       description = "Hosts managed by Skarabox.";
       default = {};
-      type = types.attrsOf (types.submodule ({ name, ... }: {
+      type = types.attrsOf (types.submodule ({ name, config, ... }: {
         options = {
           nixpkgs = mkOption {
             # No evaluation or merging is wanted here, thus the raw type.
@@ -121,6 +121,19 @@ in
             type = types.nullOr types.str;
             default = "${name}/ssh";
           };
+          sshPublicKeyPath = lib.mkOption {
+            description = ''
+              Path from the top of the repo to the ssh public file used to ssh into the host.
+
+              Should be only necessary when using an ssh agent which has a lot of ssh keys.
+              Setting this option allows the ssh agent to know which key to give,
+              otherwise ssh might fail with "too many authentication attempts".
+
+              If set, you should probably set sshPrivateKeyPath to null.
+            '';
+            type = types.nullOr types.str;
+            default = null;
+          };
           secretsFilePath = mkOption {
             description = ''
               Path from the top of the repo to the SOPS secrets file.
@@ -171,6 +184,59 @@ in
             apply = readAsStr;
           };
 
+          beacon = {
+            username = lib.mkOption {
+              description = ''
+                Username with which you can log into the beacon.
+
+                Defaults to the hostname configured for the target host which works well if using the beacon.
+
+                If installing on a cloud instance, set this to the user that can ssh into the instance as given by your cloud provider.
+              '';
+              type = types.str;
+              default = "skarabox";
+            };
+
+            sshPort = lib.mkOption {
+              type = types.int;
+              description = ''
+                Port the SSH daemon listens to on the beacon.
+
+                Defaults to the same port configured for the target host which works well if using the beacon.
+
+                If installing on a cloud instance, set this to the port that can ssh into the instance as given by your cloud provider, usually 22.
+              '';
+              default = config.sshPort;
+            };
+
+            sshPrivateKeyPath = lib.mkOption {
+              description = ''
+                Path from the top of the repo to the ssh private file used to ssh into the host.
+
+                Defaults to the ssh key for the target host `skarabox.hosts.<name>.sshPublicKeyPath`.
+
+                Set to null if you use an ssh agent. See also sshPublicKeyPath.
+              '';
+              type = types.nullOr types.str;
+              default = config.sshPrivateKeyPath;
+            };
+            sshPublicKeyPath = lib.mkOption {
+              description = ''
+                Path from the top of the repo to the ssh public file used to ssh into the host.
+
+                Defaults to the ssh key for the target host `skarabox.hosts.<name>.sshPublicKeyPath`.
+
+                Should be only necessary when using an ssh agent which has a lot of ssh keys.
+                Setting this option allows the ssh agent to know which key to give,
+                otherwise ssh might fail with "too many authentication attempts".
+
+                If set, you should probably set sshPrivateKeyPath to null.
+              '';
+              type = types.nullOr types.str;
+              default = config.sshPublicKeyPath;
+            };
+          };
+
           modules = mkOption {
             description = "Modules to add to the host nixosConfiguration. Add here all your own configuration.";
             type = types.listOf types.anything;
@@ -230,6 +296,7 @@ in
               -o UserKnownHostsFile=${cfg'.knownHosts} \
               -o ConnectTimeout=10 \
               ${if cfg'.sshPrivateKeyPath != null then "-i ${cfg'.sshPrivateKeyPath}" else ""} \
+              ${if cfg'.sshPublicKeyPath != null then "-i ${cfg'.sshPublicKeyPath}" else ""} \
               "$*"
           '';
         };
@@ -323,10 +390,10 @@ in
 
             set -x
 
-            guestport=${toString cfg'.sshPort}
-            hostport=${toString hostCfg.skarabox.sshPort}
-            guestbootport=${toString cfg'.sshBootPort}
-            hostbootport=${toString hostCfg.skarabox.boot.sshPort}
+            guestport=${toString (lib.traceValFn (x: "guestport=${toString x}") cfg'.sshPort)}
+            hostport=${toString (lib.traceValFn (x: "hostport=${toString x}")cfg'.beacon.sshPort)}
+            guestbootport=${toString (lib.traceValFn (x: "guestbootport=${toString x}")cfg'.sshBootPort)}
+            hostbootport=${toString (lib.traceValFn (x: "hostbootport=${toString x}")hostCfg.skarabox.boot.sshPort)}
 
             ${qemu} \
               -m 2048M \
@@ -418,7 +485,7 @@ in
                 mapAttrsToList mkTmpFile secrets;
             in ''
               ip=${toString cfg'.ip}
-              ssh_port=${toString cfg'.sshPort}
+              ssh_port=${toString cfg'.beacon.sshPort}
               flake=".#${toString name}"
 
               export SOPS_AGE_KEY_FILE="${cfg.sopsKeyPath}"
@@ -429,7 +496,8 @@ in
                 # Build the extra arguments list properly
                 extraArgs = []
                   ++ [ "--ssh-option" "ConnectTimeout=10" ]
-                  ++ (lib.optionals (cfg'.sshPrivateKeyPath != null) [ "-i" cfg'.sshPrivateKeyPath ])
+                  ++ (lib.optionals (cfg'.beacon.sshPrivateKeyPath != null) [ "-i" cfg'.beacon.sshPrivateKeyPath ])
+                  ++ (lib.optionals (cfg'.beacon.sshPublicKeyPath != null) [ "--ssh-public-key" cfg'.beacon.sshPublicKeyPath ])
                   ++ [ "--disk-encryption-keys" "/tmp/host_key" cfg'.hostKeyPath ]
                   ++ (lib.flatten (mapAttrsToList (name: path: [ "--disk-encryption-keys" "/tmp/${name}" "\$secret_file_${name}" ]) secrets));
 
@@ -439,7 +507,7 @@ in
 
               install-on-beacon \
                 -i "$ip" \
-                -u ${hostCfg.skarabox.username} \
+                -u ${cfg'.beacon.username} \
                 -p "$ssh_port" \
                 -f "$flake" \
                 -- \
@@ -470,6 +538,29 @@ in
                 -o UserKnownHostsFile=${cfg'.knownHosts} \
                 -o ConnectTimeout=10 \
                 ${if cfg'.sshPrivateKeyPath != null then "-i ${cfg'.sshPrivateKeyPath}" else ""} \
+                ${if cfg'.sshPublicKeyPath != null then "-i ${cfg'.sshPublicKeyPath}" else ""} \
+                "$@"
+            '';
+          };
+
+          ssh-beacon = pkgs.writeShellApplication {
+            name = "ssh";
+
+            runtimeInputs = [
+              (import ../lib/ssh.nix {
+                inherit pkgs;
+              })
+            ];
+
+            text = ''
+              ssh \
+                "${cfg'.ip}" \
+                "${toString cfg'.beacon.sshPort}" \
+                ${cfg'.beacon.username} \
+                -o ConnectTimeout=10 \
+                -o StrictHostKeyChecking=no \
+                ${if cfg'.beacon.sshPrivateKeyPath != null then "-i ${cfg'.beacon.sshPrivateKeyPath}" else ""} \
+                ${if cfg'.beacon.sshPublicKeyPath != null then "-i ${cfg'.beacon.sshPublicKeyPath}" else ""} \
                 "$@"
             '';
           };
@@ -478,11 +569,11 @@ in
             name = "get-facter";
 
             runtimeInputs = [
-              ssh
+              ssh-beacon
             ];
 
             text = ''
-              ssh -o StrictHostKeyChecking=no sudo nixos-facter
+              ssh sudo nixos-facter
             '';
           };
 
@@ -504,6 +595,7 @@ in
           "${name}-sops" = sops;
           "${name}-beacon" = beacon;
           "${name}-beacon-vm" = beacon-vm;
+          "${name}-ssh-beacon" = ssh-beacon;
           "${name}-gen-knownhosts-file" = gen-knownhosts-file;
           "${name}-install-on-beacon" = install-on-beacon;
           "${name}-ssh" = ssh;