fix acme challenge

Author: terion-labs

.github/workflows/validate.yml

@@ -19,7 +19,15 @@ jobs:
           bun install
           bun run build
           bun run docs:build
-          ansible-galaxy collection install -r requirements.yml
+          for attempt in 1 2 3 4; do
+            if ansible-galaxy collection install -r requirements.yml; then
+              break
+            fi
+            if [ "$attempt" -eq 4 ]; then
+              exit 1
+            fi
+            sleep $((attempt * 5))
+          done
       - name: Script validation
         run: |
           bash -n install.sh

ansible/roles/traefik/templates/traefik.yml.j2

@@ -18,7 +18,8 @@ certificatesResolvers:
     acme:
       email: {{ terrarium_acme_email }}
       storage: /var/lib/traefik/acme.json
-      tlsChallenge: {}
+      httpChallenge:
+        entryPoint: web
 
 log:
   level: INFO

ansible/site.yml

@@ -291,10 +291,6 @@
       changed_when: terrarium_bootstrap_tls_retired is changed
       when: terrarium_bootstrap_tls_retired is changed
 
-    - name: Restart LXD after retiring bootstrap TLS trust
-      ansible.builtin.command: systemctl try-restart snap.lxd.daemon.service
-      when: terrarium_bootstrap_tls_retired is changed
-
     - name: Restart Traefik after retiring bootstrap TLS
       ansible.builtin.systemd:
         name: traefik

scripts/terrarium-install.test.ts

@@ -81,4 +81,14 @@ describe("terrarium install CLI parsing", () => {
     expect(generated.length).toBeGreaterThanOrEqual(40);
     expect(generated).toMatch(/^trm-[A-Za-z0-9_-]+$/);
   });
+
+  test("retries transient Ansible Galaxy collection download failures", () => {
+    const source = readFileSync(join(repoRoot, "scripts/terrarium-install.ts"), "utf8");
+
+    expect(source).toContain("const ANSIBLE_GALAXY_ATTEMPTS = 4");
+    expect(source).toContain("async function installAnsibleCollections()");
+    expect(source).toContain("ansible-galaxy collection install -r requirements.yml");
+    expect(source).toContain("failed on attempt ${attempt}/${ANSIBLE_GALAXY_ATTEMPTS}; retrying");
+    expect(source).toContain("failed after ${ANSIBLE_GALAXY_ATTEMPTS} attempts");
+  });
 });

scripts/terrarium-install.ts

@@ -16,6 +16,7 @@ const REPO_URL = process.env.TERRARIUM_REPO_URL ?? "https://github.com/terion-na
 const REPO_DIR = process.env.TERRARIUM_REPO_DIR ?? "/opt/terrarium";
 const BUNDLE_DIR = process.env.TERRARIUM_BUNDLE_DIR ?? "";
 const GENERATED_ROOT_PASSWORD_PATH = "/etc/terrarium/secrets/cockpit_root_password";
+const ANSIBLE_GALAXY_ATTEMPTS = 4;
 // CAC's string transform runs after numeric coercion; readCliOption recovers exact argv values instead.
 const STRING_OPTION = {};
 
@@ -282,6 +283,24 @@ function syncLocalSourceRepo(sourcePath: string, repoDir: string): void {
   });
 }
 
+async function installAnsibleCollections(): Promise<void> {
+  let lastOutput = "";
+  for (let attempt = 1; attempt <= ANSIBLE_GALAXY_ATTEMPTS; attempt += 1) {
+    const result = await $`cd ${REPO_DIR}; ansible-galaxy collection install -r requirements.yml`.nothrow();
+    if (result.exitCode === 0) {
+      return;
+    }
+
+    lastOutput = `${result.stdout.toString()}\n${result.stderr.toString()}`.trim();
+    if (attempt < ANSIBLE_GALAXY_ATTEMPTS) {
+      warn(`ansible-galaxy collection install failed on attempt ${attempt}/${ANSIBLE_GALAXY_ATTEMPTS}; retrying`);
+      await Bun.sleep(attempt * 5000);
+    }
+  }
+
+  fail(`ansible-galaxy collection install failed after ${ANSIBLE_GALAXY_ATTEMPTS} attempts${lastOutput ? `\n${lastOutput}` : ""}`);
+}
+
 async function prepareRepo(ref: string): Promise<void> {
   const sourcePath = localSourcePath(REPO_URL);
   if (sourcePath && existsSync(join(sourcePath, "ansible", "site.yml"))) {
@@ -305,7 +324,7 @@ async function prepareRepo(ref: string): Promise<void> {
     fail("compiled Terrarium binaries are missing from the repository checkout");
   }
 
-  await $`cd ${REPO_DIR}; ansible-galaxy collection install -r requirements.yml`;
+  await installAnsibleCollections();
 }
 
 function dashedIp(ip: string): string {

scripts/terrarium-traefik-sync.ts

@@ -1328,7 +1328,9 @@ function buildStaticConfig(config: Record<string, unknown>, extraEntrypoints: Re
         acme: {
           email: configString(config, "terrarium_acme_email") || configString(config, "terrarium_email"),
           storage: "/var/lib/traefik/acme.json",
-          tlsChallenge: {}
+          httpChallenge: {
+            entryPoint: "web"
+          }
         }
       }
     },

tests/ansible-bootstrap-cert.test.ts

@@ -32,8 +32,9 @@ describe("Traefik bootstrap certificate template", () => {
 
     expect(tasks).toContain("[terrarium_auth_domain] if terrarium_bootstrap_tls_enabled else []");
     expect(tasks).not.toContain("'*.' ~ terrarium_bootstrap_tls_root_domain");
-    expect(traefikConfig).toContain("tlsChallenge: {}");
-    expect(traefikConfig).not.toContain("httpChallenge:");
+    expect(traefikConfig).toContain("httpChallenge:");
+    expect(traefikConfig).toContain("entryPoint: web");
+    expect(traefikConfig).not.toContain("tlsChallenge:");
     expect(certConfig).toContain("certificates:");
     expect(certConfig).not.toContain("defaultCertificate");
     expect(dynamicConfig).not.toContain("zitadel-root-bootstrap");
@@ -48,19 +49,23 @@ describe("Traefik bootstrap certificate template", () => {
     expect(bootstrapRoutes).toContain("zitadel-root-bootstrap");
     expect(tasks).toContain("Remove Traefik bootstrap certificate config when bootstrap TLS is not required");
     expect(tasks).toContain("Remove temporary Traefik bootstrap routes when bootstrap TLS is not required");
-    expect(tasks).toContain("systemctl try-restart snap.lxd.daemon.service");
     expect(playbook).toContain("Retire local auth bootstrap TLS before verifying public TLS");
     expect(playbook).toContain("dynamic/bootstrap-routes.yml");
     expect(playbook).toContain('"{{ terrarium_traefik_config_dir }}/dynamic/bootstrap-cert.yml"');
     expect(playbook).toContain('"{{ terrarium_traefik_config_dir }}/bootstrap-certs"');
     expect(playbook).toContain("/usr/local/share/ca-certificates/terrarium-bootstrap.crt");
     expect(playbook).toContain("Refresh system CA certificates after retiring bootstrap TLS");
-    expect(playbook).toContain("Restart LXD after retiring bootstrap TLS trust");
+    expect(playbook).not.toContain("Restart LXD after retiring bootstrap TLS trust");
+    expect(playbook).not.toContain("systemctl try-restart snap.lxd.daemon.service");
+    expect(playbook).not.toContain("lxc config device remove");
+    expect(playbook).not.toContain("lxc config device add");
+    expect(playbook).not.toContain("Wait for ZITADEL login loopback proxy after LXD restart");
     expect(playbook).toContain("Restart Traefik after retiring bootstrap TLS");
     expect(playbook).toContain("Wait for local auth domain to serve public TLS");
     expect(playbook).toContain("Restart Traefik to retry local auth ACME after public TLS wait failure");
     expect(playbook).toContain("Wait again for local auth domain to serve public TLS after ACME retry");
     expect(playbook).toContain("Verify local auth domain serves public TLS after waits");
+    expect(playbook).toContain("- -fsS");
     expect(playbook).toContain("Show local auth TLS diagnostics after public TLS failure");
     expect(playbook).toContain('TERRARIUM_AUTH_DOMAIN: "{{ terrarium_auth_domain }}"');
     expect(playbook).toContain('TERRARIUM_TRAEFIK_CONFIG_DIR: "{{ terrarium_traefik_config_dir }}"');

tests/release-workflow.test.ts

@@ -17,6 +17,15 @@ describe("release workflow", () => {
     expect(release.on.push.tags).toEqual(["*"]);
   });
 
+  test("validate workflow retries transient Ansible Galaxy collection failures", () => {
+    const source = readFileSync(join(repoRoot, ".github/workflows/validate.yml"), "utf8");
+
+    expect(source).toContain("for attempt in 1 2 3 4; do");
+    expect(source).toContain("ansible-galaxy collection install -r requirements.yml");
+    expect(source).toContain('if [ "$attempt" -eq 4 ]; then');
+    expect(source).toContain("sleep $((attempt * 5))");
+  });
+
   test("validates release tags before preflight and publishing", () => {
     const source = readFileSync(join(repoRoot, ".github/workflows/release.yml"), "utf8");
     const workflow = YAML.parse(source);

tests/terrarium-traefik-sync.test.ts

@@ -76,11 +76,12 @@ describe("terrarium route auth generation", () => {
     expect(source).toContain("readLocalZitadelPat(config)");
   });
 
-  test("uses TLS-ALPN ACME challenges in generated Traefik static config", () => {
+  test("uses HTTP ACME challenges in generated Traefik static config", () => {
     const source = readFileSync(join(repoRoot, "scripts/terrarium-traefik-sync.ts"), "utf8");
 
-    expect(source).toContain("tlsChallenge: {}");
-    expect(source).not.toContain("httpChallenge:");
+    expect(source).toContain("httpChallenge:");
+    expect(source).toContain('entryPoint: "web"');
+    expect(source).not.toContain("tlsChallenge:");
   });
 
   test("treats ZITADEL no-op updates as successful idempotent responses", () => {