app.ts

import express, { Router, type Request as ExpressRequest } from "express";
import path from "node:path";
import fs from "node:fs";
import { fileURLToPath } from "node:url";
import type { Db } from "@paperclipai/db";
import type { DeploymentExposure, DeploymentMode } from "@paperclipai/shared";
import type { StorageService } from "./storage/types.js";
import { httpLogger, errorHandler } from "./middleware/index.js";
import { actorMiddleware } from "./middleware/auth.js";
import { boardMutationGuard } from "./middleware/board-mutation-guard.js";
import { privateHostnameGuard, resolvePrivateHostnameAllowSet } from "./middleware/private-hostname-guard.js";
import { applyTrustProxy, parseTrustProxyEnv } from "./middleware/trust-proxy.js";
import { healthRoutes } from "./routes/health.js";
import { companyRoutes } from "./routes/companies.js";
import { companySkillRoutes } from "./routes/company-skills.js";
import { companySkillPolicyRoutes } from "./routes/company-skill-policy.js";
import { inboxAgentPolicyRoutes } from "./routes/inbox-agent-policy.js";
import { builtInAgentRoutes } from "./routes/built-in-agents.js";
import { folderRoutes } from "./routes/folders.js";
import { summarySlotRoutes } from "./routes/summary-slots.js";
import { teamsCatalogRoutes } from "./routes/teams-catalog.js";
import { agentRoutes } from "./routes/agents.js";
import { projectRoutes } from "./routes/projects.js";
import { issueRoutes } from "./routes/issues.js";
import { issueTreeControlRoutes } from "./routes/issue-tree-control.js";
import { caseRoutes } from "./routes/cases.js";
import { fileResourceRoutes } from "./routes/file-resources.js";
import { routineRoutes } from "./routes/routines.js";
import { pipelineRoutes } from "./routes/pipelines.js";
import { environmentRoutes } from "./routes/environments.js";
import { executionWorkspaceRoutes } from "./routes/execution-workspaces.js";
import { goalRoutes } from "./routes/goals.js";
import { boardChatRoutes } from "./routes/board-chat.js";
import { approvalRoutes } from "./routes/approvals.js";
import { secretRoutes } from "./routes/secrets.js";
import { toolAccessRoutes } from "./routes/tool-access.js";
import { smokeLabRoutes } from "./routes/smoke-lab.js";
import { costRoutes } from "./routes/costs.js";
import { activityRoutes } from "./routes/activity.js";
import { dashboardRoutes } from "./routes/dashboard.js";
import { attentionRoutes } from "./routes/attention.js";
import { decisionTrainingRoutes } from "./routes/decision-training.js";
import { userProfileRoutes } from "./routes/user-profiles.js";
import { sidebarBadgeRoutes } from "./routes/sidebar-badges.js";
import { sidebarPreferenceRoutes } from "./routes/sidebar-preferences.js";
import { resourceMembershipRoutes } from "./routes/resource-memberships.js";
import { inboxDismissalRoutes } from "./routes/inbox-dismissals.js";
import { instanceSettingsRoutes } from "./routes/instance-settings.js";
import { openApiRoutes } from "./routes/openapi.js";
import {
  instanceDatabaseBackupRoutes,
  type InstanceDatabaseBackupService,
} from "./routes/instance-database-backups.js";
import { llmRoutes } from "./routes/llms.js";
import { authRoutes } from "./routes/auth.js";
import { assetRoutes } from "./routes/assets.js";
import { accessRoutes } from "./routes/access.js";
import { pluginRoutes } from "./routes/plugins.js";
import { mcpGatewayProtocolRoutes, toolGatewayRoutes } from "./routes/tool-gateway.js";
import { adapterRoutes } from "./routes/adapters.js";
import { pluginUiStaticRoutes } from "./routes/plugin-ui-static.js";
import { readBrandedStaticIndexHtml } from "./static-index-html.js";
import { applyUiBranding, BRAND_DIR_PUBLIC_PATH, getBrandDir } from "./ui-branding.js";
import { logger } from "./middleware/logger.js";
import { DEFAULT_LOCAL_PLUGIN_DIR, pluginLoader } from "./services/plugin-loader.js";
import { createPluginWorkerManager, type PluginWorkerManager } from "./services/plugin-worker-manager.js";
import { setProcessPluginWorkerManager } from "./services/plugin-worker-process-registry.js";
import { createPluginJobScheduler } from "./services/plugin-job-scheduler.js";
import { pluginJobStore } from "./services/plugin-job-store.js";
import { createPluginToolDispatcher } from "./services/plugin-tool-dispatcher.js";
import { createToolGatewayService } from "./services/tool-gateway.js";
import { pluginLifecycleManager } from "./services/plugin-lifecycle.js";
import { decideBundledPluginAction } from "./services/bundled-plugin-heal.js";
import { createPluginJobCoordinator } from "./services/plugin-job-coordinator.js";
import { buildHostServices, flushPluginLogBuffer } from "./services/plugin-host-services.js";
import { createPluginEventBus } from "./services/plugin-event-bus.js";
import { setPluginEventBus } from "./services/activity-log.js";
import { createPluginDevWatcher } from "./services/plugin-dev-watcher.js";
import { createPluginHostServiceCleanup } from "./services/plugin-host-service-cleanup.js";
import { pluginRegistryService } from "./services/plugin-registry.js";
import { createHostClientHandlers } from "@paperclipai/plugin-sdk";
import type { BetterAuthSessionResult } from "./auth/better-auth.js";
import { createCachedViteHtmlRenderer } from "./vite-html-renderer.js";
import { DEFAULT_JSON_BODY_LIMIT, PORTABLE_JSON_BODY_LIMIT } from "./http/body-limits.js";
import { COMPANY_IMPORT_API_PATH } from "./routes/company-import-paths.js";
import { apiCompression } from "./middleware/api-compression.js";

type UiMode = "none" | "static" | "vite-dev";
const FEEDBACK_EXPORT_FLUSH_INTERVAL_MS = 5_000;
const VITE_DEV_ASSET_PREFIXES = [
  "/@fs/",
  "/@id/",
  "/@react-refresh",
  "/@vite/",
  "/assets/",
  "/node_modules/",
  "/src/",
];
const VITE_DEV_STATIC_PATHS = new Set([
  "/apple-touch-icon.png",
  "/favicon-16x16.png",
  "/favicon-32x32.png",
  "/favicon.ico",
  "/favicon.svg",
  "/site.webmanifest",
  "/sw.js",
]);

export function isDatabaseConnectionUnavailableError(err: unknown): boolean {
  const error = err as { code?: unknown; message?: unknown; cause?: unknown };
  if (error?.code === "ECONNREFUSED") return true;
  return Boolean(error?.cause && isDatabaseConnectionUnavailableError(error.cause));
}

export function resolveViteHmrPort(serverPort: number): number {
  if (serverPort <= 55_535) {
    return serverPort + 10_000;
  }
  return Math.max(1_024, serverPort - 10_000);
}

export function resolveViteHmrHost(bindHost: string): string | undefined {
  const normalized = bindHost.trim().toLowerCase();
  if (normalized === "0.0.0.0" || normalized === "::") return undefined;
  return bindHost;
}

export function shouldServeViteDevHtml(req: ExpressRequest): boolean {
  const pathname = req.path;
  if (VITE_DEV_STATIC_PATHS.has(pathname)) return false;
  if (VITE_DEV_ASSET_PREFIXES.some((prefix) => pathname.startsWith(prefix))) return false;
  return req.accepts(["html"]) === "html";
}

/**
 * Serves the deployer-mounted brand directory (PAPERCLIP_BRAND_DIR) under
 * /branding — the stylesheet link applyUiBranding injects points here. Without
 * this route the request falls through to the SPA fallback and comes back as
 * text/html, which the browser refuses to apply as a stylesheet. A missing
 * brand asset 404s for the same reason. No-op when no brand dir is configured,
 * so the default build's routing is unchanged.
 */
export function registerBrandStaticRoute(app: express.Express, env: NodeJS.ProcessEnv = process.env): boolean {
  const brandDir = getBrandDir(env);
  if (!brandDir) return false;
  app.use(BRAND_DIR_PUBLIC_PATH, express.static(brandDir, { index: false, maxAge: "5m" }));
  app.use(BRAND_DIR_PUBLIC_PATH, (_req, res) => {
    res.status(404).end();
  });
  return true;
}

export function shouldEnablePrivateHostnameGuard(opts: {
  deploymentMode: DeploymentMode;
  deploymentExposure: DeploymentExposure;
}): boolean {
  return (
    opts.deploymentExposure === "private" &&
    (opts.deploymentMode === "local_trusted" || opts.deploymentMode === "authenticated")
  );
}

export async function createApp(
  db: Db,
  opts: {
    uiMode: UiMode;
    serverPort: number;
    storageService: StorageService;
    feedbackExportService?: {
      flushPendingFeedbackTraces(input?: {
        companyId?: string;
        traceId?: string;
        limit?: number;
        now?: Date;
      }): Promise<unknown>;
    };
    databaseBackupService?: InstanceDatabaseBackupService;
    deploymentMode: DeploymentMode;
    deploymentExposure: DeploymentExposure;
    allowedHostnames: string[];
    bindHost: string;
    authReady: boolean;
    companyDeletionEnabled: boolean;
    instanceId?: string;
    hostVersion?: string;
    localPluginDir?: string;
    pluginMigrationDb?: Db;
    pluginWorkerManager?: PluginWorkerManager;
    betterAuthHandler?: express.RequestHandler;
    resolveSession?: (req: ExpressRequest) => Promise<BetterAuthSessionResult | null>;
  },
) {
  const app = express();
  app.locals.paperclipDb = db;
  const captureRawBody = (req: express.Request, _res: express.Response, buf: Buffer) => {
    (req as unknown as { rawBody: Buffer }).rawBody = buf;
  };

  // Respect the operator's `TRUST_PROXY` env var (see middleware/trust-proxy.ts).
  // Default is unset → Express trusts nothing, which is the only safe choice
  // when the server may be reachable without a known reverse proxy in front.
  applyTrustProxy(app, parseTrustProxyEnv(process.env.TRUST_PROXY));

  app.use(COMPANY_IMPORT_API_PATH, express.json({
    limit: PORTABLE_JSON_BODY_LIMIT,
    verify: captureRawBody,
  }));
  app.use(express.json({
    limit: DEFAULT_JSON_BODY_LIMIT,
    verify: captureRawBody,
  }));
  app.use("/api", apiCompression());
  app.use(httpLogger);
  const privateHostnameGateEnabled = shouldEnablePrivateHostnameGuard({
    deploymentMode: opts.deploymentMode,
    deploymentExposure: opts.deploymentExposure,
  });
  const privateHostnameAllowSet = resolvePrivateHostnameAllowSet({
    allowedHostnames: opts.allowedHostnames,
    bindHost: opts.bindHost,
  });
  app.use(
    privateHostnameGuard({
      enabled: privateHostnameGateEnabled,
      allowedHostnames: opts.allowedHostnames,
      bindHost: opts.bindHost,
    }),
  );
  app.use(
    actorMiddleware(db, {
      deploymentMode: opts.deploymentMode,
      resolveSession: opts.resolveSession,
    }),
  );
  app.use("/api/auth", authRoutes(db));
  if (opts.betterAuthHandler) {
    app.all("/api/auth/{*authPath}", opts.betterAuthHandler);
  }
  app.use(llmRoutes(db));

  const hostServicesDisposers = new Map<string, () => void>();
  const workerManager = opts.pluginWorkerManager ?? createPluginWorkerManager();
  // Register the manager for this process so that any code path which dispatches
  // a run can find it without being handed it explicitly. Without this, a
  // construction site that omits the manager builds a run engine that can never
  // acquire a sandbox lease, and it only shows up when a customer hits that path.
  setProcessPluginWorkerManager(workerManager);

  // Mount API routes
  const api = Router();
  api.use(boardMutationGuard());
  api.use(
    "/health",
    healthRoutes(db, {
      deploymentMode: opts.deploymentMode,
      deploymentExposure: opts.deploymentExposure,
      authReady: opts.authReady,
      companyDeletionEnabled: opts.companyDeletionEnabled,
    }),
  );
  api.use(openApiRoutes());
  api.use("/companies", companyRoutes(db, opts.storageService));
  api.use(llmRoutes(db));
  api.use(folderRoutes(db));
  api.use(companySkillRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(companySkillPolicyRoutes(db));
  api.use(inboxAgentPolicyRoutes(db));
  api.use(builtInAgentRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(summarySlotRoutes(db));
  api.use(teamsCatalogRoutes(db));
  api.use(agentRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(assetRoutes(db, opts.storageService));
  api.use(projectRoutes(db));
  api.use(caseRoutes(db, opts.storageService));
  api.use(issueTreeControlRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(fileResourceRoutes(db));
  api.use(routineRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(pipelineRoutes(db));
  api.use(environmentRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(executionWorkspaceRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(goalRoutes(db));
  api.use(boardChatRoutes(db, { deploymentMode: opts.deploymentMode }));
  api.use(approvalRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(secretRoutes(db));
  const trustedLocalStdioRuntimeHost =
    process.env.PAPERCLIP_TRUSTED_MCP_RUNTIME_HOST
    ?? process.env.PAPERCLIP_TOOL_RUNTIME_TRUSTED_HOST
    ?? null;
  api.use(costRoutes(db, { pluginWorkerManager: workerManager }));
  api.use(activityRoutes(db));
  api.use(dashboardRoutes(db));
  api.use(attentionRoutes(db));
  api.use(decisionTrainingRoutes(db));
  api.use(userProfileRoutes(db));
  api.use(sidebarBadgeRoutes(db));
  api.use(sidebarPreferenceRoutes(db));
  api.use(resourceMembershipRoutes(db));
  api.use(inboxDismissalRoutes(db));
  api.use(instanceSettingsRoutes(db));
  if (opts.databaseBackupService) {
    api.use(instanceDatabaseBackupRoutes(opts.databaseBackupService));
  }
  const pluginRegistry = pluginRegistryService(db);
  const eventBus = createPluginEventBus();
  setPluginEventBus(eventBus);
  const jobStore = pluginJobStore(db);
  const lifecycle = pluginLifecycleManager(db, { workerManager });
  const scheduler = createPluginJobScheduler({
    db,
    jobStore,
    workerManager,
  });
  const toolDispatcher = createPluginToolDispatcher({
    workerManager,
    lifecycleManager: lifecycle,
    db,
  });
  const toolGateway = createToolGatewayService(db, {
    pluginToolDispatcher: toolDispatcher,
    deploymentMode: opts.deploymentMode,
    deploymentExposure: opts.deploymentExposure,
    trustedLocalStdioRuntimeHost,
  });
  // Issue routes are intentionally mounted after the gateway is constructed because
  // issue approval endpoints delegate to it. The intervening routers use distinct
  // route prefixes, so this dependency does not change issue-route precedence.
  api.use(issueRoutes(db, opts.storageService, {
    feedbackExportService: opts.feedbackExportService,
    pluginWorkerManager: workerManager,
    approveToolActionRequest: (input) => toolGateway.approveActionRequest(input),
  }));
  app.use(mcpGatewayProtocolRoutes(toolGateway));
  api.use(toolAccessRoutes(db, {
    deploymentMode: opts.deploymentMode,
    deploymentExposure: opts.deploymentExposure,
    trustedLocalStdioRuntimeHost,
    toolGateway,
  }));
  api.use(smokeLabRoutes(db, {
    deploymentMode: opts.deploymentMode,
    deploymentExposure: opts.deploymentExposure,
  }));
  const jobCoordinator = createPluginJobCoordinator({
    db,
    lifecycle,
    scheduler,
    jobStore,
  });
  const hostServiceCleanup = createPluginHostServiceCleanup(lifecycle, hostServicesDisposers);
  let viteHtmlRenderer: ReturnType<typeof createCachedViteHtmlRenderer> | null = null;
  const loader = pluginLoader(
    db,
    {
      localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
      migrationDb: opts.pluginMigrationDb,
    },
    {
      workerManager,
      eventBus,
      jobScheduler: scheduler,
      jobStore,
      toolDispatcher,
      lifecycleManager: lifecycle,
      instanceInfo: {
        instanceId: opts.instanceId ?? "default",
        hostVersion: opts.hostVersion ?? "0.0.0",
        deploymentMode: opts.deploymentMode,
        deploymentExposure: opts.deploymentExposure,
      },
      buildHostHandlers: (pluginId, manifest) => {
        const notifyWorker = (method: string, params: unknown) => {
          const handle = workerManager.getWorker(pluginId);
          if (handle) handle.notify(method, params);
        };
        const services = buildHostServices(db, pluginId, manifest.id, eventBus, notifyWorker, {
          pluginWorkerManager: workerManager,
          manifest,
        });
        hostServicesDisposers.set(pluginId, () => services.dispose());
        return createHostClientHandlers({
          pluginId,
          capabilities: manifest.capabilities,
          services,
        });
      },
    },
  );
  api.use(
    toolGatewayRoutes(db, toolGateway),
  );
  api.use(
    pluginRoutes(
      db,
      loader,
      { scheduler, jobStore },
      { workerManager },
      { toolDispatcher },
      // bridgeDeps: expose the worker manager's stream bus so the SSE bridge
      // route (and any worker->host stream consumer) can subscribe to channels.
      { workerManager, streamBus: workerManager.streamBus },
      { toolGateway },
    ),
  );
  api.use(adapterRoutes());
  api.use(
    accessRoutes(db, {
      deploymentMode: opts.deploymentMode,
      deploymentExposure: opts.deploymentExposure,
      bindHost: opts.bindHost,
      allowedHostnames: opts.allowedHostnames,
    }),
  );
  app.use("/api", api);
  app.use("/api", (_req, res) => {
    res.status(404).json({ error: "API route not found" });
  });
  app.use(pluginUiStaticRoutes(db, {
    localPluginDir: opts.localPluginDir ?? DEFAULT_LOCAL_PLUGIN_DIR,
  }));
  // Deployer-mounted brand assets (must come before the SPA fallback / vite
  // middleware so /branding/brand.css never resolves to the HTML shell).
  registerBrandStaticRoute(app);

  const __dirname = path.dirname(fileURLToPath(import.meta.url));
  if (opts.uiMode === "static") {
    // Try published location first (server/ui-dist/), then monorepo dev location (../../ui/dist)
    const candidates = [
      path.resolve(__dirname, "../ui-dist"),
      path.resolve(__dirname, "../../ui/dist"),
    ];
    const uiDist = candidates.find((p) => fs.existsSync(path.join(p, "index.html")));
    if (uiDist) {
      // Hashed asset files (Vite emits them under /assets/<name>.<hash>.<ext>)
      // never change once built, so they can be cached aggressively.
      app.use(
        "/assets",
        express.static(path.join(uiDist, "assets"), {
          maxAge: "1y",
          immutable: true,
        }),
      );
      // Non-hashed static files (favicon.ico, manifest, robots.txt, etc.):
      // short cache so operators who swap them out see the new version
      // reasonably fast. Override for `index.html` specifically — it is
      // served by this middleware for `/` and `/index.html`, and it must
      // never outlive the asset hashes it points at.
      // The HTML shell MUST go through the branded fallback below, which injects
      // runtime branding + the `paperclip-default-theme` meta the pre-paint theme
      // script reads. Serving the RAW index.html here (Express's default
      // `index: 'index.html'` for `/`, or an explicit `/index.html` file hit)
      // bypasses that injection -> no theme meta -> the script defaults to dark ->
      // a dark->light flash on first paint until a branded route loads. So disable
      // directory-index serving AND route an explicit `/index.html` to the fallback.
      app.get("/index.html", (_req, res) => {
        res
          .status(200)
          .set("Content-Type", "text/html")
          .set("Cache-Control", "no-cache")
          .end(readBrandedStaticIndexHtml(uiDist));
      });
      app.use(
        express.static(uiDist, {
          index: false,
          maxAge: "1h",
          setHeaders(res, filePath) {
            if (path.basename(filePath) === "index.html") {
              res.set("Cache-Control", "no-cache");
            }
          },
        }),
      );
      // SPA fallback. Only for non-asset routes — if the browser asks for
      // /assets/something.js that doesn't exist, we must NOT serve the HTML
      // shell: the browser would try to load it as a JavaScript module, fail
      // with a MIME-type error, and cache that broken response. Return 404
      // instead. The index.html response itself is no-cache so a subsequent
      // deploy's updated asset hashes are picked up on next load.
      app.get(/.*/, (req, res) => {
        if (req.path.startsWith("/assets/")) {
          res.status(404).end();
          return;
        }
        res
          .status(200)
          .set("Content-Type", "text/html")
          .set("Cache-Control", "no-cache")
          .end(readBrandedStaticIndexHtml(uiDist));
      });
    } else {
      console.warn("[paperclip] UI dist not found; running in API-only mode");
    }
  }

  if (opts.uiMode === "vite-dev") {
    const uiRoot = path.resolve(__dirname, "../../ui");
    const publicUiRoot = path.resolve(uiRoot, "public");
    const hmrPort = resolveViteHmrPort(opts.serverPort);
    const hmrHost = resolveViteHmrHost(opts.bindHost);
    const { createServer: createViteServer } = await import("vite");
    const vite = await createViteServer({
      root: uiRoot,
      appType: "custom",
      server: {
        middlewareMode: true,
        hmr: {
          ...(hmrHost ? { host: hmrHost } : {}),
          port: hmrPort,
          clientPort: hmrPort,
        },
        allowedHosts: privateHostnameGateEnabled ? Array.from(privateHostnameAllowSet) : undefined,
      },
    });
    viteHtmlRenderer = createCachedViteHtmlRenderer({
      vite,
      uiRoot,
      brandHtml: applyUiBranding,
    });
    const renderViteHtml = viteHtmlRenderer;

    if (fs.existsSync(publicUiRoot)) {
      app.use(express.static(publicUiRoot, { index: false }));
    }
    app.get(/.*/, async (req, res, next) => {
      if (!shouldServeViteDevHtml(req)) {
        next();
        return;
      }
      try {
        const html = await renderViteHtml.render(req.originalUrl);
        res.status(200).set({ "Content-Type": "text/html" }).end(html);
      } catch (err) {
        next(err);
      }
    });
    app.use(vite.middlewares);
  }

  app.use(errorHandler);

  jobCoordinator.start();
  scheduler.start();
  let feedbackExportShuttingDown = false;
  let feedbackExportTimer: ReturnType<typeof setInterval> | null = null;
  const disableFeedbackExportFlushes = () => {
    feedbackExportShuttingDown = true;
    if (feedbackExportTimer) {
      clearInterval(feedbackExportTimer);
      feedbackExportTimer = null;
    }
  };
  const flushPendingFeedbackExports = async () => {
    if (feedbackExportShuttingDown) return;
    try {
      await opts.feedbackExportService?.flushPendingFeedbackTraces();
    } catch (err) {
      if (isDatabaseConnectionUnavailableError(err)) {
        disableFeedbackExportFlushes();
        logger.warn({ err }, "Disabling pending feedback export flushes because the database is unavailable");
        return;
      }
      logger.error({ err }, "Failed to flush pending feedback exports");
    }
  };

  feedbackExportTimer = opts.feedbackExportService
    ? setInterval(() => {
      void flushPendingFeedbackExports();
    }, FEEDBACK_EXPORT_FLUSH_INTERVAL_MS)
    : null;
  feedbackExportTimer?.unref?.();
  if (opts.feedbackExportService) {
    void flushPendingFeedbackExports();
  }
  void toolDispatcher.initialize().catch((err) => {
    logger.error({ err }, "Failed to initialize plugin tool dispatcher");
  });
  const devWatcher = createPluginDevWatcher(
    lifecycle,
    async (pluginId) => (await pluginRegistry.getById(pluginId))?.packagePath ?? null,
  );
  // Auto-install the bundled kubernetes sandbox-provider plugin so the
  // "kubernetes" sandbox provider is registered for agent runs. The plugin is
  // excluded from the pnpm workspace and built standalone into the image (see
  // Dockerfile), then installed here from its local path. This runs BEFORE
  // loadAll() so loadAll() can activate it in the same startup pass.
  //
  // SAFETY (invariant B): this is fully fail-safe. Any failure (missing path,
  // install error, load error) is caught, logged, and swallowed so the server
  // ALWAYS finishes booting. A degraded boot (no kubernetes provider, agents
  // cannot run) is strictly preferable to a crash loop.
  const ensureBundledKubernetesPlugin = async (): Promise<void> => {
    const KUBERNETES_PLUGIN_KEY = "paperclip.kubernetes-sandbox-provider";
    const pluginPath =
      process.env["PAPERCLIP_KUBERNETES_PLUGIN_PATH"] ??
      "/app/packages/plugins/sandbox-providers/kubernetes";
    const bundleManifestPath = path.join(pluginPath, "dist", "manifest.js");
    try {
      const existing = await pluginRegistry.getByKey(KUBERNETES_PLUGIN_KEY);
      const action = decideBundledPluginAction({
        existingStatus: existing?.status ?? null,
        bundlePresent: fs.existsSync(bundleManifestPath),
      });

      if (action === "skip-ready") {
        // Healthy. loadAll() (which runs right after this) lists ready plugins
        // and (re)starts their workers, so nothing to do here.
        logger.info(
          { pluginKey: KUBERNETES_PLUGIN_KEY, status: existing?.status },
          "kubernetes sandbox plugin already installed and ready; skipping auto-install",
        );
        return;
      }
      if (action === "skip-uninstalled") {
        // An admin explicitly removed it; respect that and do not silently
        // reinstall the bundle on every boot.
        logger.info(
          { pluginKey: KUBERNETES_PLUGIN_KEY, status: existing?.status },
          "kubernetes sandbox plugin is uninstalled; respecting that and skipping auto-install",
        );
        return;
      }
      if (action === "self-heal-blocked-bundle-missing") {
        logger.warn(
          { pluginKey: KUBERNETES_PLUGIN_KEY, status: existing?.status, pluginPath },
          "kubernetes sandbox plugin is stuck in a non-ready status but its bundle is missing; cannot self-heal",
        );
        return;
      }
      if (action === "self-heal" && existing) {
        // SELF-HEAL: the bundled plugin exists in the DB but is NOT ready, so
        // loadAll() (which only activates 'ready' plugins) would leave the
        // kubernetes sandbox provider dead and agents could never acquire a lease.
        // This commonly happens when an earlier boot marked it 'error' (e.g. the
        // manifest was missing before the image shipped it). Now that the bundle
        // is on disk, drive it back to 'ready' so loadAll() re-activates it
        // (activation re-reads the manifest from disk, so a stale record heals).
        try {
          // lifecycle.load() transitions <status> -> ready (error/installed/
          // disabled/upgrade_pending are all valid sources). The actual worker
          // start is performed by loader.loadAll() immediately after.
          await lifecycle.load(existing.id);
          logger.info(
            { pluginId: existing.id, pluginKey: existing.pluginKey, previousStatus: existing.status },
            "re-activated stuck kubernetes sandbox plugin (-> ready); loadAll() will start its worker",
          );
        } catch (healErr) {
          logger.error(
            { err: healErr, pluginKey: KUBERNETES_PLUGIN_KEY, status: existing.status },
            "Failed to self-heal stuck kubernetes sandbox plugin; continuing boot (degraded: kubernetes provider unavailable)",
          );
        }
        return;
      }
      if (action === "skip-bundle-missing") {
        // Skip silently when the bundle is absent (e.g. local dev or an image
        // built without the plugin). Not an error condition.
        logger.info(
          { pluginPath },
          "kubernetes sandbox plugin bundle not present; skipping auto-install",
        );
        return;
      }
      // action === "install"
      logger.info({ pluginPath }, "auto-installing bundled kubernetes sandbox plugin");
      const discovered = await loader.installPlugin({ localPath: pluginPath });
      if (!discovered.manifest) {
        logger.error("kubernetes sandbox plugin installed but manifest is missing");
        return;
      }
      // Transition installed -> ready and activate the worker.
      const installed = await pluginRegistry.getByKey(discovered.manifest.id);
      if (installed) {
        await lifecycle.load(installed.id);
        logger.info(
          { pluginId: installed.id, pluginKey: installed.pluginKey },
          "kubernetes sandbox plugin auto-installed and loaded",
        );
      } else {
        logger.error("kubernetes sandbox plugin installed but not found in registry");
      }
    } catch (err) {
      logger.error(
        { err },
        "Failed to auto-install the kubernetes sandbox plugin; continuing boot (degraded: kubernetes provider unavailable)",
      );
    }
  };
  void ensureBundledKubernetesPlugin()
    .then(() => loader.loadAll())
    .then((result) => {
    if (!result) return;
    for (const loaded of result.results) {
      if (devWatcher && loaded.success && loaded.plugin.packagePath) {
        devWatcher.watch(loaded.plugin.id, loaded.plugin.packagePath);
      }
    }
  }).catch((err) => {
    logger.error({ err }, "Failed to load ready plugins on startup");
  });
  let appServicesShutdown = false;
  const shutdownAppServices = () => {
    if (appServicesShutdown) return;
    appServicesShutdown = true;
    disableFeedbackExportFlushes();
    devWatcher?.close();
    viteHtmlRenderer?.dispose();
    hostServiceCleanup.disposeAll();
    hostServiceCleanup.teardown();
  };
  app.locals.paperclipShutdown = shutdownAppServices;

  process.once("exit", shutdownAppServices);
  process.once("beforeExit", () => {
    void flushPluginLogBuffer();
  });

  return app;
}