diff --git a/src/app.ts b/src/app.ts
index 809e478c887..01b22a44811 100644
--- a/src/app.ts
+++ b/src/app.ts
@@ -1,6 +1,5 @@
 import * as path from "@std/path";
-import { type ComponentType, h } from "preact";
-import { renderToString } from "preact-render-to-string";
+import type { ComponentType } from "preact";
 import { trace } from "@opentelemetry/api";
 
 import { DENO_DEPLOYMENT_ID } from "./runtime/build_id.ts";
@@ -15,8 +14,7 @@ import {
 } from "./config.ts";
 import { type BuildCache, ProdBuildCache } from "./build_cache.ts";
 import type { ServerIslandRegistry } from "./context.ts";
-import { FinishSetup, ForgotBuild } from "./finish_setup.tsx";
-import { HttpError } from "./error.ts";
+import { HttpError, SetupError } from "./error.ts";
 import { mergePaths } from "./utils.ts";
 
 // TODO: Completed type clashes in older Deno versions
@@ -254,11 +252,14 @@ export class App<State> {
       );
     }
 
-    if (
-      !this.#buildCache.hasSnapshot && this.config.mode === "production" &&
-      DENO_DEPLOYMENT_ID !== undefined
-    ) {
-      return missingBuildHandler;
+    if (!this.#buildCache.hasSnapshot && this.config.mode === "production") {
+      let message =
+        "The build cache is not set up. Run `deno task build` before starting the server.";
+      if (DENO_DEPLOYMENT_ID !== undefined) {
+        message +=
+          ' Go to "Settings" in Deno Deploy and set the build command to `deno task build`.';
+      }
+      throw new SetupError(message);
     }
 
     return async (
@@ -338,14 +339,3 @@ export class App<State> {
     await listenOnFreePort(options, handler);
   }
 }
-
-// deno-lint-ignore require-await
-const missingBuildHandler = async (): Promise<Response> => {
-  const headers = new Headers();
-  headers.set("Content-Type", "text/html; charset=utf-8");
-
-  const html = DENO_DEPLOYMENT_ID
-    ? renderToString(h(FinishSetup, null))
-    : renderToString(h(ForgotBuild, null));
-  return new Response(html, { headers, status: 500 });
-};
diff --git a/src/error.ts b/src/error.ts
index 610b09ba873..e093c9b930d 100644
--- a/src/error.ts
+++ b/src/error.ts
@@ -74,3 +74,26 @@ export class HttpError extends Error {
     this.status = status;
   }
 }
+
+/**
+ * Error that's throw when the app setup fails. Usually caused by
+ * `deno task build` not being run before starting the server.
+ *
+ * @example Basic usage
+ * ```ts ignore
+ * import { App, SetupError } from "fresh";
+ * import { expect } from "@std/expect";
+ *
+ * const app = new App()
+ *   .get("/", () => new Response("ok"))
+ *
+ * try {
+ *   // Throws if `deno task build` was not run beforehand
+ *   const handler = await app.handler();
+ * } catch (error) {
+ *   expect(error).toBeInstanceOf(SetupError);
+ *   expect(error.message).toBe("App not setup");
+ * }
+ * ```
+ */
+export class SetupError extends Error {}
diff --git a/src/finish_setup.tsx b/src/finish_setup.tsx
deleted file mode 100644
index b3673d22059..00000000000
--- a/src/finish_setup.tsx
+++ /dev/null
@@ -1,54 +0,0 @@
-import type { ComponentChildren } from "preact";
-
-export function FinishSetup() {
-  return (
-    <Doc>
-      <div class="frsh-error-page">
-        <div style="max-width: 48rem; padding: 2rem 1rem; margin: 0 auto; font-family: sans-serif">
-          <h1>Finish setting up Fresh</h1>
-          <ol style="line-height: 1.6; margin-bottom: 1.5rem">
-            <li>
-              Go to your project in Deno Deploy and click the{" "}
-              <code>Settings</code> tab.
-            </li>
-            <li>
-              In the Git Integration section, enter <code>deno task build</code>
-              {" "}
-              in the <code>Build Command</code> input.
-            </li>
-            <li>
-              Save the changes.
-            </li>
-          </ol>
-        </div>
-      </div>
-    </Doc>
-  );
-}
-
-export function ForgotBuild() {
-  return (
-    <Doc>
-      <div class="frsh-error-page">
-        <div style="max-width: 48rem; padding: 2rem 1rem; margin: 0 auto; font-family: sans-serif">
-          <h1>Missing build directory</h1>
-          <p>
-            Did you forget to run <code>deno task build</code>?
-          </p>
-        </div>
-      </div>
-    </Doc>
-  );
-}
-
-function Doc(props: { children?: ComponentChildren }) {
-  return (
-    <html>
-      <head>
-        <meta charset="utf-8" />
-        <title>Finish setting up Fresh</title>
-      </head>
-      <body>{props.children}</body>
-    </html>
-  );
-}
diff --git a/tests/fixture_precompile/valid/main.tsx b/tests/fixture_precompile/valid/main.tsx
index 7beef0698fd..3e95d63c73f 100644
--- a/tests/fixture_precompile/valid/main.tsx
+++ b/tests/fixture_precompile/valid/main.tsx
@@ -1,4 +1,7 @@
 import { App } from "../../../src/app.ts";
+import { Builder } from "../../../src/dev/mod.ts";
+
+const builder = new Builder();
 
 const app = new App({ staticDir: "./static" }).get(
   "/",
@@ -25,6 +28,8 @@ const app = new App({ staticDir: "./static" }).get(
     ),
 );
 
+await builder.build(app);
+
 const handler = app.handler();
 const res = await handler(new Request("http://localhost/"));
 // deno-lint-ignore no-console