fix: dynamically resolve Turbopack external module mappings

.changeset/five-gifts-hope.md

@@ -0,0 +1,13 @@
+---
+"@opennextjs/cloudflare": patch
+---
+
+Fix Turbopack external module resolution on workerd by dynamically discovering external imports at build time.
+
+When packages are listed in `serverExternalPackages`, Turbopack externalizes them via `externalImport()` which uses dynamic `await import(id)`. On workerd, the bundler can't statically analyze `import(id)` with a variable, so these modules aren't included in the worker bundle.
+
+This patch:
+
+- Discovers hashed Turbopack external module mappings from `.next/node_modules/` symlinks (e.g. `shiki-43d062b67f27bbdc` → `shiki`)
+- Scans traced chunk files for bare external imports (e.g. `externalImport("shiki")`) and subpath imports (e.g. `shiki/engine/javascript`)
+- Generates explicit `switch/case` entries so the bundler can statically resolve and include these modules

examples/e2e/app-router/app/api/shiki/route.ts

@@ -0,0 +1,19 @@
+import { createHighlighter } from "shiki";
+import { createJavaScriptRegexEngine } from "shiki/engine/javascript";
+
+export async function GET() {
+	const highlighter = await createHighlighter({
+		themes: ["vitesse-dark"],
+		langs: ["javascript"],
+		engine: createJavaScriptRegexEngine(),
+	});
+
+	const html = highlighter.codeToHtml('console.log("hello")', {
+		lang: "javascript",
+		theme: "vitesse-dark",
+	});
+
+	return new Response(JSON.stringify({ html }), {
+		headers: { "content-type": "application/json" },
+	});
+}

examples/e2e/app-router/e2e/shiki.test.ts

@@ -0,0 +1,17 @@
+import { expect, test } from "@playwright/test";
+
+// Regression test for Turbopack external module resolution on workerd.
+// When shiki is in serverExternalPackages, Turbopack externalizes it via `externalImport()`,
+// which does `await import("shiki")` with a dynamic variable. On workerd, the bundler can't
+// statically analyze `import(id)`, so the module isn't included. The patch adds explicit
+// switch cases (e.g. `case "shiki": await import("shiki")`) so the bundler can trace them.
+// This also covers subpath imports like "shiki/engine/javascript".
+test("shiki syntax highlighting via API route", async ({ request }) => {
+	const response = await request.get("/api/shiki");
+	expect(response.status()).toEqual(200);
+
+	const json = await response.json();
+	expect(json).toMatchObject({
+		html: '<pre class="shiki vitesse-dark" style="background-color:#121212;color:#dbd7caee" tabindex="0"><code><span class="line"><span style="color:#BD976A">console</span><span style="color:#666666">.</span><span style="color:#80A665">log</span><span style="color:#666666">(</span><span style="color:#C98A7D77">"</span><span style="color:#C98A7D">hello</span><span style="color:#C98A7D77">"</span><span style="color:#666666">)</span></span></code></pre>',
+	});
+});

examples/e2e/app-router/next.config.ts

@@ -4,6 +4,7 @@ const nextConfig: NextConfig = {
 	poweredByHeader: false,
 	cleanDistDir: true,
 	transpilePackages: ["@example/shared"],
+	serverExternalPackages: ["shiki"],
 	output: "standalone",
 	// outputFileTracingRoot: "../sst",
 	typescript: {

examples/e2e/app-router/package.json

@@ -21,7 +21,8 @@
 		"@example/shared": "workspace:*",
 		"next": "catalog:e2e",
 		"react": "catalog:e2e",
-		"react-dom": "catalog:e2e"
+		"react-dom": "catalog:e2e",
+		"shiki": "^3.22.0"
 	},
 	"devDependencies": {
 		"@playwright/test": "catalog:",

packages/cloudflare/src/cli/build/patches/plugins/turbopack.ts

@@ -1,3 +1,6 @@
+import fs from "node:fs";
+import path from "node:path";
+
 import { patchCode } from "@opennextjs/aws/build/patch/astCodePatcher.js";
 import type { CodePatcher } from "@opennextjs/aws/build/patch/codePatcher.js";
 import { getCrossPlatformPathRegex } from "@opennextjs/aws/utils/regex.js";
@@ -10,6 +13,189 @@ fix:
   requireChunk(chunkPath)
 `;
 
+/**
+ * Discover Turbopack external module mappings by reading symlinks in .next/node_modules/.
+ *
+ * Turbopack externalizes packages listed in serverExternalPackages and creates hashed
+ * identifiers (e.g. "shiki-43d062b67f27bbdc") with symlinks in .next/node_modules/ pointing
+ * to the real packages (e.g. ../../node_modules/shiki). At runtime, externalImport() does
+ * `await import("shiki-43d062b67f27bbdc/wasm")` which fails on workerd because those hashed
+ * names are not real modules. This function discovers the mappings so we can intercept them.
+ */
+function discoverExternalModuleMappings(filePath: string): Map<string, string> {
+	// filePath is like: .../.next/server/chunks/ssr/[turbopack]_runtime.js
+	// We need: .../.next/node_modules/
+	const dotNextDir = filePath.replace(/\/server\/chunks\/.*$/, "");
+	const nodeModulesDir = path.join(dotNextDir, "node_modules");
+
+	const mappings = new Map<string, string>();
+
+	if (!fs.existsSync(nodeModulesDir)) {
+		return mappings;
+	}
+
+	for (const entry of fs.readdirSync(nodeModulesDir)) {
+		const entryPath = path.join(nodeModulesDir, entry);
+		try {
+			const stat = fs.lstatSync(entryPath);
+			if (stat.isSymbolicLink()) {
+				const target = fs.readlinkSync(entryPath);
+				// target is like "../../node_modules/shiki" — extract package name
+				const match = target.match(/node_modules\/(.+)$/);
+				if (match?.[1]) {
+					mappings.set(entry, match[1]);
+				}
+			}
+		} catch {
+			// skip entries we can't read
+		}
+	}
+
+	return mappings;
+}
+
+/**
+ * Build a dynamic inlineExternalImportRule that includes cases for all discovered
+ * Turbopack external module hashes, mapping them back to their real package names.
+ *
+ * We use a switch for exact matches (including bare + subpath cases) and a fallback
+ * for the default case. Since switch/case can only match exact strings, we enumerate
+ * known subpaths from the traced files to cover cases like "shiki-hash/wasm".
+ */
+function buildExternalImportRule(
+	mappings: Map<string, string>,
+	tracedFiles: string[],
+	runtimeCode: string
+): string {
+	const cases: string[] = [];
+
+	// Always include the @vercel/og rewrite
+	cases.push(`    case "next/dist/compiled/@vercel/og/index.node.js":
+      $RAW = await import("next/dist/compiled/@vercel/og/index.edge.js");
+      break;`);
+
+	// Add case for each discovered external module mapping (bare import)
+	for (const [hashedName, realName] of mappings) {
+		cases.push(`    case "${hashedName}":
+      $RAW = await import("${realName}");
+      break;`);
+	}
+
+	// Discover subpath imports from the traced chunk files.
+	// Chunks reference external modules like "shiki-hash/wasm" — scan for these patterns.
+	const subpathCases = discoverExternalSubpaths(mappings, tracedFiles);
+	for (const [hashedSubpath, realSubpath] of subpathCases) {
+		cases.push(`    case "${hashedSubpath}":
+      $RAW = await import("${realSubpath}");
+      break;`);
+	}
+
+	// Discover bare external imports from chunk files (e.g. externalImport("shiki")).
+	// These need explicit switch cases so the bundler can statically resolve them.
+	const bareImports = discoverBareExternalImports(tracedFiles, runtimeCode);
+	const alreadyCased = new Set(cases.map((c) => c.match(/case "([^"]+)"/)?.[1]).filter(Boolean));
+	for (const [moduleName, realName] of bareImports) {
+		if (!alreadyCased.has(moduleName)) {
+			cases.push(`    case "${moduleName}":
+      $RAW = await import("${realName}");
+      break;`);
+		}
+	}
+
+	return `
+rule:
+  pattern: "$RAW = await import($ID)"
+  inside:
+    regex: "externalImport"
+    kind: function_declaration
+    stopBy: end
+fix: |-
+  switch ($ID) {
+${cases.join("\n")}
+    default:
+      $RAW = await import($ID);
+  }
+`;
+}
+
+/**
+ * Scan traced chunk files for bare external module imports (e.g. `externalImport("shiki")`).
+ *
+ * In some Turbopack versions, externalized packages are referenced by their real names
+ * (not hashed). On workerd, the default `await import(id)` with a variable `id` can't be
+ * statically analyzed by the bundler. By adding explicit switch cases with string literals,
+ * we make these imports statically discoverable so they get bundled into the worker.
+ */
+function discoverBareExternalImports(tracedFiles: string[], runtimeCode: string): Map<string, string> {
+	const bareImports = new Map<string, string>();
+
+	// Turbopack assigns `externalImport` to a single-letter property on the context prototype,
+	// e.g. `contextPrototype.y = externalImport`. The property name could change between versions,
+	// so we extract it dynamically from the runtime code rather than hardcoding `.y`.
+	const propMatch = runtimeCode.match(/contextPrototype\.(\w+)\s*=\s*externalImport/);
+	if (!propMatch?.[1]) {
+		return bareImports;
+	}
+	const prop = propMatch[1];
+
+	// Chunks call externalImport as e.g. `.y("shiki")` — build a regex using the discovered property name.
+	const callPattern = new RegExp(`\\.${prop}\\("([^"]+)"\\)`, "g");
+
+	const chunkFiles = tracedFiles.filter((f) => f.includes(".next/server/chunks/"));
+
+	for (const filePath of chunkFiles) {
+		try {
+			const content = fs.readFileSync(filePath, "utf-8");
+			for (const match of content.matchAll(callPattern)) {
+				const moduleName = match[1];
+				if (moduleName) {
+					// Identity mapping — the module name is already the real name
+					bareImports.set(moduleName, moduleName);
+				}
+			}
+		} catch {
+			// skip files we can't read
+		}
+	}
+
+	return bareImports;
+}
+
+/**
+ * Scan traced chunk files for external module subpath imports.
+ * E.g. find "shiki-43d062b67f27bbdc/wasm" in chunk code and map it to "shiki/wasm".
+ *
+ * Only scans files with "[externals]" in the name since those are the chunks that
+ * contain externalImport calls.
+ */
+function discoverExternalSubpaths(mappings: Map<string, string>, tracedFiles: string[]): Map<string, string> {
+	const subpaths = new Map<string, string>();
+
+	const externalChunks = tracedFiles.filter((f) => f.includes("[externals]"));
+
+	for (const [hashedName, realName] of mappings) {
+		const pattern = new RegExp(`"(${hashedName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}/[^"]*)"`, "g");
+
+		for (const filePath of externalChunks) {
+			try {
+				const content = fs.readFileSync(filePath, "utf-8");
+				for (const match of content.matchAll(pattern)) {
+					const fullHashedPath = match[1];
+					if (fullHashedPath) {
+						const subpath = fullHashedPath.slice(hashedName.length);
+						const realSubpath = realName + subpath;
+						subpaths.set(fullHashedPath, realSubpath);
+					}
+				}
+			} catch {
+				// skip files we can't read
+			}
+		}
+	}
+
+	return subpaths;
+}
+
 export const patchTurbopackRuntime: CodePatcher = {
 	name: "inline-turbopack-chunks",
 	patches: [
@@ -19,8 +205,10 @@ export const patchTurbopackRuntime: CodePatcher = {
 				escape: false,
 			}),
 			contentFilter: /loadRuntimeChunkPath/,
-			patchCode: async ({ code, tracedFiles }) => {
-				let patched = patchCode(code, inlineExternalImportRule);
+			patchCode: async ({ code, tracedFiles, filePath }) => {
+				const mappings = discoverExternalModuleMappings(filePath);
+				const externalImportRule = buildExternalImportRule(mappings, tracedFiles, code);
+				let patched = patchCode(code, externalImportRule);
 				patched = patchCode(patched, inlineChunksRule);
 
 				return `${patched}\n${inlineChunksFn(tracedFiles)}`;
@@ -63,27 +251,3 @@ ${chunks
   }
 `;
 }
-
-// Turbopack imports `og` via `externalImport`.
-// We patch it to:
-// - add the explicit path so that the file is inlined by wrangler
-// - use the edge version of the module instead of the node version.
-//
-// Modules that are not inlined (no added to the switch), would generate an error similar to:
-// Failed to load external module path/to/module: Error: No such module "path/to/module"
-const inlineExternalImportRule = `
-rule:
-  pattern: "$RAW = await import($ID)"
-  inside:
-    regex: "externalImport"
-    kind: function_declaration
-    stopBy: end
-fix: |-
-  switch ($ID) {
-    case "next/dist/compiled/@vercel/og/index.node.js":
-      $RAW = await import("next/dist/compiled/@vercel/og/index.edge.js");
-      break;
-    default:
-      $RAW = await import($ID);
-  }
-`;