work

Author: dmail

.agents/skills/dev-server/SKILL.md

@@ -79,3 +79,15 @@ return {
 The browser URL stays as requested (the redirect is internal). `?search` is carried through, so a page can read `location.search`. Inline `<script type="module">` in such a page is externalized/supervised exactly like any app page — that is expected, the code still runs.
 
 If a `transformUrlContent.html` hook injects into every page (like the devices client), exclude your own internal pages by matching their file URL (`asUrlWithoutSearch(urlInfo.url) === pageFileUrl`), since after cooking their url is the template file url, not the `/.internal/...` request path.
+
+## Out directory (`.jsenv/`) writes stay synchronous
+
+Every cooked file is written into `outDirectoryUrl` (a debug aid, on by
+default). Making these writes asynchronous is tempting and has been tried more
+than once; it loses, measured on a 500-file cold load: synchronous writes block
+the event loop ~160ms in total, asynchronous ones queue in the threadpool behind
+tens of MB (content + sourcemaps) and a response waiting for its own write waits
+~36ms on average — 18s summed. Fire-and-forget is not an option: the last write
+lands after the test that cooked it has ended and the side-effect snapshots lose
+it. Details at `writeInsideOutDirectory` in
+[url_info_transformations.js](../../../src/kitchen/url_graph/url_info_transformations.js).

dist/build/build.js

@@ -4,7 +4,7 @@ import { jsenvPluginMinification } from "@jsenv/plugin-minification";
 import { jsenvPluginTranspilation, jsenvPluginJsModuleFallback } from "@jsenv/plugin-transpilation";
 import { memoryUsage } from "node:process";
 import { readFileSync, existsSync, realpathSync, readdirSync, lstatSync, statSync } from "node:fs";
-import { lookupPackageDirectory, urlIsOrIsInsideOf, registerDirectoryLifecycle, urlToRelativeUrl, createDetailedMessage, stringifyUrlSite, generateContentFrame, validateResponseIntegrity, ensureWindowsDriveLetter, setUrlFilename, moveUrl, getCallerPosition, urlToBasename, urlToExtension, asSpecifierWithoutSearch, asUrlWithoutSearch, injectQueryParamsIntoSpecifier, bufferToEtag, isFileSystemPath, urlToPathname, setUrlBasename, urlToFileSystemPath, writeFileSync, writeFile, createLogger, URL_META, applyNodeEsmResolution, normalizeUrl, ANSI, RUNTIME_COMPAT, CONTENT_TYPE, readPackageAtOrNull, urlToFilename, DATA_URL, errorToHTML, normalizeImportMap, composeTwoImportMaps, resolveImport, JS_QUOTES, readCustomConditionsFromProcessArgs, collectFiles, readEntryStatSync, applyFileSystemMagicResolution, getExtensionsToTry, ensurePathnameTrailingSlash, compareFileUrls, setUrlExtension, isSpecifierForNodeBuiltin, injectQueryParams, renderDetails, humanizeDuration, humanizeFileSize, renderTable, renderBigSection, distributePercentages, humanizeMemory, comparePathnames, UNICODE, escapeRegexpSpecialChars, injectQueryParamIntoSpecifierWithoutEncoding, renderUrlOrRelativeUrlFilename, assertAndNormalizeDirectoryUrl, Abort, raceProcessTeardownEvents, startMonitoringCpuUsage, startMonitoringMemoryUsage, inferRuntimeCompatFromClosestPackage, browserDefaultRuntimeCompat, nodeDefaultRuntimeCompat, clearDirectorySync, createTaskLog, createLookupPackageDirectory, ensureEmptyDirectory, updateJsonFileSync, createDynamicLog } from "./jsenv_core_packages.js";
+import { lookupPackageDirectory, urlIsOrIsInsideOf, registerDirectoryLifecycle, urlToRelativeUrl, createDetailedMessage, stringifyUrlSite, generateContentFrame, validateResponseIntegrity, ensureWindowsDriveLetter, setUrlFilename, moveUrl, getCallerPosition, urlToBasename, urlToExtension, asSpecifierWithoutSearch, asUrlWithoutSearch, injectQueryParamsIntoSpecifier, bufferToEtag, isFileSystemPath, urlToPathname, setUrlBasename, urlToFileSystemPath, writeFileSync, createLogger, URL_META, applyNodeEsmResolution, normalizeUrl, ANSI, RUNTIME_COMPAT, CONTENT_TYPE, readPackageAtOrNull, urlToFilename, DATA_URL, errorToHTML, normalizeImportMap, composeTwoImportMaps, resolveImport, JS_QUOTES, readCustomConditionsFromProcessArgs, collectFiles, readEntryStatSync, applyFileSystemMagicResolution, getExtensionsToTry, ensurePathnameTrailingSlash, compareFileUrls, setUrlExtension, isSpecifierForNodeBuiltin, injectQueryParams, renderDetails, humanizeDuration, humanizeFileSize, renderTable, renderBigSection, distributePercentages, humanizeMemory, comparePathnames, UNICODE, escapeRegexpSpecialChars, injectQueryParamIntoSpecifierWithoutEncoding, renderUrlOrRelativeUrlFilename, assertAndNormalizeDirectoryUrl, Abort, raceProcessTeardownEvents, startMonitoringCpuUsage, startMonitoringMemoryUsage, inferRuntimeCompatFromClosestPackage, browserDefaultRuntimeCompat, nodeDefaultRuntimeCompat, clearDirectorySync, createTaskLog, createLookupPackageDirectory, ensureEmptyDirectory, updateJsonFileSync, createDynamicLog } from "./jsenv_core_packages.js";
 import { pathToFileURL } from "node:url";
 import { generateSourcemapFileUrl, createMagicSource, composeTwoSourcemaps, generateSourcemapDataUrl, SOURCEMAP } from "@jsenv/sourcemap";
 import { createPluginsController } from "@jsenv/server/src/plugins_controller.js";
@@ -2981,69 +2981,33 @@ const createUrlInfoTransformer = ({
 
   const applyContentEffects = (urlInfo) => {
     applySourcemapOnContent(urlInfo);
-    return writeInsideOutDirectory(urlInfo);
-  };
-
-  // The out directory is a debug aid. During dev a request must not be held
-  // by a write blocking the event loop (every other request waits too), so
-  // the file is written asynchronously; the response still waits for it, so
-  // what is on disk is what was served. Per file, writes stay ordered: a file
-  // cooked twice in a row ends up holding its latest content.
-  // During build nothing waits behind a write: it is done synchronously.
-  const pendingWritePromiseMap = new Map();
-  const writeOutFile = (urlInfo, fileUrl, content) => {
-    if (!urlInfo.context.dev) {
-      writeFileSync(fileUrl, content, { force: true });
-      return undefined;
-    }
-    const previousWritePromise =
-      pendingWritePromiseMap.get(fileUrl) || Promise.resolve();
-    const writePromise = previousWritePromise.then(async () => {
-      try {
-        await writeFile(fileUrl, content);
-      } catch {
-        try {
-          // a directory where the file goes, or a file where a directory goes
-          writeFileSync(fileUrl, content, { force: true });
-        } catch (e) {
-          logger.debug(`error while writing ${fileUrl}: ${e.message}`);
-        }
-      }
-    });
-    pendingWritePromiseMap.set(fileUrl, writePromise);
-    writePromise.then(() => {
-      if (pendingWritePromiseMap.get(fileUrl) === writePromise) {
-        pendingWritePromiseMap.delete(fileUrl);
-      }
-    });
-    return writePromise;
+    writeInsideOutDirectory(urlInfo);
   };
 
   const writeInsideOutDirectory = (urlInfo) => {
     // writing result inside ".jsenv" directory (debug purposes)
     if (!outDirectoryUrl) {
-      return undefined;
+      return;
     }
     const { generatedUrl } = urlInfo;
     if (!generatedUrl) {
-      return undefined;
+      return;
     }
     if (!generatedUrl.startsWith("file:")) {
-      return undefined;
+      return;
     }
     if (urlToPathname(generatedUrl).endsWith("/")) {
       // when users explicitely request a directory
       // we can't write the content returned by the server in ".jsenv" at that url
       // because it would try to write a directory
       // ideally we would decide a filename for this
       // for now we just don't write anything
-      return undefined;
+      return;
     }
     if (urlInfo.type === "directory") {
       // no need to write the directory
-      return undefined;
+      return;
     }
-    const writePromises = [];
     // if (urlInfo.content === undefined) {
     //   // Some error might lead to urlInfo.content to be null
     //   // (error hapenning before urlInfo.content can be set, or 404 for instance)
@@ -3068,19 +3032,15 @@ const createUrlInfoTransformer = ({
       const outFileUrl = setUrlBasename(generatedUrlObject, baseName);
       let outFilePath = urlToFileSystemPath(outFileUrl);
       outFilePath = truncate(outFilePath, 2055); // for windows
-      writePromises.push(writeOutFile(urlInfo, outFilePath, urlInfo.content));
+      writeFileSync(outFilePath, urlInfo.content, { force: true });
     }
     const { sourcemapGeneratedUrl, sourcemapReference } = urlInfo;
     if (sourcemapGeneratedUrl && sourcemapReference) {
-      writePromises.push(
-        writeOutFile(
-          urlInfo,
-          sourcemapGeneratedUrl,
-          sourcemapReference.urlInfo.content,
-        ),
+      writeFileSync(
+        new URL(sourcemapGeneratedUrl),
+        sourcemapReference.urlInfo.content,
       );
     }
-    return Promise.all(writePromises);
   };
 
   const applySourcemapOnContent = (
@@ -3187,9 +3147,8 @@ const createUrlInfoTransformer = ({
       );
       applyTransformations(urlInfo, injectionTransformations);
     }
-    const contentEffectsPromise = applyContentEffects(urlInfo);
+    applyContentEffects(urlInfo);
     urlInfo.contentFinalized = true;
-    return contentEffectsPromise;
   };
 
   return {
@@ -3842,11 +3801,7 @@ ${ANSI.color(normalizedReturnValue, ANSI.YELLOW)}
           "finalizeUrlContent",
           urlInfo,
         );
-      const outDirectoryWritePromise = urlInfoTransformer.endTransformations(
-        urlInfo,
-        finalizeReturnValue,
-      );
-      return { outDirectoryWritePromise };
+      urlInfoTransformer.endTransformations(urlInfo, finalizeReturnValue);
     } catch (error) {
       throw createFinalizeUrlContentError({
         jsenvPluginsController,
@@ -3880,9 +3835,8 @@ ${ANSI.color(normalizedReturnValue, ANSI.YELLOW)}
           // to cook a file goes (fetch vs transform vs finalize).
           const timePhase = async (name, phase) => {
             const start = performance.now();
-            const result = await phase();
+            await phase();
             urlInfo.timing[name] = performance.now() - start;
-            return result;
           };
 
           // "fetchUrlContent" hook
@@ -3892,18 +3846,7 @@ ${ANSI.color(normalizedReturnValue, ANSI.YELLOW)}
           await timePhase("transform", () => urlInfo.transformContent());
 
           // "finalize" hook
-          const { outDirectoryWritePromise } = await timePhase("finalize", () =>
-            urlInfo.finalizeContent(),
-          );
-
-          // Timed apart: it is disk I/O the response waits for (see
-          // writeInsideOutDirectory), not part of cooking.
-          if (outDirectoryWritePromise) {
-            await timePhase(
-              "write in out directory",
-              () => outDirectoryWritePromise,
-            );
-          }
+          await timePhase("finalize", () => urlInfo.finalizeContent());
         });
       } catch (e) {
         urlInfo.error = e;

dist/build/jsenv_core_packages.js

@@ -1,6 +1,6 @@
 import { createSupportsColor, isUnicodeSupported, stripAnsi, eastAsianWidth, clearTerminal, eraseLines } from "./jsenv_core_node_modules.js";
-import { readFileSync as readFileSync$1, existsSync, readdir, chmod, stat, lstat, chmodSync, statSync, lstatSync, promises, writeFile as writeFile$1, readdirSync, openSync, closeSync, unlinkSync, rmdirSync, mkdirSync, writeFileSync as writeFileSync$1, unlink, rmdir, watch, realpathSync } from "node:fs";
-import { dirname, extname } from "node:path";
+import { extname } from "node:path";
+import { readFileSync as readFileSync$1, existsSync, readdir, chmod, stat, lstat, chmodSync, statSync, lstatSync, promises, readdirSync, openSync, closeSync, unlinkSync, rmdirSync, mkdirSync, writeFileSync as writeFileSync$1, unlink, rmdir, watch, realpathSync } from "node:fs";
 import crypto, { createHash } from "node:crypto";
 import { pathToFileURL, fileURLToPath } from "node:url";
 import { cpus, totalmem, freemem } from "node:os";
@@ -3867,44 +3867,6 @@ const writeDirectory = async (
   }
 };
 
-const ensureParentDirectories = async (destination) => {
-  const destinationUrl = assertAndNormalizeFileUrl(destination);
-  const destinationPath = urlToFileSystemPath(destinationUrl);
-  const destinationParentPath = dirname(destinationPath);
-
-  await writeDirectory(destinationParentPath, {
-    recursive: true,
-    allowUseless: true,
-  });
-};
-
-const writeFile = async (destination, content = "") => {
-  const destinationUrl = assertAndNormalizeFileUrl(destination);
-  const destinationUrlObject = new URL(destinationUrl);
-  try {
-    await writeFileNaive(destinationUrlObject, content);
-  } catch (error) {
-    if (error.code === "ENOENT") {
-      await ensureParentDirectories(destinationUrl);
-      await writeFileNaive(destinationUrlObject, content);
-      return;
-    }
-    throw error;
-  }
-};
-
-const writeFileNaive = (urlObject, content) => {
-  return new Promise((resolve, reject) => {
-    writeFile$1(urlObject, content, (error) => {
-      if (error) {
-        reject(error);
-      } else {
-        resolve();
-      }
-    });
-  });
-};
-
 const mediaTypeInfos = {
   "application/json": {
     extensions: ["json", "map"],
@@ -5060,18 +5022,12 @@ const registerDirectoryLifecycle = (
   };
   const tracker = trackResources();
   const infoMap = new Map();
-  // type is given when already known (from the directory listing it comes
-  // from): there is then no need to stat the entry, which matters when
-  // indexing a large tree at startup
-  const readEntryInfo = (url, { type } = {}) => {
+  const readEntryInfo = (url) => {
     try {
       const relativeUrl = urlToRelativeUrl(url, source);
       const previousInfo = infoMap.get(relativeUrl);
-      let stat = null;
-      if (type === undefined) {
-        stat = readEntryStatSync(new URL(url));
-        type = statsToType(stat);
-      }
+      const stat = readEntryStatSync(new URL(url));
+      const type = statsToType(stat);
       const patternValue = previousInfo
         ? previousInfo.patternValue
         : getWatchPatternValue({ url, type });
@@ -5228,12 +5184,10 @@ const registerDirectoryLifecycle = (
         const directoryUrl = entryInfo.url.endsWith("/")
           ? entryInfo.url
           : `${entryInfo.url}/`;
-        let direntArray;
+        let entryNameArray;
         try {
           const directoryUrlObject = new URL(directoryUrl);
-          direntArray = readdirSync(directoryUrlObject, {
-            withFileTypes: true,
-          });
+          entryNameArray = readdirSync(directoryUrlObject);
         } catch (e) {
           if (
             e.code === "ENOENT" ||
@@ -5245,17 +5199,12 @@ const registerDirectoryLifecycle = (
           }
           throw e;
         }
-        for (const dirent of direntArray) {
-          const childEntryUrl = new URL(dirent.name, directoryUrl).href;
+        for (const entryName of entryNameArray) {
+          const childEntryUrl = new URL(entryName, directoryUrl).href;
           if (seenSet.has(childEntryUrl)) {
             continue;
           }
-          // the stat is read (and its mtime reported) only when the entry is
-          // notified; a symlink or an exotic entry is stat'ed too, so that it
-          // is typed as what it points to, as for any later event
-          const childEntryInfo = readEntryInfo(childEntryUrl, {
-            type: notify ? undefined : direntToType(dirent),
-          });
+          const childEntryInfo = readEntryInfo(childEntryUrl);
           if (childEntryInfo.type !== null && childEntryInfo.patternValue) {
             applyEntryDiscoveredEffects(childEntryInfo);
           }
@@ -5297,7 +5246,7 @@ const registerDirectoryLifecycle = (
           relativeUrl: entryInfo.relativeUrl,
           type: entryInfo.type,
           patternValue: entryInfo.patternValue,
-          mtime: entryInfo.stat ? entryInfo.stat.mtimeMs : undefined,
+          mtime: entryInfo.stat.mtimeMs,
         });
       }
     };
@@ -5311,7 +5260,7 @@ const registerDirectoryLifecycle = (
         relativeUrl: entryInfo.relativeUrl,
         type: entryInfo.type,
         patternValue: entryInfo.patternValue,
-        mtime: entryInfo.stat ? entryInfo.stat.mtimeMs : undefined,
+        mtime: entryInfo.stat.mtimeMs,
       });
     }
   };
@@ -5323,9 +5272,7 @@ const registerDirectoryLifecycle = (
         type: entryInfo.type,
         patternValue: entryInfo.patternValue,
         mtime: entryInfo.stat.mtimeMs,
-        previousMtime: entryInfo.previousInfo.stat
-          ? entryInfo.previousInfo.stat.mtimeMs
-          : undefined,
+        previousMtime: entryInfo.previousInfo.stat.mtimeMs,
       });
     }
   };
@@ -5361,21 +5308,8 @@ ${relativeUrls.join("\n")}`,
   return tracker.cleanup;
 };
 
-const direntToType = (dirent) => {
-  if (dirent.isFile()) {
-    return "file";
-  }
-  if (dirent.isDirectory()) {
-    return "directory";
-  }
-  return undefined;
-};
-
 const shouldCallUpdated = (entryInfo) => {
   const { stat, previousInfo } = entryInfo;
-  if (!previousInfo.stat) {
-    return true;
-  }
   if (!stat.atimeMs) {
     return true;
   }
@@ -11297,4 +11231,4 @@ const escapeRegexpSpecialChars = (string) => {
   });
 };
 
-export { ANSI, Abort, CONTENT_TYPE, DATA_URL, JS_QUOTES, RUNTIME_COMPAT, UNICODE, URL_META, applyFileSystemMagicResolution, applyNodeEsmResolution, asSpecifierWithoutSearch, asUrlWithoutSearch, assertAndNormalizeDirectoryUrl, browserDefaultRuntimeCompat, bufferToEtag, clearDirectorySync, collectFiles, compareFileUrls, comparePathnames, composeTwoImportMaps, createDetailedMessage$1 as createDetailedMessage, createDynamicLog, createLogger, createLookupPackageDirectory, createTaskLog, distributePercentages, ensureEmptyDirectory, ensurePathnameTrailingSlash, ensureWindowsDriveLetter, errorToHTML, escapeRegexpSpecialChars, generateContentFrame, getCallerPosition, getExtensionsToTry, humanizeDuration, humanizeFileSize, humanizeMemory, inferRuntimeCompatFromClosestPackage, injectQueryParamIntoSpecifierWithoutEncoding, injectQueryParams, injectQueryParamsIntoSpecifier, isFileSystemPath, isSpecifierForNodeBuiltin, lookupPackageDirectory, moveUrl, nodeDefaultRuntimeCompat, normalizeImportMap, normalizeUrl, raceProcessTeardownEvents, readCustomConditionsFromProcessArgs, readEntryStatSync, readPackageAtOrNull, registerDirectoryLifecycle, renderBigSection, renderDetails, renderTable, renderUrlOrRelativeUrlFilename, resolveImport, setUrlBasename, setUrlExtension, setUrlFilename, startMonitoringCpuUsage, startMonitoringMemoryUsage, stringifyUrlSite, updateJsonFileSync, urlIsOrIsInsideOf, urlToBasename, urlToExtension$1 as urlToExtension, urlToFileSystemPath, urlToFilename$1 as urlToFilename, urlToPathname$1 as urlToPathname, urlToRelativeUrl, validateResponseIntegrity, writeFile, writeFileSync };
+export { ANSI, Abort, CONTENT_TYPE, DATA_URL, JS_QUOTES, RUNTIME_COMPAT, UNICODE, URL_META, applyFileSystemMagicResolution, applyNodeEsmResolution, asSpecifierWithoutSearch, asUrlWithoutSearch, assertAndNormalizeDirectoryUrl, browserDefaultRuntimeCompat, bufferToEtag, clearDirectorySync, collectFiles, compareFileUrls, comparePathnames, composeTwoImportMaps, createDetailedMessage$1 as createDetailedMessage, createDynamicLog, createLogger, createLookupPackageDirectory, createTaskLog, distributePercentages, ensureEmptyDirectory, ensurePathnameTrailingSlash, ensureWindowsDriveLetter, errorToHTML, escapeRegexpSpecialChars, generateContentFrame, getCallerPosition, getExtensionsToTry, humanizeDuration, humanizeFileSize, humanizeMemory, inferRuntimeCompatFromClosestPackage, injectQueryParamIntoSpecifierWithoutEncoding, injectQueryParams, injectQueryParamsIntoSpecifier, isFileSystemPath, isSpecifierForNodeBuiltin, lookupPackageDirectory, moveUrl, nodeDefaultRuntimeCompat, normalizeImportMap, normalizeUrl, raceProcessTeardownEvents, readCustomConditionsFromProcessArgs, readEntryStatSync, readPackageAtOrNull, registerDirectoryLifecycle, renderBigSection, renderDetails, renderTable, renderUrlOrRelativeUrlFilename, resolveImport, setUrlBasename, setUrlExtension, setUrlFilename, startMonitoringCpuUsage, startMonitoringMemoryUsage, stringifyUrlSite, updateJsonFileSync, urlIsOrIsInsideOf, urlToBasename, urlToExtension$1 as urlToExtension, urlToFileSystemPath, urlToFilename$1 as urlToFilename, urlToPathname$1 as urlToPathname, urlToRelativeUrl, validateResponseIntegrity, writeFileSync };