🌍 Use undici instead of node-fetch

.changeset/small-jokes-count.md

@@ -0,0 +1,11 @@
+---
+'myst-cli': minor
+'mystmd': minor
+'myst-frontmatter': patch
+'myst-cli-utils': patch
+'myst-templates': patch
+'myst-execute': patch
+'jtex': patch
+---
+
+Enable HTTP caching

packages/jtex/package.json

@@ -52,7 +52,7 @@
     "myst-frontmatter": "^1.7.3",
     "myst-templates": "^1.0.23",
     "myst-common": "^1.7.3",
-    "node-fetch": "^3.3.1",
+    "undici": "^7.16.0",
     "nunjucks": "^3.2.4",
     "pretty-hrtime": "^1.0.3",
     "simple-validators": "^1.0.6"

packages/myst-cli-utils/package.json

@@ -37,6 +37,6 @@
   },
   "devDependencies": {
     "@types/pretty-hrtime": "^1.0.1",
-    "node-fetch": "^3.3.0"
+    "undici": "^7.16.0"
   }
 }

packages/myst-cli-utils/src/session.ts

@@ -1,4 +1,4 @@
-import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+import type { RequestInfo, RequestInit, Response } from 'undici';
 import { chalkLogger, LogLevel } from './logger.js';
 import type { Logger, ISession } from './types.js';
 
@@ -7,7 +7,7 @@
   constructor(opts?: { logger?: Logger }) {
     this.log = opts?.logger ?? chalkLogger(LogLevel.debug, process.cwd());
   }
   fetch(url: URL | RequestInfo, init?: RequestInit | undefined): Promise<Response> {
     throw new Error('fetch not implemented on session');
   }
 }

packages/myst-cli-utils/src/types.ts

@@ -1,4 +1,4 @@
-import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+import type { RequestInfo, RequestInit, Response } from 'undici';
 
 export type Logger = Pick<typeof console, 'debug' | 'info' | 'warn' | 'error'>;
 

packages/myst-cli/package.json

@@ -94,7 +94,7 @@
     "myst-transforms": "^1.3.42",
     "nanoid": "^5.1.6",
     "nbtx": "^0.4.0",
-    "node-fetch": "^3.3.1",
+    "undici": "^7.16.0",
     "p-limit": "^3.1.0",
     "picomatch": "^2.3.1",
     "redux": "^5.0.1",

packages/myst-cli/src/build/html/index.ts

@@ -1,5 +1,6 @@
 import fs from 'fs-extra';
 import path from 'node:path';
+import { pipeline } from 'node:stream/promises';
 import { writeFileToFolder } from 'myst-cli-utils';
 import type { MystXRefs } from 'myst-transforms';
 import type { ISession } from '../../session/types.js';
@@ -159,13 +160,10 @@ export async function buildHtml(session: ISession, opts: StartOptions) {
           return;
         }
         if (route.binary && resp.body) {
-          await new Promise<void>((resolve) => {
-            const filename = path.join(htmlDir, route.path);
-            if (!fs.existsSync(filename)) fs.mkdirSync(path.dirname(filename), { recursive: true });
-            const fileWriteStream = fs.createWriteStream(filename);
-            resp.body!.pipe(fileWriteStream);
-            fileWriteStream.on('finish', resolve);
-          });
+          const filename = path.join(htmlDir, route.path);
+          if (!fs.existsSync(filename)) fs.mkdirSync(path.dirname(filename), { recursive: true });
+          const fileWriteStream = fs.createWriteStream(filename);
+          await pipeline(resp.body!, fileWriteStream);
         } else {
           const content = await resp.text();
           writeFileToFolder(path.join(htmlDir, route.path), content);

packages/myst-cli/src/process/loadReferences.ts

@@ -1,5 +1,5 @@
 import fs from 'node:fs';
-import type { Response } from 'node-fetch';
+import type { Response } from 'undici';
 import { Inventory } from 'intersphinx';
 import { tic, isUrl, computeHash } from 'myst-cli-utils';
 import { RuleId, fileError } from 'myst-common';

packages/myst-cli/src/session/session.ts

@@ -7,7 +7,6 @@ import type { RuleId, ValidatedMystPlugin } from 'myst-common';
 import latestVersion from 'latest-version';
 import boxen from 'boxen';
 import chalk from 'chalk';
-import { HttpsProxyAgent } from 'https-proxy-agent';
 import pLimit from 'p-limit';
 import type { Limit } from 'p-limit';
 import {
@@ -26,17 +25,9 @@ import { isWhiteLabelled } from '../utils/whiteLabelling.js';
 import { KernelManager, ServerConnection, SessionManager } from '@jupyterlab/services';
 import type { JupyterServerSettings } from 'myst-execute';
 import { launchJupyterServer } from 'myst-execute';
-import type { RequestInfo, RequestInit } from 'node-fetch';
-import { default as nodeFetch, Headers, Request, Response } from 'node-fetch';
 import type { PluginInfo } from 'myst-config';
-
-// fetch polyfill for node<18
-if (!globalThis.fetch) {
-  globalThis.fetch = nodeFetch as any;
-  globalThis.Headers = Headers as any;
-  globalThis.Request = Request as any;
-  globalThis.Response = Response as any;
-}
+import { fetch as fetchImpl, Agent, ProxyAgent } from 'undici';
+import type { RequestInfo, RequestInit, Response, Dispatcher } from 'undici';
 
 const CONFIG_FILES = ['myst.yml'];
 const API_URL = 'https://api.mystmd.org';
@@ -88,7 +79,9 @@ export class Session implements ISession {
   $logger: Logger;
   doiLimiter: Limit;
 
-  proxyAgent?: HttpsProxyAgent<string>;
+  localDispatcher: Dispatcher;
+  dispatcher: Dispatcher;
+
   _shownUpgrade = false;
   _latestVersion?: string;
   _jupyterSessionManagerPromise?: Promise<SessionManager | undefined>;
@@ -102,8 +95,14 @@ export class Session implements ISession {
     this.configFiles = (opts.configFiles ? opts.configFiles : CONFIG_FILES).slice();
     this.$logger = opts.logger ?? chalkLogger(LogLevel.info, process.cwd());
     this.doiLimiter = opts.doiLimiter ?? pLimit(3);
+
     const proxyUrl = process.env.HTTPS_PROXY;
-    if (proxyUrl) this.proxyAgent = new HttpsProxyAgent(proxyUrl);
+    if (proxyUrl !== undefined) {
+      this.log.debug(`Using HTTPS proxy ${proxyUrl}`);
+    }
+    this.dispatcher = proxyUrl ? new ProxyAgent(proxyUrl) : new Agent();
+    this.localDispatcher = new Agent();
+
     this.store = createStore(rootReducer);
     // Allow the latest version to be loaded
     latestVersion('mystmd')
@@ -144,16 +143,21 @@ export class Session implements ISession {
   async fetch(url: URL | RequestInfo, init?: RequestInit): Promise<Response> {
     const urlOnly = new URL((url as Request).url ?? (url as URL | string));
     this.log.debug(`Fetching: ${urlOnly}`);
-    if (this.proxyAgent && !LOCALHOSTS.includes(urlOnly.hostname)) {
-      if (!init) init = {};
-      init = { agent: this.proxyAgent, ...init };
-      this.log.debug(`Using HTTPS proxy: ${this.proxyAgent.proxy}`);
+
+    const isLocal = LOCALHOSTS.includes(urlOnly.hostname);
+    if (isLocal) {
+      this.log.debug(`Using local dispatcher for request to ${urlOnly.hostname}`);
     }
+
     const logData = { url: urlOnly, done: false };
     setTimeout(() => {
       if (!logData.done) this.log.info(`⏳ Waiting for response from ${url}`);
     }, 5000);
-    const resp = await nodeFetch(url, init);
+
+    const resp = await fetchImpl(url, {
+      ...init,
+      dispatcher: isLocal ? this.localDispatcher! : this.dispatcher,
+    });
     logData.done = true;
     return resp;
   }

packages/myst-cli/src/session/types.ts

@@ -4,7 +4,7 @@ import type { MystPlugin, RuleId, ValidatedMystPlugin } from 'myst-common';
 import type { ResolvedExternalReference } from 'myst-transforms';
 import type { MinifiedContentCache } from 'nbtx';
 import type { Store } from 'redux';
-import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+import type { RequestInfo, RequestInit, Response } from 'undici';
 import type { Limit } from 'p-limit';
 import type { BuildWarning, RootState } from '../store/index.js';
 import type { PreRendererData, RendererData, SingleCitationRenderer } from '../transforms/types.js';

packages/myst-cli/src/transforms/images.ts

@@ -1,4 +1,5 @@
 import fs from 'node:fs';
+import { pipeline } from 'node:stream/promises';
 import mime from 'mime-types';
 import type { GenericNode, GenericParent } from 'myst-common';
 import { RuleId, plural } from 'myst-common';
@@ -88,15 +89,9 @@ export async function downloadAndSaveImage(
     if (!fs.existsSync(fileFolder)) fs.mkdirSync(fileFolder, { recursive: true });
     // Write to a file
     const fileStream = fs.createWriteStream(`${filePath}.${extension}`);
-    await new Promise((resolve, reject) => {
-      if (!res.body) {
-        reject(`no response body from ${url}`);
-      } else {
-        res.body.pipe(fileStream);
-        res.body.on('error', reject);
-        fileStream.on('finish', resolve as () => void);
-      }
-    });
+    if (!res.body) throw new Error('No response from body');
+
+    await pipeline(res.body, fileStream);
     await new Promise((r) => setTimeout(r, 50));
     const fileName = `${file}.${extension}`;
     session.log.debug(`Image successfully saved to: ${fileName}`);

packages/myst-cli/src/utils/fetchWithRetry.ts

@@ -1,4 +1,4 @@
-import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+import type { RequestInfo, RequestInit, Response } from 'undici';
 import type { ISession } from '../session/types.js';
 
 /**

packages/myst-execute/package.json

@@ -37,7 +37,7 @@
     "chalk": "^5.2.0",
     "myst-cli-utils": "^2.0.12",
     "myst-common": "^1.9.0",
-    "node-fetch": "^3.3.0",
+    "undici": "^7.16.0",
     "unist-util-select": "^4.0.3",
     "vfile": "^5.3.7",
     "which": "^4.0.0"

packages/myst-execute/tests/run.spec.ts

@@ -8,7 +8,8 @@ import type { ExecutionResult } from '../src/types.js';
 import type { GenericParent } from 'myst-common';
 import { VFile } from 'vfile';
 import { KernelManager, ServerConnection, SessionManager } from '@jupyterlab/services';
-import { default as nodeFetch, Headers, Request, Response } from 'node-fetch';
+import { fetch as nodeFetch, Headers, Request, Response } from 'undici';
+import type { IOutput } from '@jupyterlab/nbformat';
 
 // fetch polyfill for node<18
 if (!globalThis.fetch) {

packages/myst-frontmatter/bin/fetchLicenses.mjs

@@ -1,8 +1,8 @@
 import fs from 'fs';
-import fetch from 'node-fetch';
+import { fetch as fetch as nodeFetch } from 'undici';
 
 (async () => {
-  const data = await (await fetch('https://spdx.org/licenses/licenses.json')).json();
+  const data = await (await fetch as nodeFetch('https://spdx.org/licenses/licenses.json')).json();
   fs.writeFileSync(
     'licenses.json',
     JSON.stringify(

packages/myst-frontmatter/package.json

@@ -44,6 +44,6 @@
     "glob": "^10.3.1",
     "js-yaml": "^4.1.0",
     "moment": "^2.29.4",
-    "node-fetch": "^3.3.2"
+    "undici": "^7.16.0"
   }
 }

packages/myst-frontmatter/src/licenses/licenses.spec.ts

@@ -1,7 +1,7 @@
 import { describe, expect, beforeEach, it } from 'vitest';
 import type { ValidationOptions } from 'simple-validators';
 import licenses from './licenses';
-import fetch from 'node-fetch';
+import { fetch as nodeFetch } from 'undici';
 import {
   licensesToString,
   simplifyLicenses,
@@ -25,7 +25,7 @@ beforeEach(() => {
 
 describe('licenses are upto date with SPDX', () => {
   it('compare with https://spdx.org/licenses/licenses.json', async () => {
-    const data: any = await (await fetch('https://spdx.org/licenses/licenses.json')).json();
+    const data: any = await (await nodeFetch('https://spdx.org/licenses/licenses.json')).json();
     const onlineLicenses = Object.fromEntries(
       (data.licenses as any[])
         .filter((l) => !l.isDeprecatedLicenseId)

packages/myst-templates/package.json

@@ -45,7 +45,7 @@
     "myst-cli-utils": "^2.0.12",
     "myst-common": "^1.8.3",
     "myst-frontmatter": "^1.8.3",
-    "node-fetch": "^3.3.1",
+    "undici": "^7.16.0",
     "pretty-hrtime": "^1.0.3",
     "simple-validators": "^1.0.6"
   },

packages/myst-templates/src/download.ts

@@ -1,5 +1,6 @@
 import fs, { createWriteStream, mkdirSync } from 'node:fs';
 import { extname, join, parse, sep } from 'node:path';
+import { pipeline } from 'node:stream/promises';
 import { createHash } from 'node:crypto';
 import AdmZip from 'adm-zip';
 import { glob } from 'glob';
@@ -214,12 +215,9 @@ export async function downloadAndUnzipTemplate(
   const { templatePath } = opts;
   const zipFile = join(templatePath, 'template.zip');
   mkdirSync(templatePath, { recursive: true });
-  const fileStream = createWriteStream(zipFile);
-  await new Promise((resolve, reject) => {
-    res.body?.pipe(fileStream);
-    res.body?.on('error', reject);
-    fileStream.on('finish', resolve as () => void);
-  });
+  if (res.body) {
+    await pipeline(res.body, createWriteStream(zipFile));
+  }
   session.log.debug(`Unzipping template on disk ${zipFile}`);
 
   const zip = new AdmZip(zipFile);

packages/myst-templates/src/session.ts

@@ -1,8 +1,8 @@
 import { chalkLogger, LogLevel } from 'myst-cli-utils';
 import type { Logger } from 'myst-cli-utils';
 import type { ISession } from './types.js';
-import { default as nodeFetch } from 'node-fetch';
-import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+import { fetch as nodeFetch } from 'undici';
+import type { RequestInfo, RequestInit, Response } from 'undici';
 
 export class Session implements ISession {
   API_URL = 'https://api.mystmd.org';

packages/mystmd/package.json

@@ -37,7 +37,7 @@
     "lint:format": "npm run copy:version; prettier --check \"src/**/*.ts\"",
     "test": "npm run link; npm run copy:version; vitest run",
     "test:watch": "npm run link; npm run copy:version; vitest watch",
-    "build:cli": "esbuild src/index.ts --bundle --outfile=dist/myst.cjs --platform=node --external:fsevents --target=node14",
+    "build:cli": "esbuild src/index.ts --bundle --outfile=dist/myst.cjs --platform=node --external:fsevents --target=node18",
     "build": "npm-run-all -l clean copy:version -p build:cli"
   },
   "devDependencies": {