Migrate node-fetch to built-in fetch (undici) (#5487)

* Migrate node-fetch to built-in fetch (undici), remove the node-fetch dep

* Better changelog details, fix IPv6 test issues (CI)

* Fix batch total race

Author: myovchev

.changeset/pro-9597-fetch-apostrophe.md

@@ -0,0 +1,19 @@
+---
+"apostrophe": minor
+---
+
+The server-side HTTP client (`apos.http`) now uses Node's built-in `fetch` instead of `node-fetch`.
+
+`node-fetch` is no longer maintained, and Node's built-in `fetch` is its standard, actively maintained successor, available in every Node.js version Apostrophe supports - so this is the right time to adopt it. We do not consider this a breaking change: common `apos.http.*` usage is unchanged, and we deliberately preserved compatibility where it mattered - `form-data` request bodies, cookie jars, the `timeout` option (now backed by an `AbortSignal`), and absolute redirect `Location` headers all behave as before.
+
+Most code that calls `apos.http.get()`, `apos.http.post()`, etc. needs no changes. A few things to be aware of if you use advanced options or read raw responses:
+
+- The `agent` option is no longer supported (the built-in `fetch` has no equivalent). Pass an undici `dispatcher` instead; `apos.http` throws if `agent` is given.
+- A `Host` request header can no longer be set (it is disallowed by the fetch standard and is silently ignored).
+- `originalResponse: true` now resolves with the built-in `fetch` `Response`. Its `body` is a web `ReadableStream` (use `require('node:stream').Readable.fromWeb()` to read it as a Node stream), and node-fetch-only helpers such as `.buffer()` are no longer available.
+- Requests that send a conditional header (`If-None-Match` / `If-Modified-Since`) now also send `Cache-Control: no-cache`, as required by the fetch standard. An endpoint that returns `304 Not Modified` based on those headers may return `200` to such a request.
+
+New capabilities:
+
+- The `timeout` option (in milliseconds) and the standard `signal` (`AbortSignal`) and undici `dispatcher` options are supported.
+- A request `body` may be a native `FormData`, in addition to a `form-data` package instance.

.changeset/pro-9597-fetch-oembetter.md

@@ -0,0 +1,5 @@
+---
+"oembetter": patch
+---
+
+Replaced the `node-fetch` dependency with Node's built-in `fetch`. This is an internal change with no effect on the public API.

.changeset/yummy-buckets-fly.md

@@ -0,0 +1,5 @@
+---
+"apostrophe": patch
+---
+
+Batch jobs now reliably record their total item count, so completion notifications no longer occasionally report a null total.

packages/apostrophe/modules/@apostrophecms/http/index.js

@@ -110,7 +110,9 @@ module.exports = {
           // sends a response with a jobId to the browser
           job = await self.start(options);
 
-          self.setTotal(job, ids.length);
+          // Persist the total before work begins so the completed notification
+          // and job document always report it, even if processing finishes fast.
+          await self.setTotal(job, ids.length);
           // Runs after response is already sent
           run();
 

packages/apostrophe/modules/@apostrophecms/job/index.js

@@ -23,7 +23,7 @@
   },
   "homepage": "https://github.com/apostrophecms/apostrophe/tree/main/packages/apostrophe",
   "engines": {
-    "node": ">=16.0.0"
+    "node": ">=22.0.0"
   },
   "keywords": [
     "apostrophe",
@@ -102,7 +102,6 @@
     "minimatch": "^3.1.4",
     "mkdirp": "^0.5.5",
     "multer": "^2.1.1",
-    "node-fetch": "^2.6.1",
     "nodemailer": "^9.0.1",
     "nunjucks": "^3.2.1",
     "oembetter": "workspace:^",

packages/apostrophe/package.json

@@ -1,5 +1,6 @@
 const fs = require('fs-extra');
 const path = require('path');
+const http = require('node:http');
 
 const setupPackages = ({ folder = 'test' }) => {
   const testNodeModules = path.join(__dirname, '../', folder, 'node_modules/');
@@ -57,5 +58,36 @@ const setupPackages = ({ folder = 'test' }) => {
 };
 setupPackages({ folder: 'test' });
 
+// Performs a GET via the raw node:http client, sending `headers` to the server
+// verbatim. Use this in tests that must control headers the built-in fetch
+// (used by apos.http) would otherwise refuse or rewrite: e.g. a forbidden
+// `Host` header, or the `Cache-Control: no-cache` it adds to any request that
+// carries a conditional header (If-None-Match / If-Modified-Since). `url` may
+// be absolute or site-relative (resolved against `apos.http.getBase()`).
+// Resolves with a fullResponse-shaped { status, headers, body }.
+const rawGet = (apos, url, headers = {}) => {
+  const target = url.startsWith('/') ? `${apos.http.getBase()}${url}` : url;
+  return new Promise((resolve, reject) => {
+    // Pass the URL string (rather than a split hostname/port) so an IPv6 base
+    // such as `http://[::1]:3000` from getBase() is handled correctly; a
+    // bracketed hostname passed on its own is treated as a name to DNS-resolve.
+    const req = http.request(target, {
+      method: 'GET',
+      headers
+    }, (res) => {
+      const chunks = [];
+      res.on('data', (chunk) => chunks.push(chunk));
+      res.on('end', () => resolve({
+        status: res.statusCode,
+        headers: res.headers,
+        body: Buffer.concat(chunks).toString()
+      }));
+    });
+    req.on('error', reject);
+    req.end();
+  });
+};
+
 module.exports = require('./util.js');
 module.exports.setupPackages = setupPackages;
+module.exports.rawGet = rawGet;

packages/apostrophe/test-lib/test.js

@@ -1,6 +1,9 @@
 const t = require('../test-lib/test.js');
 const assert = require('assert/strict');
 const fs = require('fs');
+// rawGet (raw node:http) lets this suite send a spoofed `Host` header, which
+// the built-in fetch used by apos.http would drop as a forbidden header.
+const { rawGet } = t;
 
 describe('Files', function() {
 
@@ -142,17 +145,13 @@ describe('Files', function() {
       const attachment = apos.attachment.first(file);
       const url = apos.attachment.url(attachment);
       assert(url);
-      // Send an attacker-controlled Host header (e.g. the cloud metadata
-      // address from the advisory). The upstream fetch must be resolved
-      // against the server-trusted baseUrl, not this header, so the
-      // legitimate content is still served and the request is never
-      // steered at the spoofed host.
-      const response = await apos.http.get(url, {
-        headers: {
-          Host: '169.254.169.254'
-        },
-        fullResponse: true
-      });
+      // Spoof the Host header (the cloud-metadata address from the advisory)
+      // over a raw request: apos.http uses the built-in fetch, which drops a
+      // forbidden `Host` header and so cannot deliver the spoof. The server
+      // must resolve the upstream fetch against its trusted baseUrl, not this
+      // header, so the legitimate content is still served and the request is
+      // never steered at the spoofed host.
+      const response = await rawGet(apos, url, { Host: '169.254.169.254' });
       assert.strictEqual(response.status, 200);
       assert.strictEqual(response.body, attachment.data);
     } finally {

packages/apostrophe/test/files.js

@@ -1,6 +1,12 @@
 const t = require('../test-lib/test.js');
 const assert = require('assert');
 const _ = require('lodash');
+// The REST API etag tests below issue their conditional request via rawGet
+// (raw node:http): the built-in fetch used by apos.http adds Cache-Control:
+// no-cache to any request carrying a conditional header (Fetch standard), which
+// would suppress the asserted 304s. The page-serving etag tests stay on
+// apos.http (those routes set 304 explicitly).
+const { rawGet } = t;
 
 describe('Pages', function() {
   let apos;
@@ -1087,11 +1093,8 @@ describe('Pages', function() {
     };
 
     const response1 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, { fullResponse: true });
-    const response2 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, `/api/v1/@apostrophecms/page/${homeId}`, {
+      'if-none-match': response1.headers.etag
     });
 
     assert(response1.status === 200);
@@ -1125,11 +1128,8 @@ describe('Pages', function() {
     // so requesting it again should not return a 304 status code
     const pageUpdateResponse = await apos.doc.update(apos.task.getReq(), pageDoc);
 
-    const response2 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, `/api/v1/@apostrophecms/page/${homeId}`, {
+      'if-none-match': response1.headers.etag
     });
 
     const eTag1Parts = response1.headers.etag.split(':');
@@ -1165,11 +1165,8 @@ describe('Pages', function() {
     outOfDateETagParts[2] = Number(outOfDateETagParts[2]) -
       (4444 + 1) * 1000; // 1s outdated
 
-    const response2 = await apos.http.get(`/api/v1/@apostrophecms/page/${homeId}`, {
-      fullResponse: true,
-      headers: {
-        'if-none-match': outOfDateETagParts.join(':')
-      }
+    const response2 = await rawGet(apos, `/api/v1/@apostrophecms/page/${homeId}`, {
+      'if-none-match': outOfDateETagParts.join(':')
     });
 
     const eTag1Parts = response1.headers.etag.split(':');

packages/apostrophe/test/pages.js

@@ -6,6 +6,12 @@ const _ = require('lodash');
 const FormData = require('form-data');
 const t = require('../test-lib/test.js');
 
+// The etag tests below issue their conditional request via rawGet (raw
+// node:http): the built-in fetch used by apos.http adds Cache-Control: no-cache
+// to any request carrying a conditional header (Fetch standard), which would
+// suppress the asserted 304s.
+const { rawGet } = t;
+
 describe('Pieces', function() {
 
   let apos;
@@ -1807,11 +1813,8 @@ describe('Pieces', function() {
     };
 
     const response1 = await apos.http.get('/api/v1/thing/testThing:en:published', { fullResponse: true });
-    const response2 = await apos.http.get('/api/v1/thing/testThing:en:published', {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, '/api/v1/thing/testThing:en:published', {
+      'if-none-match': response1.headers.etag
     });
 
     assert(response1.status === 200);
@@ -1845,11 +1848,8 @@ describe('Pieces', function() {
     // so requesting it again should not return a 304 status code
     const pieceUpdateResponse = await apos.doc.update(apos.task.getReq(), pieceDoc);
 
-    const response2 = await apos.http.get('/api/v1/thing/testThing:en:published', {
-      fullResponse: true,
-      headers: {
-        'if-none-match': response1.headers.etag
-      }
+    const response2 = await rawGet(apos, '/api/v1/thing/testThing:en:published', {
+      'if-none-match': response1.headers.etag
     });
 
     const eTag1Parts = response1.headers.etag.split(':');
@@ -1885,11 +1885,8 @@ describe('Pieces', function() {
     outOfDateETagParts[2] = Number(outOfDateETagParts[2]) -
       (4444 + 1) * 1000; // 1s outdated
 
-    const response2 = await apos.http.get('/api/v1/thing/testThing:en:published', {
-      fullResponse: true,
-      headers: {
-        'if-none-match': outOfDateETagParts.join(':')
-      }
+    const response2 = await rawGet(apos, '/api/v1/thing/testThing:en:published', {
+      'if-none-match': outOfDateETagParts.join(':')
     });
 
     const eTag1Parts = response1.headers.etag.split(':');