fix: support accept header as per conformance tests (#940)

Enable accept header related conformance tests and update implementation
to make them pass.

Author: achingbrain

README.md

@@ -19,9 +19,8 @@
 
 ## About
 
-This project demonstrates
-the use of [Helia](https://github.com/ipfs/helia) (IPFS implementation in JS)
-and the [`verified-fetch` library](https://github.com/ipfs/helia-verified-fetch)
+This project demonstrates the use of [Helia](https://github.com/ipfs/helia)
+(IPFS implementation in JS) and the [`verified-fetch` library](https://github.com/ipfs/helia-verified-fetch)
 ([Fetch API](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) for IPFS)
 within a [Service Worker](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API)
 to facilitate direct verified retrieval of content-addressed data.
@@ -41,7 +40,6 @@ tasks.
 
 This project  was brought to you by the [Shipyard](http://ipshipyard.com/) team.
 
-
 ### Goals
 
 The main goals of this project are:
@@ -65,7 +63,6 @@ The main goals of this project are:
      gaps, and improve quality of libraries, specifications, and interop tests.
 
 
-
 ### Feature Set
 
 - 🚧 **WIP:** Web interface for adjusting routing and retrieval settings.

build.js

@@ -429,8 +429,7 @@ if (watchRequested) {
 if (serveRequested) {
   const { port } = await ctx.serve({
     servedir: 'dist',
-    port: 8345,
-    host: 'localhost'
+    port: 8345
   })
   console.info(`Listening on http://localhost:${port}`)
 }

package.json

@@ -21,7 +21,6 @@
     "dev": "node dist-tsc/serve.js --load-fixtures",
     "prestart": "run-s build:binary",
     "start": "./service-worker-gateway",
-    "start:frontend": "node build.js --watch --serve",
     "start:cloudflare": "npm run build && npx wrangler pages dev dist",
     "prestart:hosted": "run-s build:tsc",
     "test": "run-s test:iso 'test:node -- -b false'",
@@ -63,7 +62,7 @@
     "@helia/http": "^3.0.8",
     "@helia/interface": "^6.0.2",
     "@helia/routers": "^4.0.3",
-    "@helia/verified-fetch": "^5.0.0",
+    "@helia/verified-fetch": "^5.0.4",
     "@ipld/dag-cbor": "^9.2.5",
     "@ipld/dag-json": "^10.2.5",
     "@ipld/dag-pb": "^4.1.5",
@@ -119,9 +118,11 @@
     "fast-glob": "^3.3.3",
     "glob": "^13.0.0",
     "http-server": "^14.1.1",
+    "ipfs-unixfs": "^12.0.0",
     "ipfs-unixfs-importer": "^16.0.1",
     "ipns": "^10.1.3",
     "it-all": "^3.0.7",
+    "it-drain": "^3.0.10",
     "it-to-buffer": "^4.0.10",
     "kubo": "^0.39.0",
     "kubo-rpc-client": "^6.0.2",

serve.ts

@@ -10,18 +10,17 @@
  * This file expects that `build:tsc` was ran first, and this will be handled
  * for you if ran via `npm run start`.
  */
+import { existsSync, createReadStream } from 'node:fs'
+import { createServer, Server } from 'node:http'
+import { join, extname } from 'node:path'
 import { pathToFileURL } from 'node:url'
-import { logger } from '@libp2p/logger'
 import { execa } from 'execa'
 import { createKuboNode } from './test-e2e/fixtures/create-kubo-node.js'
 import { loadIpnsRecords } from './test-e2e/fixtures/load-ipns-records.js'
 import { downloadFixtures, getIpfsNsMap, loadCarFixtures } from './test-e2e/fixtures/load-kubo-fixtures.js'
 import { setupIpfsGateway } from './test-e2e/ipfs-gateway.js'
-import { createReverseProxy } from './test-e2e/reverse-proxy.js'
 import type { KuboNode } from 'ipfsd-ctl'
 
-const log = logger('serve')
-
 // if user passes "--load-fixtures" flag, load all the fixtures
 const args = process.argv.slice(2)
 
@@ -34,14 +33,49 @@ async function loadFixtures (): Promise<{
   const controller = await createKuboNode(IPFS_NS_MAP)
   await controller.start()
 
-  await loadCarFixtures()
-  await loadIpnsRecords(controller, log)
+  await loadCarFixtures(controller)
+  await loadIpnsRecords(controller)
 
   return {
     controller
   }
 }
 
+const MIME_TYPES: Record<string, string> = {
+  '.js': 'text/javascript; charset=utf8',
+  '.png': 'image/png',
+  '.map': 'text/plain',
+  '.css': 'text/css; charset=utf8',
+  '.svg': 'image/svg+xml',
+  '.json': 'application/json; charset=utf8',
+  '.html': 'text/html; charset=utf8'
+}
+
+/**
+ * create a web server that serves an asset if it exists or index.html if not,
+ * this simulates the behaviour of Cloudflare with `./dist/_redirects`
+ */
+function createFrontend (): Server {
+  return createServer((req, res) => {
+    let file = req.url
+
+    if (file == null || file === '/') {
+      file = 'index.html'
+    }
+
+    let asset = join('./dist', file)
+
+    if (!existsSync(asset)) {
+      // serve index.html instead of 404ing
+      asset = './dist/index.html'
+    }
+
+    res.statusCode = 200
+    res.setHeader('content-type', MIME_TYPES[extname(asset)] ?? 'application/octet-stream')
+    createReadStream(asset).pipe(res)
+  })
+}
+
 export interface ServeOptions {
   shouldLoadFixtures?: boolean
   shouldStartFrontend?: boolean
@@ -54,37 +88,42 @@ export async function serve ({ shouldLoadFixtures = false, shouldStartFrontend =
   controller: KuboNode
 }> {
   let controller: KuboNode
+
   if (shouldLoadFixtures) {
     const fixtures = await loadFixtures()
     controller = fixtures.controller
-    // IPFS_NS_MAP = fixtures.IPFS_NS_MAP
   } else {
     controller = await createKuboNode()
   }
 
   // sets up kubo node and ipfs gateway
   const ipfsGateway = await setupIpfsGateway()
-
+  /*
   // sets up reverse proxy for front-end assets being auto-loaded
   const reverseProxy = createReverseProxy({
     backendPort: shouldStartFrontend ? 8345 : 3000, // front-end server port. 3000 if playwright is running, 8345 if build.js starts the server
     proxyPort: 3333
   })
+*/
+  // rebuild the app on changes
+  const build = shouldStartFrontend ? execa('node', ['build.js', '--watch']) : undefined
+  build?.stdout?.pipe(process.stdout)
+  build?.stderr?.pipe(process.stderr)
 
-  // call build.js with --serve and --watch flags, piping the output to the console
-  const frontend = shouldStartFrontend ? execa('node', ['build.js', '--serve', '--watch']) : undefined
-
-  frontend?.stdout?.pipe(process.stdout)
-  frontend?.stderr?.pipe(process.stderr)
+  // serve the dist folder similarly to how cloudflare does
+  const frontend = createFrontend()
+  frontend.listen(3333)
 
   const cleanup = async (): Promise<void> => {
-    frontend?.kill()
-    reverseProxy.close()
+    build?.kill()
+    // reverseProxy.close()
     await ipfsGateway.stop()
+    frontend.close()
+    frontend.closeAllConnections()
   }
 
   // when the process exits, stop the reverse proxy
-  frontend?.on('exit', () => { void cleanup() })
+  build?.on('exit', () => { void cleanup() })
   process.on('SIGINT', () => { void cleanup() })
   process.on('SIGTERM', () => { void cleanup() })
 
@@ -95,5 +134,14 @@ export async function serve ({ shouldLoadFixtures = false, shouldStartFrontend =
 
 // Run main function if this file is being executed directly
 if (import.meta.url === pathToFileURL(process.argv[1]).href) {
-  await serve({ shouldLoadFixtures: args.includes('--load-fixtures'), shouldStartFrontend: true })
+  const {
+    controller
+  } = await serve({ shouldLoadFixtures: args.includes('--load-fixtures'), shouldStartFrontend: true })
+
+  const info = await controller.info()
+
+  // eslint-disable-next-line no-console
+  console.info('Kubo gateway:', info.gateway)
+  // eslint-disable-next-line no-console
+  console.info('Kubo RPC API:', info.api)
 }

src/lib/path-or-subdomain.ts

@@ -77,11 +77,6 @@ export const toSubdomainRequest = (location: URL): URL => {
     .split('/')
     .filter(segment => segment !== '')
 
-  // remove cloudflare redirect
-  if (segments[0] === 'index.html') {
-    segments.shift()
-  }
-
   if (segments.length < 2) {
     throw new InvalidParametersError(`Invalid location ${location}`)
   }