fix: parse path/subdomain gateway url as ipfs/ipns url (#946)

Converts path/subdomain gateway urls to ipfs/ipns urls before handing off to verified fetch.

Enables cache and directory listing conformance tests.

Author: achingbrain

package.json

@@ -62,7 +62,7 @@
     "@helia/http": "^3.0.8",
     "@helia/interface": "^6.0.2",
     "@helia/routers": "^4.0.3",
-    "@helia/verified-fetch": "^5.1.0",
+    "@helia/verified-fetch": "^6.0.0",
     "@ipld/dag-cbor": "^9.2.5",
     "@ipld/dag-json": "^10.2.5",
     "@ipld/dag-pb": "^4.1.5",

src/sw/handlers/content-request-handler.ts

@@ -12,6 +12,7 @@ import { isBitswapProvider, isTrustlessGatewayProvider } from '../../lib/provide
 import { APP_NAME, APP_VERSION, GIT_REVISION } from '../../version.js'
 import { getConfig } from '../lib/config.js'
 import { getInstallTime } from '../lib/install-time.js'
+import { httpResourceToIpfsUrl } from '../lib/resource-to-url.ts'
 import { getVerifiedFetch } from '../lib/verified-fetch.js'
 import { fetchErrorPageResponse } from '../pages/fetch-error-page.js'
 import { originIsolationWarningPageResponse } from '../pages/origin-isolation-warning-page.js'
@@ -223,11 +224,27 @@ async function fetchHandler ({ url, headers, renderPreview, event, logs, subdoma
     otherCount: 0
   }
 
-  const resource = url.href
+  const resource = httpResourceToIpfsUrl(url)
+
+  // check for redirect
+  if (resource instanceof Response) {
+    return resource
+  }
 
   const firstInstallTime = await getInstallTime()
   const start = Date.now()
 
+  let ifNoneMatch = headers.get('if-none-match')
+
+  // these tokens are added to the header by the entity renderer response,
+  // remove them for internal comparison by verified fetch
+  if (ifNoneMatch != null) {
+    ifNoneMatch = ifNoneMatch.replace('DirIndex-.*_CID-', '')
+    ifNoneMatch = ifNoneMatch.replace('DagIndex-', '')
+
+    headers.set('if-none-match', ifNoneMatch)
+  }
+
   /**
    * Note that there are existing bugs regarding service worker signal handling:
    * https://bugs.chromium.org/p/chromium/issues/detail?id=823697
@@ -359,7 +376,8 @@ async function fetchHandler ({ url, headers, renderPreview, event, logs, subdoma
       status: 500,
       statusText: 'Internal Server Error',
       headers: {
-        'content-type': 'application/json'
+        'content-type': 'application/json',
+        'x-error-message': btoa(err.message)
       }
     }), JSON.stringify(errorToObject(err), null, 2), providers, config, firstInstallTime, logs)
   } finally {
@@ -384,10 +402,6 @@ function shouldRenderDirectory (url: URL, config: ConfigDb, accept?: string | nu
     return true
   }
 
-  if (config.renderHTMLViews === false) {
-    return false
-  }
-
   return accept?.includes('text/html') === true
 }
 

src/sw/index.ts

@@ -6,6 +6,7 @@ import { APP_NAME, APP_VERSION, GIT_REVISION } from '../version.js'
 import { handlers } from './handlers/index.js'
 import { getConfig } from './lib/config.js'
 import { getInstallTime, setInstallTime } from './lib/install-time.js'
+import { updateRedirect } from './lib/update-redirect.ts'
 import { serverErrorPageResponse } from './pages/server-error-page.js'
 
 /**
@@ -116,6 +117,10 @@ self.addEventListener('fetch', (event) => {
           )
         }
 
+        // if verified-fetch has redirected us, update the location header in
+        // the response to be a HTTP location not ipfs/ipns
+        updateRedirect(url, response)
+
         // add the server header to the response so we can be sure this response
         // came from the service worker - sometimes these are read-only so we
         // cannot just `response.headers.set('server', ...)`

src/sw/lib/resource-to-url.ts

@@ -0,0 +1,132 @@
+import { InvalidParametersError } from '@libp2p/interface'
+import { peerIdFromString } from '@libp2p/peer-id'
+
+export const SUBDOMAIN_GATEWAY_REGEX = /^(?<cidOrPeerIdOrDnsLink>[^/?]+)\.(?<protocol>ip[fn]s)\.(?<host>[^/?]+)$/
+
+export interface SubdomainMatchGroups {
+  protocol: 'ipfs' | 'ipns'
+  cidOrPeerIdOrDnsLink: string
+  host: string
+}
+
+export function matchSubdomainGroupsGuard (groups?: null | { [key in string]: string; } | SubdomainMatchGroups): groups is SubdomainMatchGroups {
+  const protocol = groups?.protocol
+
+  if (protocol !== 'ipfs' && protocol !== 'ipns') {
+    return false
+  }
+
+  const cidOrPeerIdOrDnsLink = groups?.cidOrPeerIdOrDnsLink
+
+  if (cidOrPeerIdOrDnsLink == null) {
+    return false
+  }
+
+  return true
+}
+
+// DNS label can have up to 63 characters, consisting of alphanumeric
+// characters or hyphens -, but it must not start or end with a hyphen.
+const dnsLabelRegex = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/
+
+/**
+ * Checks if label looks like inlined DNSLink.
+ * (https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header)
+ */
+function isInlinedDnsLink (label: string): boolean {
+  return dnsLabelRegex.test(label) && label.includes('-') && !label.includes('.')
+}
+
+/**
+ * DNSLink label decoding
+ * - Every standalone - is replaced with .
+ * - Every remaining -- is replaced with -
+ *
+ * @example en-wikipedia--on--ipfs-org -> en.wikipedia-on-ipfs.org
+ */
+export function decodeDNSLinkLabel (label: string): string {
+  return label.replace(/--/g, '%').replace(/-/g, '.').replace(/%/g, '-')
+}
+
+/**
+ * If the caller has passed a case-sensitive identifier (like a base58btc
+ * encoded CID or PeerId) in a case-insensitive location (like a subdomain),
+ * be nice and return the original identifier from the passed string
+ */
+function findOriginalCidOrPeer (needle: string, haystack: URL): string {
+  const start = haystack.href.toLowerCase().indexOf(needle)
+
+  if (start === -1) {
+    return needle
+  }
+
+  return haystack.href.substring(start, start + needle.length)
+}
+
+/**
+ * Takes a subdomain or path gateway URL and turns it into an IPFS/IPNS URL, or
+ * a redirect response that directs the user to a canonical URL for the resource
+ */
+export function httpResourceToIpfsUrl (resource: URL): URL | Response {
+  // test for subdomain gateway URL - match hostname to exclude port
+  const subdomainMatch = resource.hostname.match(SUBDOMAIN_GATEWAY_REGEX)
+
+  if (matchSubdomainGroupsGuard(subdomainMatch?.groups)) {
+    const groups = subdomainMatch.groups
+
+    if (groups.protocol === 'ipns' && isInlinedDnsLink(groups.cidOrPeerIdOrDnsLink)) {
+      // decode inline dnslink domain if present
+      groups.cidOrPeerIdOrDnsLink = decodeDNSLinkLabel(groups.cidOrPeerIdOrDnsLink)
+    }
+
+    const cidOrPeerIdOrDnsLink = findOriginalCidOrPeer(groups.cidOrPeerIdOrDnsLink, resource)
+
+    // parse url as not http(s):// - this is necessary because URL makes
+    // `.pathname` default to `/` for http URLs, even if no trailing slash was
+    // present in the string URL and we need to be able to round-trip the user's
+    // input while also maintaining a sane canonical URL for the resource. Phew.
+    const wat = new URL(`not-${resource}`)
+
+    return new URL(`${groups.protocol}://${cidOrPeerIdOrDnsLink}${wat.pathname}${resource.search}${resource.hash}`)
+  }
+
+  // test for IPFS path gateway URL
+  if (resource.pathname.startsWith('/ipfs/')) {
+    const parts = resource.pathname.substring(6).split('/')
+    const cid = parts.shift()
+
+    if (cid == null) {
+      throw new InvalidParametersError(`Path gateway URL ${resource} had no CID`)
+    }
+
+    return new URL(`ipfs://${cid}${resource.pathname.replace(`/ipfs/${cid}`, '')}${resource.search}${resource.hash}`)
+  }
+
+  // test for IPNS path gateway URL
+  if (resource.pathname.startsWith('/ipns/')) {
+    const parts = resource.pathname.substring(6).split('/')
+    let name = parts.shift()
+
+    if (name == null) {
+      throw new InvalidParametersError(`Path gateway URL ${resource} had no name`)
+    }
+
+    // special case - re-encode base58btc IPNS name as base36 CID and redirect
+    // @see TestRedirectCanonicalIPNS/GET_for_%2Fipns%2F%7Bb58-multihash-of-ed25519-key%7D_redirects_to_%2Fipns%2F%7Bcidv1-libp2p-key-base36%7D
+    if (name.startsWith('12D3K')) {
+      const peerId = peerIdFromString(name)
+      name = peerId.toCID().toString()
+
+      return new Response('', {
+        status: 301,
+        headers: {
+          location: `/ipns/${name}/${parts.join('/')}${resource.search}${resource.hash}`
+        }
+      })
+    }
+
+    return new URL(`ipns://${name}${resource.pathname.replace(`/ipns/${name}`, '')}${resource.search}${resource.hash}`)
+  }
+
+  throw new TypeError(`Invalid URL: ${resource}, please use subdomain or path gateway URLs only`)
+}

src/sw/lib/update-redirect.ts

@@ -0,0 +1,39 @@
+import { matchSubdomainGroupsGuard, SUBDOMAIN_GATEWAY_REGEX } from './resource-to-url.ts'
+
+/**
+ * If the response has a location header with an ipfs/ipns URL, translate it
+ * into a HTTP URL that a browser can use
+ */
+export function updateRedirect (resource: URL, response: Response): Response {
+  let location = response.headers.get('location')
+
+  if (location == null || location.trim() === '') {
+    return response
+  }
+
+  if (location.startsWith('?') || location.startsWith('/') || location.startsWith('#')) {
+    // partial location, prefix with current origin
+    location = `${resource.href}${location}`
+  }
+
+  const url = new URL(location)
+
+  if (url.protocol.startsWith('http')) {
+    return response
+  }
+
+  // match host to include port (if present)
+  const subdomainMatch = resource.host.match(SUBDOMAIN_GATEWAY_REGEX)
+
+  if (matchSubdomainGroupsGuard(subdomainMatch?.groups)) {
+    const { host } = subdomainMatch.groups
+
+    location = `${resource.protocol}//${url.hostname}.${url.protocol.replace(':', '')}.${host}${url.pathname}${url.search}${url.hash}`
+  } else {
+    location = `${resource.protocol}//${resource.host}/${url.protocol.replace(':', '')}/${url.hostname}${url.pathname}${url.search}${url.hash}`
+  }
+
+  response.headers.set('location', location)
+
+  return response
+}

src/sw/pages/fetch-error-page.ts

@@ -58,12 +58,12 @@ function toErrorPageProviders (providers: Providers): ErrorPageProviders {
 }
 
 export interface RequestDetails {
-  resource: string
+  resource: URL
   method: string
   headers: Record<string, string>
 }
 
-function getRequestDetails (resource: string, init: RequestInit): RequestDetails {
+function getRequestDetails (resource: URL, init: RequestInit): RequestDetails {
   const requestHeaders = new Headers(init.headers)
   const headers: Record<string, string> = {}
   requestHeaders.forEach((value, key) => {
@@ -103,7 +103,7 @@ function getResponseDetails (response: Response, body: string): ResponseDetails
 /**
  * Shows an error page to the user
  */
-export function fetchErrorPageResponse (resource: string, request: RequestInit, fetchResponse: Response, responseBody: string, providers: Providers, config: ConfigDb, installTime: number, logs: string[]): Response {
+export function fetchErrorPageResponse (resource: URL, request: RequestInit, fetchResponse: Response, responseBody: string, providers: Providers, config: ConfigDb, installTime: number, logs: string[]): Response {
   const responseContentType = fetchResponse.headers.get('Content-Type')
 
   if (responseContentType?.includes('text/html')) {

src/sw/pages/render-entity.ts

@@ -1,3 +1,4 @@
+import { MEDIA_TYPE_DAG_PB } from '@helia/verified-fetch'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
 import { headersToObject } from '../../lib/headers-to-object.ts'
 import { APP_NAME, APP_VERSION, GIT_REVISION } from '../../version.js'
@@ -33,7 +34,11 @@ export function renderEntityPageResponse (url: URL, headers: Headers, response:
   const page = htmlPage(props.cid ?? '', 'renderEntity', props)
   mergedHeaders.set('content-length', `${page.length}`)
 
-  mergedHeaders.set('etag', `"DagIndex-${props.cid}.html"`)
+  if (contentType === MEDIA_TYPE_DAG_PB) {
+    mergedHeaders.set('etag', `"DirIndex-.*_CID-${props.cid}"`)
+  } else {
+    mergedHeaders.set('etag', `"DagIndex-${props.cid}"`)
+  }
 
   return new Response(page, {
     status: response.status,

src/ui/utils/links.ts

@@ -38,7 +38,11 @@ export function createLink ({ ipfsPath, params, hash }: CreateLinkOptions): stri
   if (isSubdomain) {
     const host = (url.host.includes('.ipfs.') ? url.host.split('.ipfs.') : url.host.split('.ipns.'))[1]
 
-    return `${url.protocol}//${CID.parse(cid).toV1()}.ipfs.${host}${path === '/' ? '' : path}${search}${hash ?? url.hash}`
+    try {
+      return `${url.protocol}//${CID.parse(cid).toV1()}.ipfs.${host}${path === '/' ? '' : path}${search}${hash ?? url.hash}`
+    } catch {
+      return `${url.protocol}//${cid}.ipns.${host}${path === '/' ? '' : path}${search}${hash ?? url.hash}`
+    }
   }
 
   return `${url.protocol}//${url.host}/ipfs/${cid}${path === '/' ? '' : path}${search}${hash ?? url.hash}`

test-conformance/fixtures/expected-passing-tests.json

@@ -387,7 +387,6 @@
   "TestGatewayIPNSPath/GET_for_%2Fipns%2Fname_with_valid_V2-only_signature_succeeds",
   "TestGatewayIPNSPath/GET_for_%2Fipns%2Fname_with_valid_V2-only_signature_succeeds/Body",
   "TestGatewayIPNSPath/GET_for_%2Fipns%2Fname_with_valid_V1_and_broken_V2_signature_MUST_fail_with_5XX",
-  "TestRedirectCanonicalIPNS",
   "TestGatewayBlock",
   "TestGatewayBlock/GET_with_format=raw_param_returns_a_raw_block",
   "TestGatewayBlock/GET_with_format=raw_param_returns_a_raw_block/Status_code",
@@ -441,6 +440,27 @@
   "TestUnixFSDirectoryListing/GET_for_%2Fipfs%2Fcid%2Ffile_UnixFS_file_that_does_not_exist_returns_404",
   "TestUnixFSDirectoryListing/GET_for_%2Fipfs%2Fcid%2Ffile_UnixFS_file_that_does_not_exist_returns_404/Status_code",
   "TestGatewayCache",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_0",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_0/Status_code",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_0/Header_X-Ipfs-Path",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_0/Header_X-Ipfs-Roots",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_0/Header_Etag",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_1",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_1/Check_0",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_listing_succeeds/Check_1/Check_1",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_with_index.html_succeeds",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_with_index.html_succeeds/Status_code",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_with_index.html_succeeds/Header_Cache-Control",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_with_index.html_succeeds/Header_X-Ipfs-Path",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_with_index.html_succeeds/Header_X-Ipfs-Roots",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_with_index.html_succeeds/Header_Etag",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_file_succeeds",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_file_succeeds/Status_code",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_file_succeeds/Header_Cache-Control",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_file_succeeds/Header_X-Ipfs-Path",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_file_succeeds/Header_X-Ipfs-Roots",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_file_succeeds/Header_Etag",
   "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_as_DAG-JSON_succeeds",
   "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_as_DAG-JSON_succeeds/Status_code",
   "TestGatewayCache/GET_for_%2Fipfs%2F_unixfs_dir_as_DAG-JSON_succeeds/Header_Cache-Control",
@@ -461,7 +481,39 @@
   "TestGatewayCache/GET_for_%2Fipfs%2F_file_with_matching_weak_Etag_in_If-None-Match_returns_304_Not_Modified/Status_code",
   "TestGatewayCache/GET_for_%2Fipfs%2F_file_with_wildcard_Etag_in_If-None-Match_returns_304_Not_Modified",
   "TestGatewayCache/GET_for_%2Fipfs%2F_file_with_wildcard_Etag_in_If-None-Match_returns_304_Not_Modified/Status_code",
+  "TestGatewayCache/DirIndex_etag_is_based_on_xxhash%28.%2Fassets%2Fdir-index-html%29%2C_so_we_need_to_fetch_it_dynamically",
+  "TestGatewayCache/DirIndex_etag_is_based_on_xxhash%28.%2Fassets%2Fdir-index-html%29%2C_so_we_need_to_fetch_it_dynamically/Status_code",
+  "TestGatewayCache/DirIndex_etag_is_based_on_xxhash%28.%2Fassets%2Fdir-index-html%29%2C_so_we_need_to_fetch_it_dynamically/Header_Etag",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_dir_listing_with_matching_strong_Etag_in_If-None-Match_returns_304_Not_Modified",
+  "TestGatewayCache/GET_for_%2Fipfs%2F_dir_listing_with_matching_strong_Etag_in_If-None-Match_returns_304_Not_Modified/Status_code",
   "TestGatewayCacheWithIPNS",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_0",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_0/Status_code",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_0/Header_X-Ipfs-Path",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_0/Header_X-Ipfs-Roots",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_0/Header_Etag",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_1",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_1/Check_0",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_listing_succeeds/Check_1/Check_1",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_0",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_0/Status_code",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_0/Header_X-Ipfs-Path",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_0/Header_X-Ipfs-Roots",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_0/Header_Etag",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_1",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_1/Check_0",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_with_index.html_succeeds/Check_1/Check_1",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_0",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_0/Status_code",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_0/Header_X-Ipfs-Path",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_0/Header_X-Ipfs-Roots",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_0/Header_Etag",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_1",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_1/Check_0",
+  "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_file_succeeds/Check_1/Check_1",
   "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_as_DAG-JSON_succeeds",
   "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_as_DAG-JSON_succeeds/Check_0",
   "TestGatewayCacheWithIPNS/GET_for_%2Fipns%2F_unixfs_dir_as_DAG-JSON_succeeds/Check_0/Status_code",