feat: Use Indexing Service when feature flag is present

package.json

@@ -38,10 +38,11 @@
     "@microlabs/otel-cf-workers": "^1.0.0-rc.48",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/sdk-trace-base": "^1.27.0",
+    "@storacha/indexing-service-client": "^2.0.0",
     "@ucanto/client": "^9.0.1",
     "@ucanto/principal": "^9.0.1",
     "@ucanto/transport": "^9.1.1",
-    "@web3-storage/blob-fetcher": "^2.3.1",
+    "@web3-storage/blob-fetcher": "^2.4.3",
     "@web3-storage/capabilities": "^17.4.1",
     "@web3-storage/gateway-lib": "^5.1.2",
     "dagula": "^8.0.0",
@@ -51,17 +52,18 @@
   },
   "devDependencies": {
     "@cloudflare/workers-types": "^4.20231218.0",
+    "@storacha/cli": "^1.0.1",
+    "@storacha/client": "^1.0.5",
     "@types/chai": "^5.0.0",
     "@types/mocha": "^10.0.9",
     "@types/node-fetch": "^2.6.11",
     "@types/sinon": "^17.0.3",
     "@web3-storage/content-claims": "^5.0.0",
     "@web3-storage/public-bucket": "^1.1.0",
     "@web3-storage/upload-client": "^16.1.1",
-    "@web3-storage/w3cli": "^7.8.2",
     "carstream": "^2.1.0",
     "chai": "^5.1.1",
-    "esbuild": "^0.18.20",
+    "esbuild": "^0.24.0",
     "files-from-path": "^0.2.6",
     "miniflare": "^3.20240909.5",
     "mocha": "^10.7.3",
@@ -70,7 +72,7 @@
     "standard": "^17.1.0",
     "tree-kill": "^1.2.2",
     "typescript": "^5.6.3",
-    "wrangler": "^3.86.1"
+    "wrangler": "^3.90.0"
   },
   "standard": {
     "ignore": [

scripts/delegate-serve.d.ts

@@ -1,4 +1,4 @@
-declare module '@web3-storage/w3cli/lib.js' {
-  import { Client } from '@web3-storage/w3up-client'
+declare module '@storacha/cli/lib.js' {
+  import { Client } from '@storacha/client'
   export declare function getClient(): Promise<Client>
 }

scripts/delegate-serve.js

@@ -1,73 +1,137 @@
 import sade from 'sade'
-import { getClient } from '@web3-storage/w3cli/lib.js'
+import { Schema } from '@ucanto/core'
+import { getClient } from '@storacha/cli/lib.js'
 import { Space } from '@web3-storage/capabilities'
+import * as serve from '../src/capabilities/serve.js'
 
-const cli = sade('delegate-serve.js [space] [token] [accountDID] [gatewayDID]')
+const MailtoDID =
+  /** @type {import('@ucanto/validator').StringSchema<`did:mailto:${string}:${string}`, unknown>} */ (
+    Schema.text({ pattern: /^did:mailto:.*:.*$/ })
+  )
 
-cli
-  .option('--space', 'The space DID to delegate. If not provided, a new space will be created.')
-  .option('--token', 'The auth token to use. If not provided, the delegation will not be authenticated.')
+sade('delegate-serve.js [space]')
+  .option(
+    '--token',
+    'The auth token to use. If not provided, the delegation will not be authenticated.'
+  )
   .option('--accountDID', 'The account DID to use when creating a new space.')
-  .option('--gatewayDID', 'The gateway DID to use when delegating the space/content/serve capability. Defaults to did:web:staging.w3s.link.')
+  .option(
+    '--gatewayDID',
+    'The gateway DID to use when delegating the space/content/serve capability. Defaults to did:web:staging.w3s.link.'
+  )
   .describe(
-    `Delegates ${Space.contentServe.can} to the Gateway for a test space generated by the script, with an optional auth token. Outputs a base64url string suitable for the stub_delegation query parameter. Pipe the output to pbcopy or similar for the quickest workflow.`
+    `Delegates ${Space.contentServe.can} to the Gateway for a test space generated by the script, with an optional auth token. Outputs a base64url string suitable for the stub_delegation query parameter.`
   )
-  .action(async (space, token, accountDID, gatewayDID, options) => {
-    const { space: spaceOption, token: tokenOption, accountDID: accountDIDOption, gatewayDID: gatewayDIDOption } = options
-    space = spaceOption || undefined
-    token = tokenOption || undefined
-    accountDID = accountDIDOption || undefined
-    gatewayDID = gatewayDIDOption || 'did:web:staging.w3s.link'
-    const client = await getClient()
+  .action(
+    /**
+     * @param {string} [space]
+     * @param {object} [options]
+     * @param {string} [options.token]
+     * @param {string} [options.accountDID]
+     * @param {string} [options.gatewayDID]
+     */
+    async (
+      space,
+      { token, accountDID, gatewayDID = 'did:web:staging.w3s.link' } = {}
+    ) => {
+      const client = await getClient()
+
+      space ??= await createSpace(client, accountDID)
 
-    let spaceDID
-    let proofs = []
-    if (!space) {
-      const provider = /** @type {`did:web:${string}`} */ (client.defaultProvider())
-      const account = client.accounts()[accountDID]
-      const newSpace = await client.agent.createSpace('test')
-      const provision = await account.provision(newSpace.did(), { provider })
-      if (provision.error) throw provision.error
-      await newSpace.save()
-      const authProof = await newSpace.createAuthorization(client.agent)
-      proofs = [authProof]
-      spaceDID = newSpace.did()
-    } else {
-      client.addSpace(space)
-      spaceDID = space
-      proofs = client.proofs([
+      if (!Schema.did({}).is(space)) {
+        throw new Error(`Invalid space DID: ${space}`)
+      }
+
+      const proofs = client.proofs([
         {
           can: Space.contentServe.can,
-          with: spaceDID
+          with: space
         }
       ])
-    }
 
-    /** @type {import('@ucanto/client').Principal<`did:${string}:${string}`>} */
-    const gatewayIdentity = {
-      did: () => gatewayDID
-    }
+      if (proofs.length === 0) {
+        throw new Error(
+          `No proofs found. Are you authorized to ${serve.star.can} ${space}?`
+        )
+      }
+
+      if (!Schema.did({}).is(gatewayDID)) {
+        throw new Error(`Invalid gateway DID: ${gatewayDID}`)
+      }
 
-    // @ts-expect-error - The client still needs to be updated to support the capability type
-    const delegation = await client.createDelegation(gatewayIdentity, [Space.contentServe.can], {
-      expiration: Infinity,
-      proofs
-    })
+      const gatewayIdentity = {
+        did: () => gatewayDID
+      }
 
-    await client.capability.access.delegate({
-      delegations: [delegation]
-    })
+      // NOTE: This type assertion is wrong. It's a hack to let us use this
+      // ability. `client.createDelegation` currently only accepts abilities it
+      // knows about. That should probably be expanded, but this little script
+      // isn't going to be the reason to go change that, as it involves updating
+      // multiple packages.
+      const ability = /** @type {"*"} */ (Space.contentServe.can)
+
+      client.setCurrentSpace(space)
+      const delegation = await client.createDelegation(
+        gatewayIdentity,
+        [ability],
+        {
+          expiration: Infinity,
+          proofs
+        }
+      )
+
+      await client.capability.access.delegate({
+        delegations: [delegation]
+      })
+
+      const carResult = await delegation.archive()
+      if (carResult.error) throw carResult.error
+      const base64Url = Buffer.from(carResult.ok).toString('base64url')
+      process.stdout.write(
+        `Agent Proofs: ${proofs
+          .flatMap((p) => p.capabilities)
+          .map((c) => `${c.can} with ${c.with}`)
+          .join('\n')}\n`
+      )
+      process.stdout.write(`Issuer: ${client.agent.issuer.did()}\n`)
+      process.stdout.write(`Audience: ${gatewayIdentity.did()}\n`)
+      process.stdout.write(`Space: ${space}\n`)
+      process.stdout.write(`Token: ${token ?? 'none'}\n`)
+      process.stdout.write(
+        `Delegation: ${delegation.capabilities
+          .map((c) => `${c.can} with ${c.with}`)
+          .join('\n')}\n`
+      )
+      process.stdout.write(
+        `Stubs: stub_space=${space}&stub_delegation=${base64Url}&authToken=${
+          token ?? ''
+        }\n`
+      )
+    }
+  )
+  .parse(process.argv)
 
-    const carResult = await delegation.archive()
-    if (carResult.error) throw carResult.error
-    const base64Url = Buffer.from(carResult.ok).toString('base64url')
-    process.stdout.write(`Agent Proofs: ${proofs.flatMap(p => p.capabilities).map(c => `${c.can} with ${c.with}`).join('\n')}\n`)
-    process.stdout.write(`Issuer: ${client.agent.issuer.did()}\n`)
-    process.stdout.write(`Audience: ${gatewayIdentity.did()}\n`)
-    process.stdout.write(`Space: ${spaceDID}\n`)
-    process.stdout.write(`Token: ${token ?? 'none'}\n`)
-    process.stdout.write(`Delegation: ${delegation.capabilities.map(c => `${c.can} with ${c.with}`).join('\n')}\n`)
-    process.stdout.write(`Stubs: stub_space=${spaceDID}&stub_delegation=${base64Url}&authToken=${token ?? ''}\n`)
-  })
+/**
+ * @param {import('@storacha/client').Client} client
+ * @param {string} [accountDID]
+ */
+async function createSpace (client, accountDID) {
+  const provider = client.defaultProvider()
+  if (!Schema.did({ method: 'web' }).is(provider)) {
+    throw new Error(`Invalid provider DID: ${provider}`)
+  }
+  if (!accountDID) {
+    throw new Error('Must provide an account DID to create a space')
+  }
 
-cli.parse(process.argv)
+  if (!MailtoDID.is(accountDID)) {
+    throw new Error(`Invalid account DID: ${accountDID}`)
+  }
+  const account = client.accounts()[accountDID]
+  const newSpace = await client.agent.createSpace('test')
+  const provision = await account.provision(newSpace.did(), { provider })
+  if (provision.error) throw provision.error
+  await newSpace.save()
+  await newSpace.createAuthorization(client.agent)
+  return newSpace.did()
+}

src/middleware/withAuthorizedSpace.js

@@ -5,11 +5,11 @@ import * as serve from '../capabilities/serve.js'
 
 /**
  * @import * as Ucanto from '@ucanto/interface'
- * @import { Locator } from '@web3-storage/blob-fetcher'
  * @import { IpfsUrlContext, Middleware } from '@web3-storage/gateway-lib'
  * @import { LocatorContext } from './withLocator.types.js'
  * @import { AuthTokenContext } from './withAuthToken.types.js'
- * @import { SpaceContext, DelegationsStorageContext } from './withAuthorizedSpace.types.js'
+ * @import { SpaceContext, DelegationsStorageContext, DelegationProofsContext } from './withAuthorizedSpace.types.js'
+ * @import { GatewayIdentityContext } from './withGatewayIdentity.types.js'
  */
 
 /**
@@ -21,9 +21,8 @@ import * as serve from '../capabilities/serve.js'
  * @throws {Error} If the locator fails in any other way.
  * @type {(
  *   Middleware<
- *     LocatorContext & IpfsUrlContext & AuthTokenContext & DelegationsStorageContext & SpaceContext,
- *     LocatorContext & IpfsUrlContext & AuthTokenContext & DelegationsStorageContext,
- *     {}
+ *     LocatorContext & IpfsUrlContext & AuthTokenContext & GatewayIdentityContext & DelegationProofsContext & DelegationsStorageContext & SpaceContext,
+ *     LocatorContext & IpfsUrlContext & AuthTokenContext & GatewayIdentityContext & DelegationProofsContext & DelegationsStorageContext
  *   >
  * )}
  */
@@ -68,14 +67,23 @@ export function withAuthorizedSpace (handler) {
         ...ctx,
         space: selectedSpace,
         delegationProofs,
-        locator: spaceScopedLocator(locator, selectedSpace)
+        locator: locator.scopeToSpaces([selectedSpace])
       })
     } catch (error) {
       // If all Spaces failed to authorize, throw the first error.
       if (
         error instanceof AggregateError &&
         error.errors.every((e) => e instanceof Unauthorized)
       ) {
+        if (env.DEBUG === 'true') {
+          console.log(
+            [
+              'Authorization Failures:',
+              ...error.errors.map((e) => e.message)
+            ].join('\n\n')
+          )
+        }
+
         throw new HttpError('Not Found', { status: 404, cause: error })
       } else {
         throw error
@@ -90,7 +98,7 @@ export function withAuthorizedSpace (handler) {
  * {@link DelegationsStorageContext.delegationsStorage}.
  *
  * @param {Ucanto.DID} space
- * @param {AuthTokenContext & DelegationsStorageContext} ctx
+ * @param {AuthTokenContext & DelegationsStorageContext & GatewayIdentityContext} ctx
  * @returns {Promise<Ucanto.Result<{space: Ucanto.DID, delegationProofs: Ucanto.Delegation[]}, Ucanto.Failure>>}
  */
 const authorize = async (space, ctx) => {
@@ -134,26 +142,3 @@ const authorize = async (space, ctx) => {
     }
   }
 }
-
-/**
- * Wraps a {@link Locator} and locates content only from a specific Space.
- *
- * @param {Locator} locator
- * @param {Ucanto.DID} space
- * @returns {Locator}
- */
-const spaceScopedLocator = (locator, space) => ({
-  locate: async (digest) => {
-    const locateResult = await locator.locate(digest)
-    if (locateResult.error) {
-      return locateResult
-    } else {
-      return {
-        ok: {
-          ...locateResult.ok,
-          site: locateResult.ok.site.filter((site) => site.space === space)
-        }
-      }
-    }
-  }
-})

src/middleware/withAuthorizedSpace.types.ts

@@ -1,11 +1,11 @@
 import * as Ucanto from '@ucanto/interface'
 import { Context as MiddlewareContext } from '@web3-storage/gateway-lib'
-import { GatewayIdentityContext as GatewayIdentityContext } from './withGatewayIdentity.types.js'
 
-export interface DelegationsStorageContext
-  extends MiddlewareContext,
-    GatewayIdentityContext {
+export interface DelegationsStorageContext extends MiddlewareContext {
   delegationsStorage: DelegationsStorage
+}
+
+export interface DelegationProofsContext extends MiddlewareContext {
   /**
    * The delegation proofs to use for the egress record
    * The proofs must be valid for the space and the owner of the space

src/middleware/withDelegationStubs.js

@@ -1,7 +1,6 @@
-import { Delegation } from '@ucanto/core'
+import { Delegation, Schema } from '@ucanto/core'
 
 /**
- * @import * as Ucanto from '@ucanto/interface'
  * @import {
  *   Middleware,
  *   Context as MiddlewareContext
@@ -22,7 +21,7 @@ import { Delegation } from '@ucanto/core'
  *
  * @type {(
  *   Middleware<
- *     MiddlewareContext & LocatorContext & DelegationsStorageContext,
+ *     MiddlewareContext & LocatorContext & GatewayIdentityContext & DelegationsStorageContext,
  *     MiddlewareContext & LocatorContext & GatewayIdentityContext,
  *     {}
  *   >
@@ -50,32 +49,9 @@ export const withDelegationStubs = (handler) => async (request, env, ctx) => {
   return handler(request, env, {
     ...ctx,
     delegationsStorage: { find: async () => ({ ok: stubDelegations }) },
-    delegationProofs: [], // Delegation proofs are set by withAuthorizedSpace handler
     locator:
-      stubSpace && isDIDKey(stubSpace)
-        ? {
-            locate: async (digest, options) => {
-              const locateResult = await ctx.locator.locate(digest, options)
-              if (locateResult.error) return locateResult
-              return {
-                ok: {
-                  ...locateResult.ok,
-                  site: locateResult.ok.site.map((site) => ({
-                    ...site,
-                    space: stubSpace
-                  }))
-                }
-              }
-            }
-          }
+      stubSpace && Schema.did({ method: 'key' }).is(stubSpace)
+        ? ctx.locator.scopeToSpaces([stubSpace])
         : ctx.locator
   })
 }
-
-/**
- * True if the given string is a `key:` DID.
- *
- * @param {string} did
- * @returns {did is Ucanto.DIDKey}
- */
-const isDIDKey = (did) => did.startsWith('did:key:')