fix: script to generate proof + ff telemetry

Author: fforbeck

scripts/delegate-serve.js

@@ -1,62 +1,73 @@
 import sade from 'sade'
 import { getClient } from '@web3-storage/w3cli/lib.js'
-import * as ed25519 from '@ucanto/principal/ed25519'
 import { Space } from '@web3-storage/capabilities'
 
-const cli = sade('delegate-serve.js [space] [token]')
+const cli = sade('delegate-serve.js [space] [token] [accountDID] [gatewayDID]')
 
 cli
+  .option('--space', 'The space DID to delegate. If not provided, a new space will be created.')
+  .option('--token', 'The auth token to use. If not provided, the delegation will not be authenticated.')
+  .option('--accountDID', 'The account DID to use when creating a new space.')
+  .option('--gatewayDID', 'The gateway DID to use when delegating the space/content/serve capability. Defaults to did:web:staging.w3s.link.')
   .describe(
     `Delegates ${Space.contentServe.can} to the Gateway for a test space generated by the script, with an optional auth token. Outputs a base64url string suitable for the stub_delegation query parameter. Pipe the output to pbcopy or similar for the quickest workflow. If the GATEWAY_PRINCIPAL_KEY environment variable is not set, a new key pair will be generated.`
   )
-  .action(async (space, token) => {
+  .action(async (space, token, accountDID, gatewayDID, options) => {
+    const { space: spaceOption, token: tokenOption, accountDID: accountDIDOption, gatewayDID: gatewayDIDOption } = options
+    space = spaceOption || undefined
+    token = tokenOption || undefined
+    accountDID = accountDIDOption || undefined
+    gatewayDID = gatewayDIDOption || 'did:web:staging.w3s.link'
     const client = await getClient()
 
-    let newSpace
+    let spaceDID
     let proofs = []
     if (!space) {
-      newSpace = await client.createSpace('test')
+      const provider = /** @type {`did:web:${string}`} */ (client.defaultProvider())
+      const account = client.accounts()[accountDID]
+      const newSpace = await client.agent.createSpace('test')
+      const provision = await account.provision(newSpace.did(), { provider })
+      if (provision.error) throw provision.error
+      await newSpace.save()
       const authProof = await newSpace.createAuthorization(client.agent)
-      await client.addSpace(authProof)
       proofs = [authProof]
+      spaceDID = newSpace.did()
     } else {
-      newSpace = space
+      client.addSpace(space)
+      spaceDID = space
       proofs = client.proofs([
         {
           can: Space.contentServe.can,
-          with: newSpace.did(),
+          with: spaceDID,
         }
       ])
     }
 
-    const signer =
-      process.env.GATEWAY_PRINCIPAL_KEY
-        ? ed25519.Signer.parse(process.env.GATEWAY_PRINCIPAL_KEY)
-        : await ed25519.Signer.generate()
+    /** @type {import('@ucanto/client').Principal<`did:${string}:${string}`>} */
+    const gatewayIdentity = {
+      did: () => gatewayDID,
+    }
 
-    const gatewayIdentity = signer.withDID('did:web:w3s.link')
-    const delegation = await Space.contentServe.delegate({
-      issuer: client.agent.issuer,
-      audience: gatewayIdentity,
-      with: newSpace.did(),
+    // @ts-expect-error - The client still needs to be updated to support the capability type
+    const delegation = await client.createDelegation(gatewayIdentity, [Space.contentServe.can], {
       expiration: Infinity,
-      proofs
+      proofs,
     })
-    
+
     await client.capability.access.delegate({
       delegations: [delegation],
     })
-    
+
     const carResult = await delegation.archive()
     if (carResult.error) throw carResult.error
     const base64Url = Buffer.from(carResult.ok).toString('base64url')
     process.stdout.write(`Agent Proofs: ${proofs.flatMap(p => p.capabilities).map(c => `${c.can} with ${c.with}`).join('\n')}\n`)
     process.stdout.write(`Issuer: ${client.agent.issuer.did()}\n`)
     process.stdout.write(`Audience: ${gatewayIdentity.did()}\n`)
-    process.stdout.write(`Space: ${newSpace.did()}\n`)
+    process.stdout.write(`Space: ${spaceDID}\n`)
     process.stdout.write(`Token: ${token ?? 'none'}\n`)
     process.stdout.write(`Delegation: ${delegation.capabilities.map(c => `${c.can} with ${c.with}`).join('\n')}\n`)
-    process.stdout.write(`Stubs: stub_space=${newSpace.did()}&stub_delegation=${base64Url}&authToken=${token ?? ''}\n`)
+    process.stdout.write(`Stubs: stub_space=${spaceDID}&stub_delegation=${base64Url}&authToken=${token ?? ''}\n`)
   })
 
 cli.parse(process.argv)

src/index.js

@@ -27,7 +27,7 @@ import {
   withDelegationStubs
 } from './middleware/index.js'
 import { instrument } from '@microlabs/otel-cf-workers'
-// import { NoopSpanProcessor } from '@opentelemetry/sdk-trace-base'
+import { NoopSpanProcessor } from '@opentelemetry/sdk-trace-base'
 import { withEgressClient } from './middleware/withEgressClient.js'
 import { withGatewayIdentity } from './middleware/withGatewayIdentity.js'
 
@@ -46,7 +46,7 @@ import { withGatewayIdentity } from './middleware/withGatewayIdentity.js'
 
 const handler = {
   /** @type {Handler<Context, Environment>} */
-  fetch (request, env, ctx) {
+  fetch(request, env, ctx) {
     console.log(request.method, request.url)
     const middleware = composeMiddleware(
       // Prepare the Context
@@ -61,7 +61,7 @@ const handler = {
       withLocator,
       withGatewayIdentity,
       withDelegationStubs,
-      
+
       // Rate-limit requests
       withRateLimit,
 
@@ -91,7 +91,7 @@ const handler = {
  * @param {Environment} env
  * @param {*} _trigger
  */
-function config (env, _trigger) {
+function config(env, _trigger) {
   if (env.HONEYCOMB_API_KEY) {
     return {
       exporter: {
@@ -102,17 +102,19 @@ function config (env, _trigger) {
     }
   }
   return {
-    // spanProcessors: new NoopSpanProcessor(),
+    spanProcessors: new NoopSpanProcessor(),
     service: { name: 'freeway' },
   }
 }
 
-export default handler //instrument(handler, config)
+export default process.env.FF_TELEMETRY_ENABLED === 'true'
+  ? instrument(handler, config)
+  : handler
 
 /**
  * @type {Middleware<BlockContext & UnixfsContext & IpfsUrlContext, BlockContext & UnixfsContext & IpfsUrlContext, Environment>}
  */
-export function withFormatRawHandler (handler) {
+export function withFormatRawHandler(handler) {
   return async (request, env, ctx) => {
     const { headers } = request
     const { searchParams } = ctx
@@ -130,7 +132,7 @@ export function withFormatRawHandler (handler) {
 /**
  * @type {Middleware<DagContext & IpfsUrlContext, DagContext & IpfsUrlContext, Environment>}
  */
-export function withFormatCarHandler (handler) {
+export function withFormatCarHandler(handler) {
   return async (request, env, ctx) => {
     const { headers } = request
     const { searchParams } = ctx

src/middleware/index.js

@@ -9,5 +9,5 @@ export { withEgressTracker } from './withEgressTracker.js'
 export { withEgressClient } from './withEgressClient.js'
 export { withDelegationStubs } from './withDelegationStubs.js'
 
-export const GATEWAY_DID = 'did:web:w3s.link'
-export const UPLOAD_DID = 'did:web:web3.storage'
+export const GATEWAY_DID = 'did:web:staging.w3s.link'
+export const UPLOAD_DID = 'did:web:staging.web3.storage'

src/middleware/withAuthorizedSpace.types.ts

@@ -6,7 +6,13 @@ export interface DelegationsStorageContext
   extends MiddlewareContext,
     GatewayIdentityContext {
   delegationsStorage: DelegationsStorage
-  delegationProofs?: Ucanto.Delegation[]
+  /**
+   * The delegation proofs to use for the egress record
+   * The proofs must be valid for the space and the owner of the space
+   * must have delegated the right to the Gateway to serve content and record egress traffic.
+   * The `space/content/serve/*` capability must be granted to the Gateway Web DID.
+   */
+  delegationProofs: Ucanto.Delegation[]
 }
 
 export interface SpaceContext extends MiddlewareContext {

src/middleware/withDelegationStubs.js

@@ -50,6 +50,7 @@ export const withDelegationStubs = (handler) => async (request, env, ctx) => {
   return handler(request, env, {
     ...ctx,
     delegationsStorage: { find: async () => ({ ok: stubDelegations }) },
+    delegationProofs: [], // Delegation proofs are set by withAuthorizedSpace handler
     locator:
       stubSpace && isDIDKey(stubSpace)
         ? {

src/middleware/withEgressClient.js

@@ -31,7 +31,7 @@ export function withEgressClient(handler) {
  * @returns {Promise<import('./withEgressClient.types.js').EgressClient>}
  */
 async function create(env, ctx) {
-    return {
+  return {
     /**
      * Records the egress bytes for the given resource.
      *
@@ -58,7 +58,7 @@ async function connect(serverUrl, principal) {
   const connection = await UCantoClient.connect({
     id: principal,
     codec: CAR.outbound,
-    channel: HTTP.open({ url: new URL(serverUrl)}),
+    channel: HTTP.open({ url: new URL(serverUrl) }),
   })
 
   return connection
@@ -76,7 +76,7 @@ async function connect(serverUrl, principal) {
  * @returns {Promise<void>}
  */
 async function record(space, resource, bytes, servedAt, env, ctx) {
-  const uploadServicePrincipal = DID.parse('did:web:staging.web3.storage')
+  const uploadServicePrincipal = DID.parse('did:web:staging.web3.storage') // TODO move to env var
   const connection = await connect(env.UPLOAD_API_URL, uploadServicePrincipal)
 
   const invocation = Space.egressRecord.invoke({
@@ -88,8 +88,7 @@ async function record(space, resource, bytes, servedAt, env, ctx) {
       bytes,
       servedAt: Math.floor(servedAt.getTime() / 1000)
     },
-    proofs: ctx.delegationProofs ? ctx.delegationProofs : [],
-
+    proofs: ctx.delegationProofs,
   })
   const res = await invocation.execute(connection)
   if (res.out.error) {

wrangler.toml

@@ -45,6 +45,7 @@ command = "npm run build"
 MAX_SHARDS = "825"
 FF_RATE_LIMITER_ENABLED = "false"
 FF_EGRESS_TRACKER_ENABLED = "false"
+FF_TELEMETRY_ENABLED = "true"
 CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage"
 CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link"
 
@@ -63,6 +64,7 @@ command = "npm run build"
 MAX_SHARDS = "825"
 FF_RATE_LIMITER_ENABLED = "false"
 FF_EGRESS_TRACKER_ENABLED = "false"
+FF_TELEMETRY_ENABLED = "false"
 CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage"
 CARPARK_PUBLIC_BUCKET_URL = "https://carpark-staging-0.r2.w3s.link"
 
@@ -77,6 +79,7 @@ r2_buckets = [
 DEBUG = "true"
 FF_RATE_LIMITER_ENABLED = "false"
 FF_EGRESS_TRACKER_ENABLED = "false"
+FF_TELEMETRY_ENABLED = "true"
 MAX_SHARDS = "120"
 CONTENT_CLAIMS_SERVICE_URL = "https://test.claims.web3.storage"
 
@@ -91,6 +94,7 @@ r2_buckets = [
 DEBUG = "true"
 FF_RATE_LIMITER_ENABLED = "false"
 FF_EGRESS_TRACKER_ENABLED = "false"
+FF_TELEMETRY_ENABLED = "true"
 CONTENT_CLAIMS_SERVICE_URL = "https://dev.claims.web3.storage"
 
 [env.fforbeck]
@@ -101,21 +105,26 @@ upload_source_maps = true
 # account_id = "9e46c5ddfefedb9bae5d81a0dd911e5a"
 # Company Account
 account_id = "fffa4b4363a7e5250af8357087263b3a"
+# r2_buckets = [
+#  { binding = "CARPARK", bucket_name = "carpark-fforbeck-0", preview_bucket_name = "carpark-fforbeck-preview-0" }
+# ]
 r2_buckets = [
-  { binding = "CARPARK", bucket_name = "carpark-fforbeck-0", preview_bucket_name = "carpark-fforbeck-preview-0" }
+  { binding = "CARPARK", bucket_name = "carpark-staging-0" }
 ]
 
 [env.fforbeck.vars]
 DEBUG = "true"
 # Feature Flags
 FF_RATE_LIMITER_ENABLED = "false"
 FF_EGRESS_TRACKER_ENABLED = "true"
+FF_TELEMETRY_ENABLED = "false"
 # DIDs
 GATEWAY_SERVICE_DID = "did:web:staging.w3s.link"
 UPLOAD_SERVICE_DID = "did:web:staging.web3.storage"
 # SERVICE URLs
 CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage"
 UPLOAD_API_URL = "https://staging.up.web3.storage"
+#UPLOAD_API_URL = "https://pr435.up.storacha.network"
 
 [[env.fforbeck.unsafe.bindings]]
 name = "RATE_LIMITER"
@@ -140,6 +149,7 @@ r2_buckets = [
 DEBUG = "true"
 FF_RATE_LIMITER_ENABLED = "false"
 FF_EGRESS_TRACKER_ENABLED = "false"
+FF_TELEMETRY_ENABLED = "true"
 CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage"
 CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link"
 
@@ -156,6 +166,7 @@ r2_buckets = [
 DEBUG = "true"
 FF_RATE_LIMITER_ENABLED = "true"
 FF_EGRESS_TRACKER_ENABLED = "true"
+FF_TELEMETRY_ENABLED = "true"
 CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage"
 
 [[env.integration.unsafe.bindings]]