Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,476 changes: 705 additions & 771 deletions package-lock.json

Large diffs are not rendered by default.

32 changes: 17 additions & 15 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,41 +39,43 @@
"@microlabs/otel-cf-workers": "^1.0.0-rc.48",
"@opentelemetry/api": "^1.9.0",
"@opentelemetry/sdk-trace-base": "^1.27.0",
"@storacha/indexing-service-client": "^2.1.0",
"@storacha/capabilities": "^1.7.0",
"@storacha/indexing-service-client": "^2.6.4",
"@types/node": "^24.9.1",
"@ucanto/client": "^9.0.1",
"@ucanto/principal": "^9.0.1",
"@ucanto/transport": "^9.1.1",
"@web3-storage/blob-fetcher": "^3.0.0",
"@web3-storage/capabilities": "^17.4.1",
"@ucanto/core": "^10.4.0",
"@ucanto/interface": "^10.3.0",
"@ucanto/principal": "^9.0.2",
"@ucanto/server": "^10.2.0",
"@ucanto/transport": "^9.2.0",
"@ucanto/validator": "^9.1.0",
"@web3-storage/blob-fetcher": "^4.1.0",
"@web3-storage/gateway-lib": "^5.2.2",
"dagula": "^8.0.0",
"http-range-parse": "^1.0.0",
"lnmap": "^2.0.0",
"multiformats": "^13.0.1"
},
"devDependencies": {
"@cloudflare/workers-types": "^4.20231218.0",
"@storacha/cli": "^1.0.1",
"@storacha/client": "^1.0.5",
"@storacha/cli": "^1.6.2",
"@storacha/client": "^1.8.2",
"@types/chai": "^5.0.0",
"@types/mocha": "^10.0.9",
"@types/node-fetch": "^2.6.11",
"@types/sinon": "^17.0.3",
"@web3-storage/content-claims": "^5.0.0",
"@web3-storage/public-bucket": "^1.4.0",
"@web3-storage/upload-client": "^16.1.1",
"carstream": "^2.1.0",
"carstream": "^2.2.0",
"chai": "^5.1.1",
"esbuild": "^0.24.0",
"files-from-path": "^0.2.6",
"miniflare": "^3.20240909.5",
"miniflare": "^4.20251011.1",
"mocha": "^10.7.3",
"multipart-byte-range": "^3.0.1",
"sinon": "^19.0.2",
"standard": "^17.1.0",
"standard": "^17.1.2",
"tree-kill": "^1.2.2",
"typescript": "^5.6.3",
"wrangler": "^3.90.0"
"typescript": "^5.8.3",
"wrangler": "^4.45.0"
},
"standard": {
"ignore": [
Expand Down
2 changes: 1 addition & 1 deletion scripts/delegate-serve.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import sade from 'sade'
import { Schema } from '@ucanto/core'
import { getClient } from '@storacha/cli/lib.js'
import { Space } from '@web3-storage/capabilities'
import { Space } from '@storacha/capabilities'

const MailtoDID =
/** @type {import('@ucanto/validator').StringSchema<`did:mailto:${string}:${string}`, unknown>} */ (
Expand Down
6 changes: 3 additions & 3 deletions src/middleware/withAuthorizedSpace.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Verifier } from '@ucanto/principal'
import { ok, access, Unauthorized } from '@ucanto/validator'
import { HttpError } from '@web3-storage/gateway-lib/util'
import * as serve from '../capabilities/serve.js'
import { SpaceDID } from '@web3-storage/capabilities/utils'
import { SpaceDID } from '@storacha/capabilities/utils'

/**
* @import * as Ucanto from '@ucanto/interface'
Expand Down Expand Up @@ -100,9 +100,9 @@ export function withAuthorizedSpace (handler) {
* authorizing delegations in the
* {@link DelegationsStorageContext.delegationsStorage}.
*
* @param {import('@web3-storage/capabilities/types').SpaceDID} space
* @param {import('@storacha/capabilities/types').SpaceDID} space
* @param {AuthTokenContext & DelegationsStorageContext & GatewayIdentityContext} ctx
* @returns {Promise<Ucanto.Result<{space: import('@web3-storage/capabilities/types').SpaceDID, delegationProofs: Ucanto.Delegation[]}, Ucanto.Failure>>}
* @returns {Promise<Ucanto.Result<{space: import('@storacha/capabilities/types').SpaceDID, delegationProofs: Ucanto.Delegation[]}, Ucanto.Failure>>}
*/
const authorize = async (space, ctx) => {
// Look up delegations that might authorize us to serve the content.
Expand Down
2 changes: 1 addition & 1 deletion src/middleware/withAuthorizedSpace.types.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import * as Ucanto from '@ucanto/interface'
import { Context as MiddlewareContext } from '@web3-storage/gateway-lib'
import { SpaceDID } from '@web3-storage/capabilities/types'
import { SpaceDID } from '@storacha/capabilities/types'
export interface DelegationProofsContext extends MiddlewareContext {
/**
* The delegation proofs to use for the egress record
Expand Down
17 changes: 10 additions & 7 deletions src/middleware/withCarBlockHandler.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import httpRangeParse from 'http-range-parse'
import { base58btc } from 'multiformats/bases/base58'

/**
* @import { R2Bucket, KVNamespace, RateLimit } from '@cloudflare/workers-types'
* @import {
* IpfsUrlContext,
* Middleware,
Expand Down Expand Up @@ -36,11 +35,13 @@ export function withCarBlockHandler (handler) {
// if not CAR codec, or if trusted gateway format has been requested...
const formatParam = searchParams.get('format')
const acceptHeader = request.headers.get('Accept')
if (dataCid.code !== CAR_CODE ||
if (
dataCid.code !== CAR_CODE ||
formatParam === 'car' ||
acceptHeader === 'application/vnd.ipld.car' ||
formatParam === 'raw' ||
acceptHeader === 'application/vnd.ipld.raw') {
acceptHeader === 'application/vnd.ipld.raw'
) {
return handler(request, env, ctx) // pass to other handlers
}

Expand Down Expand Up @@ -132,15 +133,17 @@ export async function handleCarBlock (request, env, ctx) {
}
headers.set('Content-Length', contentLength.toString())

// @ts-expect-error ReadableStream types incompatible
return new Response(obj.body.pipeThrough(new FixedLengthStream(contentLength)), { status, headers })
return new Response(
obj.body.pipeThrough(new FixedLengthStream(contentLength)),
{ status, headers }
)
}

/** @param {import('multiformats').UnknownLink} cid */
const toCARKey = cid => `${cid}/${cid}.car`
const toCARKey = (cid) => `${cid}/${cid}.car`

/** @param {import('multiformats').MultihashDigest} digest */
const toBlobKey = digest => {
const toBlobKey = (digest) => {
const mhStr = base58btc.encode(digest.bytes)
return `${mhStr}/${mhStr}.blob`
}
Expand Down
1 change: 0 additions & 1 deletion src/middleware/withCarBlockHandler.types.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { Environment as MiddlewareEnvironment } from '@web3-storage/gateway-lib'
import { R2Bucket } from '@cloudflare/workers-types'

export interface Environment extends MiddlewareEnvironment {
CARPARK: R2Bucket
Expand Down
1 change: 0 additions & 1 deletion src/middleware/withCarParkFetch.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import {
Environment as MiddlewareEnvironment,
Context as MiddlewareContext,
} from '@web3-storage/gateway-lib'
import { R2Bucket } from '@cloudflare/workers-types'

export interface CarParkFetchEnvironment extends MiddlewareEnvironment {
CARPARK: R2Bucket
Expand Down
1 change: 0 additions & 1 deletion src/middleware/withContentClaimsDagula.types.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import { KVNamespace } from '@cloudflare/workers-types'
import { Environment as MiddlewareEnvironment } from '@web3-storage/gateway-lib'

export interface Environment extends MiddlewareEnvironment {
Expand Down
58 changes: 37 additions & 21 deletions src/middleware/withDelegationsStorage.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,16 @@ import { ok, error, Failure } from '@ucanto/server'
* Middleware<DelegationsStorageContext, DelegationsStorageContext, DelegationsStorageEnvironment>
* )}
*/
export const withDelegationsStorage = (handler) => async (request, env, ctx) => {
if (env.FF_DELEGATIONS_STORAGE_ENABLED !== 'true') {
return handler(request, env, ctx)
export const withDelegationsStorage =
(handler) => async (request, env, ctx) => {
if (env.FF_DELEGATIONS_STORAGE_ENABLED !== 'true') {
return handler(request, env, ctx)
}
return handler(request, env, {
...ctx,
delegationsStorage: createStorage(env)
})
}
return handler(request, env, {
...ctx,
delegationsStorage: createStorage(env)
})
}

/**
* @param {DelegationsStorageEnvironment} env
Expand All @@ -34,35 +35,46 @@ function createStorage (env) {
/**
* Finds the delegation proofs for the given space
*
* @param {import('@web3-storage/capabilities/types').SpaceDID} space
* @param {import('@storacha/capabilities/types').SpaceDID} space
* @returns {Promise<Ucanto.Result<Ucanto.Delegation<Ucanto.Capabilities>[], DelegationNotFound | Ucanto.Failure>>}
*/
find: async (space) => {
/** @type {Ucanto.Delegation<Ucanto.Capabilities>[]} */
const delegations = []
const result = await env.CONTENT_SERVE_DELEGATIONS_STORE.list({ prefix: space })
await Promise.all(result.keys.map(async (key) => {
const delegation = await env.CONTENT_SERVE_DELEGATIONS_STORE.get(key.name, 'arrayBuffer')
if (delegation) {
const d = await Delegation.extract(new Uint8Array(delegation))
if (d.ok) delegations.push(d.ok)
else console.error('error while extracting delegation', d.error)
}
}))
const result = await env.CONTENT_SERVE_DELEGATIONS_STORE.list({
prefix: space
})
await Promise.all(
result.keys.map(async (key) => {
const delegation = await env.CONTENT_SERVE_DELEGATIONS_STORE.get(
key.name,
'arrayBuffer'
)
if (delegation) {
const d = await Delegation.extract(new Uint8Array(delegation))
if (d.ok) delegations.push(d.ok)
else console.error('error while extracting delegation', d.error)
}
})
)
return ok(delegations)
},

/**
* Stores the delegation proofs for the given space.
* If the delegation has an expiration, it will be stored with an expiration time in seconds since unix epoch.
*
* @param {import('@web3-storage/capabilities/types').SpaceDID} space
* @param {import('@storacha/capabilities/types').SpaceDID} space
* @param {Ucanto.Delegation<Ucanto.Capabilities>} delegation
* @returns {Promise<Ucanto.Result<Ucanto.Unit, StoreOperationFailed | Ucanto.Failure>>}
*/
store: async (space, delegation) => {
let options = {}
if (delegation.expiration && delegation.expiration > 0 && delegation.expiration !== Infinity) {
if (
delegation.expiration &&
delegation.expiration > 0 &&
delegation.expiration !== Infinity
) {
// expire the key-value pair when the delegation expires (seconds since epoch)
options = { expiration: delegation.expiration }
}
Expand All @@ -74,7 +86,11 @@ function createStorage (env) {
}

try {
await env.CONTENT_SERVE_DELEGATIONS_STORE.put(`${space}:${delegation.cid.toString()}`, value.ok.buffer, options)
await env.CONTENT_SERVE_DELEGATIONS_STORE.put(
`${space}:${delegation.cid.toString()}`,
value.ok.buffer,
options
)
return ok({})
} catch (/** @type {any} */ err) {
const message = `error while storing delegation for space ${space}`
Expand Down
7 changes: 3 additions & 4 deletions src/middleware/withDelegationsStorage.types.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import * as Ucanto from '@ucanto/interface'
import { Environment as MiddlewareEnvironment, Context as MiddlewareContext } from '@web3-storage/gateway-lib'
import { KVNamespace } from '@cloudflare/workers-types'
import { GatewayIdentityContext } from './withGatewayIdentity.types.js'
import { StoreOperationFailed, DelegationNotFound } from './withDelegationsStorage.js'
import { SpaceDID } from '@web3-storage/capabilities/types'
import { SpaceDID } from '@storacha/capabilities/types'

export interface DelegationsStorageEnvironment extends MiddlewareEnvironment {
CONTENT_SERVE_DELEGATIONS_STORE: KVNamespace
Expand All @@ -20,7 +19,7 @@ export interface DelegationsStorage {
/**
* Finds the delegation proofs for the given space
*
* @param {import('@web3-storage/capabilities/types').SpaceDID} space
* @param {import('@storacha/capabilities/types').SpaceDID} space
* @returns {Promise<Ucanto.Result<Ucanto.Delegation<Ucanto.Capabilities>[], DelegationNotFound | Ucanto.Failure>>}
*/
find: (
Expand All @@ -30,7 +29,7 @@ export interface DelegationsStorage {
/**
* Stores the delegation proofs for the given space
*
* @param {import('@web3-storage/capabilities/types').SpaceDID} space
* @param {import('@storacha/capabilities/types').SpaceDID} space
* @param {Ucanto.Delegation<Ucanto.Capabilities>} delegation
* @returns {Promise<Ucanto.Result<Ucanto.Unit, StoreOperationFailed | Ucanto.Failure>>}
*/
Expand Down
5 changes: 2 additions & 3 deletions src/middleware/withEgressClient.js
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import * as UCantoClient from '@ucanto/client'
import * as CAR from '@ucanto/transport/car'
import { SpaceDID } from '@web3-storage/capabilities/utils'
import { SpaceDID } from '@storacha/capabilities/utils'
import { HTTP } from '@ucanto/transport'
import { Space } from '@web3-storage/capabilities'
import { Space } from '@storacha/capabilities'
import { DID } from '@ucanto/core'

/**
Expand Down Expand Up @@ -46,7 +46,6 @@ async function create (env, ctx) {
*/
record: async (space, resource, bytes, servedAt) =>
record(space, resource, bytes, servedAt, env, ctx)

}
}

Expand Down
35 changes: 21 additions & 14 deletions src/middleware/withLocator.js
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,24 @@ export function withLocator (handler) {
span.setAttribute('legacyRequest', legacyRequest)
}

const client = !legacyRequest && useIndexingService
? new Client({
serviceURL: env.INDEXING_SERVICE_URL
? new URL(env.INDEXING_SERVICE_URL)
: undefined
})
: new ContentClaimsClient({
serviceURL: env.CONTENT_CLAIMS_SERVICE_URL
? new URL(env.CONTENT_CLAIMS_SERVICE_URL)
: undefined,
carpark: !legacyRequest ? env.CARPARK : undefined,
carparkPublicBucketURL: !legacyRequest && env.CARPARK_PUBLIC_BUCKET_URL ? new URL(env.CARPARK_PUBLIC_BUCKET_URL) : undefined
})
const client =
!legacyRequest && useIndexingService
? new Client({
serviceURL: env.INDEXING_SERVICE_URL
? new URL(env.INDEXING_SERVICE_URL)
: undefined
})
: new ContentClaimsClient({
serviceURL: env.CONTENT_CLAIMS_SERVICE_URL
? new URL(env.CONTENT_CLAIMS_SERVICE_URL)
: undefined,
// @ts-expect-error not yet sure how to square with types in libraries
carpark: !legacyRequest ? env.CARPARK : undefined,
carparkPublicBucketURL:
!legacyRequest && env.CARPARK_PUBLIC_BUCKET_URL
? new URL(env.CARPARK_PUBLIC_BUCKET_URL)
: undefined
})

const locator = Locator.create({ client })
return handler(request, env, { ...ctx, locator })
Expand All @@ -64,7 +69,9 @@ function isIndexingServiceEnabled (request, env) {
const withIndexingServicesArg = new URL(request.url).searchParams
.getAll('ff')
.includes('indexing-service')
const probability = env.FF_RAMP_UP_PROBABILITY ? Number(env.FF_RAMP_UP_PROBABILITY) : 0
const probability = env.FF_RAMP_UP_PROBABILITY
? Number(env.FF_RAMP_UP_PROBABILITY)
: 0
const withIndexerEnabled = Math.random() * 100 <= probability
return withIndexingServicesArg || withIndexerEnabled
}
1 change: 0 additions & 1 deletion src/middleware/withLocator.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import {
Environment as MiddlewareEnvironment,
Context as MiddlewareContext,
} from '@web3-storage/gateway-lib'
import { R2Bucket } from '@cloudflare/workers-types'

export interface LocatorEnvironment extends MiddlewareEnvironment {
CONTENT_CLAIMS_SERVICE_URL?: string
Expand Down
5 changes: 3 additions & 2 deletions src/middleware/withRateLimit.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { RATE_LIMIT_EXCEEDED } from '../constants.js'

/**
* @import { Middleware } from '@web3-storage/gateway-lib'
* @import { R2Bucket, KVNamespace, RateLimit } from '@cloudflare/workers-types'
* @import {
* Environment,
* Context,
Expand Down Expand Up @@ -112,7 +111,9 @@ async function getTokenMetadata (env, authToken, ctx) {
const tokenMetadata = await locateTokenMetadata(authToken)
if (tokenMetadata) {
// NOTE: non-blocking call to the auth token metadata cache
ctx.waitUntil(env.AUTH_TOKEN_METADATA.put(authToken, encode(tokenMetadata)))
ctx.waitUntil(
env.AUTH_TOKEN_METADATA.put(authToken, encode(tokenMetadata))
)
return tokenMetadata
}

Expand Down
1 change: 0 additions & 1 deletion src/middleware/withRateLimit.types.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { CID } from '@web3-storage/gateway-lib/handlers'
import { IpfsUrlContext, Environment as MiddlewareEnvironment } from '@web3-storage/gateway-lib'
import { KVNamespace, RateLimit } from '@cloudflare/workers-types'
import { RATE_LIMIT_EXCEEDED } from '../constants.js'

export interface Environment extends MiddlewareEnvironment {
Expand Down
Loading