From ac2f45c9e2edfebaceb00095b050bc1021a053e5 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Tue, 11 Nov 2025 15:44:43 -0300 Subject: [PATCH 01/13] wip --- package.json | 1 + scripts/mk-validator-proof.js | 92 ++++++++++++++++++ src/bindings.d.ts | 1 + src/middleware/withAuthorizedSpace.js | 23 +++-- .../withDelegationsStorage.types.ts | 3 +- src/middleware/withRateLimit.types.ts | 1 + src/middleware/withUcanInvocationHandler.js | 4 +- .../withUcanInvocationHandler.types.ts | 1 + src/server/index.js | 77 ++++++++++++++- src/server/service.js | 10 +- src/server/utils.js | 96 ++++++++++++------- wrangler.toml | 36 +++++-- 12 files changed, 284 insertions(+), 61 deletions(-) create mode 100644 scripts/mk-validator-proof.js diff --git a/package.json b/package.json index db34e57..93228fb 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ }, "devDependencies": { "@storacha/blob-index": "^1.2.4", + "@ipld/dag-ucan": "^3.4.5", "@storacha/cli": "^1.6.2", "@storacha/client": "^1.8.2", "@storacha/upload-client": "^1.3.6", diff --git a/scripts/mk-validator-proof.js b/scripts/mk-validator-proof.js new file mode 100644 index 0000000..c45f0b1 --- /dev/null +++ b/scripts/mk-validator-proof.js @@ -0,0 +1,92 @@ +/** + * Create a "ucan/attest" delegation allowing the gateway to validate + * attestations issued by the upload-service. + * + * This generates the GATEWAY_VALIDATOR_PROOF environment variable value. + * + * Usage: node scripts/mk-validator-proof.js + * + * Example (staging): + * node scripts/mk-validator-proof.js \ + * did:web:staging.up.storacha.network \ + * MgCZT5J+...your-key-here... \ + * did:web:staging.w3s.link + * + * Example (production): + * node scripts/mk-validator-proof.js \ + * did:web:up.storacha.network \ + * MgCZT5J+...your-key-here... \ + * did:web:w3s.link + */ +import * as DID from '@ipld/dag-ucan/did' +import { CAR, delegate } from '@ucanto/core' +import * as ed25519 from '@ucanto/principal/ed25519' +import { base64 } from 'multiformats/bases/base64' +import { identity } from 'multiformats/hashes/identity' +import * as Link from 'multiformats/link' + +// CORRECT DIRECTION (staging): +// - issuer should be did:web:staging.up.storacha.network (upload-service) +// - audience should be did:web:staging.w3s.link (gateway) +// - can should be 'ucan/attest' +// - with should be issuer.did() (i.e. did:web:staging.up.storacha.network) +// The private key must be the upload-service private key. This makes the +// gateway trust attestations issued by the upload-service. + + +const uploadServiceDIDWeb = process.argv[2] +const uploadServicePrivateKey = process.argv[3] +const gatewayDIDWeb = process.argv[4] + +if (!uploadServiceDIDWeb || !uploadServicePrivateKey || !gatewayDIDWeb) { + console.error('Error: Missing required arguments') + console.error('Usage: node scripts/mk-validator-proof.js ') + console.error('') + console.error('Example (staging):') + console.error(' node scripts/mk-validator-proof.js \\') + console.error(' did:web:staging.up.storacha.network \\') + console.error(' MgCZT5J+...your-key-here... \\') + console.error(' did:web:staging.w3s.link') + process.exit(1) +} + +console.log(`Upload Service DID: ${uploadServiceDIDWeb}`) +console.log(`Upload Service Private Key: ${uploadServicePrivateKey.slice(0, 7)}...${uploadServicePrivateKey.slice(-7)}`) +console.log(`Gateway DID: ${gatewayDIDWeb}`) +console.log('') + +const issuer = ed25519 + .parse(uploadServicePrivateKey) + .withDID(DID.parse(uploadServiceDIDWeb).did()) +const audience = DID.parse(gatewayDIDWeb) + +// Note: variable names are confusing - "uploadService" is actually the issuer (gateway in our case) +// and "gateway" is actually the audience (upload service in our case) +// The 'with' should be the issuer's DID per colleague's instructions +const delegation = await delegate({ + issuer: issuer, + audience: audience, + capabilities: [{ can: 'ucan/attest', with: issuer.did() }], + expiration: Infinity +}) + +console.log('✅ Delegation created:') +console.log(` Issuer: ${issuer.did()}`) +console.log(` Audience: ${audience.did()}`) +console.log(` Capability: ucan/attest with ${issuer.did()}`) +console.log('') + +const res = await delegation.archive() +if (res.error) { + console.error('❌ Error archiving delegation:', res.error) + throw res.error +} + +const proof = Link.create(CAR.code, identity.digest(res.ok)).toString(base64) + +console.log('✅ Validator proof generated successfully!') +console.log('') +console.log('Add this to your environment variables:') +console.log('') +console.log('GATEWAY_VALIDATOR_PROOF=' + proof) +console.log('') diff --git a/src/bindings.d.ts b/src/bindings.d.ts index 09411f3..8295cc6 100644 --- a/src/bindings.d.ts +++ b/src/bindings.d.ts @@ -26,4 +26,5 @@ export interface Environment HONEYCOMB_API_KEY: string FF_TELEMETRY_ENABLED: string TELEMETRY_RATIO: string + GATEWAY_VALIDATOR_PROOF?: string } diff --git a/src/middleware/withAuthorizedSpace.js b/src/middleware/withAuthorizedSpace.js index 55a5019..812543c 100644 --- a/src/middleware/withAuthorizedSpace.js +++ b/src/middleware/withAuthorizedSpace.js @@ -1,8 +1,10 @@ import { Verifier } from '@ucanto/principal' import { ok, access, fail, Unauthorized } from '@ucanto/validator' +import { resolveDIDKey, getValidatorProofs } from '../server/index.js' import { HttpError } from '@web3-storage/gateway-lib/util' import * as serve from '../capabilities/serve.js' import { SpaceDID } from '@storacha/capabilities/utils' + /** * Extracts a SpaceDID string from various space object formats. @@ -51,9 +53,9 @@ function extractSpaceDID (space) { * @import * as Ucanto from '@ucanto/interface' * @import { IpfsUrlContext, Middleware } from '@web3-storage/gateway-lib' * @import { LocatorContext } from './withLocator.types.js' - * @import { AuthTokenContext } from './withAuthToken.types.js' + * @import { AuthTokenContext, Environment } from './withAuthToken.types.js' * @import { SpaceContext } from './withAuthorizedSpace.types.js' - * @import { DelegationsStorageContext } from './withDelegationsStorage.types.js' + * @import { DelegationsStorageContext, DelegationsStorageEnvironment } from './withDelegationsStorage.types.js' * @import { GatewayIdentityContext } from './withGatewayIdentity.types.js' * @import { DelegationProofsContext } from './withAuthorizedSpace.types.js' */ @@ -117,10 +119,9 @@ export function withAuthorizedSpace (handler) { // First space to successfully authorize is the one we'll use. const { space: selectedSpace, delegationProofs } = await Promise.any( spaces.map(async (space) => { - const result = await authorize(SpaceDID.from(space.toString()), ctx) - if (result.error) { - throw result.error - } + // @ts-ignore + const result = await authorize(SpaceDID.from(space), ctx, env) + if (result.error) throw result.error return result.ok }) ) @@ -168,9 +169,10 @@ export function withAuthorizedSpace (handler) { * * @param {import('@storacha/capabilities/types').SpaceDID} space * @param {AuthTokenContext & DelegationsStorageContext & GatewayIdentityContext} ctx + * @param {import('./withRateLimit.types.js').Environment} env * @returns {Promise>} */ -const authorize = async (space, ctx) => { +const authorize = async (space, ctx, env) => { // Look up delegations that might authorize us to serve the content. const relevantDelegationsResult = await ctx.delegationsStorage.find(space) if (relevantDelegationsResult.error) { @@ -196,12 +198,15 @@ const authorize = async (space, ctx) => { proofs: delegationProofs }) .delegate() - - // Validate the invocation. + + // Load validator proofs and validate the invocation + const validatorProofs = await getValidatorProofs(env) const accessResult = await access(invocation, { capability: serve.transportHttp, authority: ctx.gatewayIdentity, principal: Verifier, + proofs: validatorProofs, + resolveDIDKey, validateAuthorization: () => ok({}) }) diff --git a/src/middleware/withDelegationsStorage.types.ts b/src/middleware/withDelegationsStorage.types.ts index e6d7821..cc80669 100644 --- a/src/middleware/withDelegationsStorage.types.ts +++ b/src/middleware/withDelegationsStorage.types.ts @@ -7,6 +7,7 @@ import { SpaceDID } from '@storacha/capabilities/types' export interface DelegationsStorageEnvironment extends MiddlewareEnvironment { CONTENT_SERVE_DELEGATIONS_STORE: KVNamespace FF_DELEGATIONS_STORAGE_ENABLED: string + GATEWAY_VALIDATOR_PROOF?: string } export interface DelegationsStorageContext @@ -35,6 +36,6 @@ export interface DelegationsStorage { */ store: ( space: SpaceDID, - delegation: Ucanto.Delegation + delegation: Ucanto.Delegation, ) => Promise> } diff --git a/src/middleware/withRateLimit.types.ts b/src/middleware/withRateLimit.types.ts index f13888d..dd0fef2 100644 --- a/src/middleware/withRateLimit.types.ts +++ b/src/middleware/withRateLimit.types.ts @@ -6,6 +6,7 @@ export interface Environment extends MiddlewareEnvironment { RATE_LIMITER: RateLimit AUTH_TOKEN_METADATA: KVNamespace FF_RATE_LIMITER_ENABLED: string + GATEWAY_VALIDATOR_PROOF?: string } export interface TokenMetadata { diff --git a/src/middleware/withUcanInvocationHandler.js b/src/middleware/withUcanInvocationHandler.js index 5361deb..85cab7c 100644 --- a/src/middleware/withUcanInvocationHandler.js +++ b/src/middleware/withUcanInvocationHandler.js @@ -22,8 +22,8 @@ export function withUcanInvocationHandler (handler) { return handler(request, env, ctx) } - const service = ctx.service ?? createService(ctx) - const server = ctx.server ?? createServer(ctx, service) + const service = ctx.service ?? await createService(ctx, env) + const server = ctx.server ?? await createServer(ctx, service, env) const { headers, body, status } = await server.request({ body: new Uint8Array(await request.arrayBuffer()), diff --git a/src/middleware/withUcanInvocationHandler.types.ts b/src/middleware/withUcanInvocationHandler.types.ts index a313234..2aad1f4 100644 --- a/src/middleware/withUcanInvocationHandler.types.ts +++ b/src/middleware/withUcanInvocationHandler.types.ts @@ -4,6 +4,7 @@ import { DelegationsStorageContext } from './withDelegationsStorage.types.js' import { Service } from '../server/api.types.js' import * as Server from '@ucanto/server' export interface Environment extends MiddlewareEnvironment { + GATEWAY_VALIDATOR_PROOF?: string } export interface Context diff --git a/src/server/index.js b/src/server/index.js index 7b1ecae..45234ed 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -1,5 +1,65 @@ import * as Server from '@ucanto/server' import * as CAR from '@ucanto/transport/car' +import { DIDResolutionError } from '@ucanto/validator' +import * as Proof from '@storacha/client/proof' + +/** + * Known did:web to did:key mappings for signature verification + * @type {Record<`did:${string}:${string}`, `did:key:${string}`>} + */ +const knownWebDIDs = { + // Production + 'did:web:up.storacha.network': 'did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi', + 'did:web:web3.storage': 'did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi', + 'did:web:w3s.link': 'did:key:z6Mkha3NLZ38QiZXsUHKRHecoumtha3LnbYEL21kXYBFXvo5', + + // Staging + 'did:web:staging.up.storacha.network': 'did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn', + 'did:web:staging.web3.storage': 'did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn', + 'did:web:staging.w3s.link': 'did:key:z6MkqK1d4thaCEXSGZ6EchJw3tDPhQriwynWDuR55ayATMNf', +} + +/** + * Resolves did:web DIDs to their corresponding did:key DIDs + * @param {import('@ucanto/interface').DID} did + */ +export const resolveDIDKey = async (did) => { + if (knownWebDIDs[did]) { + const didKey = /** @type {`did:key:${string}`} */ (knownWebDIDs[did]) + return Server.ok([didKey]) // Return array of did:keys + } + return Server.error(new DIDResolutionError(did)) +} + +/** + * @type {import('@ucanto/interface').Delegation[]} + */ +let cachedValidatorProofs + +/** + * Loads validator proofs from environment variable. + * These proofs allow the gateway to validate ucan/attest delegations. + * + * @param {{ GATEWAY_VALIDATOR_PROOF?: string }} env + * @returns {Promise} + */ +export const getValidatorProofs = async (env) => { + if (cachedValidatorProofs) { + return cachedValidatorProofs + } + cachedValidatorProofs = [] + if (env.GATEWAY_VALIDATOR_PROOF) { + try { + const proof = await Proof.parse(env.GATEWAY_VALIDATOR_PROOF) + const delegation = /** @type {import('@ucanto/interface').Delegation} */ (proof) + console.log(`Gateway validator proof loaded: [issuer: ${delegation.issuer.did()}, audience: ${delegation.audience.did()}]`) + cachedValidatorProofs = [delegation] + } catch (error) { + console.error('Failed to parse GATEWAY_VALIDATOR_PROOF:', error) + } + } + return cachedValidatorProofs +} /** * Creates a UCAN server. @@ -7,14 +67,27 @@ import * as CAR from '@ucanto/transport/car' * @template T * @param {import('../middleware/withUcanInvocationHandler.types.js').Context} ctx * @param {import('./api.types.js').Service} service + * @param {{ GATEWAY_VALIDATOR_PROOF?: string }} env */ -export function createServer (ctx, service) { +export async function createServer (ctx, service, env) { + const proofs = await getValidatorProofs(env) + console.log('Server validator proofs loaded:', proofs.length) + if (proofs.length > 0) { + console.log('First proof details:', { + issuer: proofs[0].issuer.did(), + audience: proofs[0].audience.did(), + capabilities: proofs[0].capabilities.map(c => ({ can: c.can, with: c.with })), + cid: proofs[0].cid.toString() + }) + } return Server.create({ id: ctx.gatewaySigner, codec: CAR.inbound, service, catch: err => console.error(err), // TODO: wire into revocations - validateAuthorization: () => ({ ok: {} }) + validateAuthorization: () => ({ ok: {} }), + resolveDIDKey, + proofs, }) } diff --git a/src/server/service.js b/src/server/service.js index bb3ebab..c34e8f8 100644 --- a/src/server/service.js +++ b/src/server/service.js @@ -17,11 +17,12 @@ export function createService (ctx) { access: { delegate: UcantoServer.provideAdvanced({ capability: AccessCapabilities.delegate, - audience: Schema.did({ method: 'web' }), + audience: Schema.did({ method: 'web' }).or(Schema.did({ method: 'key' })), handler: async ({ capability, invocation, context }) => { - const result = extractContentServeDelegations( + console.log('extracting delegations: ', capability) + const result = await extractContentServeDelegations( capability, - invocation.proofs + invocation ) if (result.error) { console.error('error while extracting delegation', result.error) @@ -29,6 +30,7 @@ export function createService (ctx) { } const delegations = result.ok + console.log('executing claim for delegations: ', delegations) const validationResults = await Promise.all( delegations.map(async (delegation) => { const validationResult = await claim( @@ -36,7 +38,7 @@ export function createService (ctx) { [delegation], { ...context, - authority: ctx.gatewayIdentity + authority: ctx.gatewayIdentity, } ) if (validationResult.error) { diff --git a/src/server/utils.js b/src/server/utils.js index 225b352..58eb7e8 100644 --- a/src/server/utils.js +++ b/src/server/utils.js @@ -1,49 +1,79 @@ import { Space as SpaceCapabilities } from '@storacha/capabilities' import { InvalidDelegation } from '../middleware/withDelegationsStorage.js' +import { Delegation } from '@ucanto/core' /** * Checks if the space/content/serve/* delegation is for the gateway and it is not expired. * * @param {import('@ucanto/interface').InferInvokedCapability} capability - The capability to validate - * @param {import('@ucanto/interface').Proof[]} proofs - The proofs to validate + * @param {import('@ucanto/interface').Invocation} invocation - The invocation containing attached blocks */ -export const extractContentServeDelegations = (capability, proofs) => { - const nbDelegations = new Set(Object.values(capability.nb.delegations)) - if (nbDelegations.size !== 1) { - return { - error: new InvalidDelegation( - 'nb.delegations has more than one delegation' - ) - } +export const extractContentServeDelegations = async (capability, invocation) => { + console.log('extracting delegations', capability, invocation) + /** @type {Map} */ + const blocks = new Map() + for (const block of invocation.iterateIPLDBlocks()) { + blocks.set(block.cid.toString(), block) } - const delegations = [] - for (const delegationLink of nbDelegations) { - const proofDelegations = proofs.flatMap((proof) => - 'capabilities' in proof ? [proof] : [] - ) - const delegationProof = proofDelegations.find((p) => - delegationLink.equals(p.cid) - ) - if (!delegationProof) { - return { - error: new InvalidDelegation( - `delegation not found in proofs: ${delegationLink}` - ) + + const delegations = await Promise.all( + Object.values(capability.nb.delegations).map(async (cid) => { + const block = blocks.get(cid.toString()) + if (!block) { + throw new Error(`Block not found for delegation CID: ${cid}`) } - } + + // Try to extract delegation from CAR archive + try { + const delegation = await Delegation.extract(block.bytes) + if (delegation.error) { + throw delegation.error + } + return delegation.ok + } catch (error) { + // Fallback: treat as raw delegation block + const link = cid.link() + const cidv1 = link.version === 0 ? link.toV1() : link + return Delegation.create({ + root: { + cid: /** @type {import('@ucanto/interface').UCANLink} */ (cidv1), + bytes: block.bytes, + } + }) + } + }) + ) - if ( - !delegationProof.capabilities.some( - (c) => c.can === SpaceCapabilities.contentServe.can - ) - ) { - return { - error: new InvalidDelegation( - `delegation does not contain ${SpaceCapabilities.contentServe.can} capability` - ) + const validDelegations = delegations + .map((delegation) => { + // check for each capability if it is a content serve capability + if (delegation.capabilities.length === 0) { + return false } + const result = delegation.capabilities.some((c) => { + return SpaceCapabilities.contentServe.match({ + capability: /** @type {any} */ (c), + delegation, + }).ok + }) + return result + }) + .filter((r) => r) + + if (validDelegations.length === 0) { + return { + error: new InvalidDelegation('no valid delegations found') } - delegations.push(delegationProof) } + + // Only allow a single delegation reference for now + const nbDelegations = new Set(Object.values(capability.nb.delegations)) + if (nbDelegations.size !== 1) { + return { + error: new InvalidDelegation('nb.delegations has more than one delegation') + } + } + return { ok: delegations } } + diff --git a/wrangler.toml b/wrangler.toml index 60803d2..706ceed 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -166,16 +166,22 @@ account_id = "fffa4b4363a7e5250af8357087263b3a" # r2_buckets = [ # { binding = "CARPARK", bucket_name = "carpark-fforbeck-0", preview_bucket_name = "carpark-fforbeck-preview-0" } # ] -# r2_buckets = [ -# { binding = "CARPARK", bucket_name = "carpark-staging-0" } -# ] +#r2_buckets = [ +# { binding = "CARPARK", bucket_name = "carpark-prod-0", preview_bucket_name = "carpark-prod-0" } +#] +#kv_namespaces = [ +# { binding = "AUTH_TOKEN_METADATA", id = "b618bb05deb8493f944ef4a0f538030c" }, +# { binding = "CONTENT_SERVE_DELEGATIONS_STORE", id = "26cc47fec09749bb9ee42bc6407f9a9d" }, +# { binding = "DAGPB_CONTENT_CACHE", id = "3f0c253b90fc48c1b384f1563ede54f9" } +#] +# Copied from STAGING env r2_buckets = [ { binding = "CARPARK", bucket_name = "carpark-staging-0" } ] kv_namespaces = [ { binding = "AUTH_TOKEN_METADATA", id = "b618bb05deb8493f944ef4a0f538030c" }, { binding = "CONTENT_SERVE_DELEGATIONS_STORE", id = "99ae45f8b5b3478a9df09302c27e81a3" }, - { binding = "DAGPB_CONTENT_CACHE", id = "3f0c253b90fc48c1b384f1563ede54f9" } + { binding = "DAGPB_CONTENT_CACHE", id = "c70a74363e7a4f06ad39fa3022aab7c7" } ] [env.fforbeck.observability.logs] @@ -201,12 +207,15 @@ UPLOAD_SERVICE_DID = "did:web:staging.up.storacha.network" UPLOAD_API_URL = "https://staging.up.storacha.network" INDEXING_SERVICE_URL = "https://staging.indexer.storacha.network/" ### prod -#CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage" -#CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link" -#GATEWAY_SERVICE_DID = "did:web:w3s.link" -#UPLOAD_SERVICE_DID = "did:web:web3.storage" -#UPLOAD_API_URL = "https://up.web3.storage" -#INDEXING_SERVICE_URL = "https://indexer.storacha.network/" +CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage" +# CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link" +GATEWAY_SERVICE_DID = "did:web:staging.w3s.link" +UPLOAD_SERVICE_DID = "did:web:staging.web3.storage" +UPLOAD_API_URL = "https://staging.up.web3.storage" +# gateway validator proof (gateway -> gateway) +GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" +# up validator proof (gateway -> upload-service) +#GATEWAY_VALIDATOR_PROOF = "mAYIEAJgDOqJlcm9vdHOB2CpYJQABcRIgN620cPucqWQBOpxv6IdEa2h9qIjbLjuVeX5EjKZYntlndmVyc2lvbgGBAgFxEiCZ8EADJu8q8uWgve3nZa/+mzdDfX1IyQ4riBFs3lOiP6dhc1hE7aEDQNWyV/OoO65h7E1sJhiVj/PtTd5rKWmgWRjct8CvfSsoEILycg7GF5CdF7KR/oryw5mjIiYqm1CNmaZr1o7c2ANhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgjZGlkOndlYjpzdGFnaW5nLnVwLnN0b3JhY2hhLm5ldHdvcmtjYXVkVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjZXhw9mNpc3NYIZ0ad2ViOnN0YWdpbmcudXAuc3RvcmFjaGEubmV0d29ya2NwcmaAWQFxEiA3rbRw+5ypZAE6nG/oh0RraH2oiNsuO5V5fkSMplie2aFqdWNhbkAwLjkuMdgqWCUAAXESIJnwQAMm7yry5aC97edlr/6bN0N9fUjJDiuIEWzeU6I/" [[env.fforbeck.unsafe.bindings]] name = "RATE_LIMITER" @@ -218,6 +227,13 @@ simple = { limit = 5, period = 60 } queue = "egress-tracking-queue-staging" binding = "EGRESS_QUEUE" +[env.fforbeck.observability] +[env.fforbeck.observability.logs] +enabled = true +head_sampling_rate = 1 +invocation_logs = true +persist = false + [env.hannahhoward] name = "freeway-hannahhoward" workers_dev = true From 706722190a261ce3dd12bfbeb6196335cd2fa37b Mon Sep 17 00:00:00 2001 From: Hannah Howard Date: Mon, 17 Nov 2025 04:52:17 -0800 Subject: [PATCH 02/13] fix(service): properly handle attested delegations (#190) - use the full signer + did:web as the authority for the server - do not fail when multiple delegations are stored by content/serve - pass blocks with each stored delegation when serving - allow alternate authority on content serve delegations # Why - The first issue we're having is attestations must match the authority, and we were using the signer, as opposed to the identity, so the dids did not match - Second, we didn't support storing multiple delegations, which is problematic cause the `claim` check in the implementation of access/delegate will fail if you don't also give it the attestation - Third, we were not passing in attached blocks with the our delegations - Finally, it's never come up before, but who exactly is the "authority" on a storing of space/content/serve? Here, my belief is it should be the upload-service, as it's the ultimate arbiter of whether egress can be billed. I don't know if the upload service is properly checking each submitted content serve for validity, but if it is, its authority will be itself. So I added facilities to use the upload service as the authority -- of note, the authority does NOT require a private key, so don't worry we're just storing the upload service pub key in wrangler.toml --- scripts/mk-validator-proof.js | 5 +-- src/middleware/withAuthorizedSpace.js | 3 +- src/middleware/withGatewayIdentity.js | 6 ++- src/middleware/withGatewayIdentity.types.ts | 2 +- src/middleware/withUcanInvocationHandler.js | 15 +++++-- .../withUcanInvocationHandler.types.ts | 2 + src/server/index.js | 41 +++++++++++------ src/server/service.js | 44 ++++++++++--------- src/server/utils.js | 29 ++++++------ wrangler.toml | 13 +++--- 10 files changed, 97 insertions(+), 63 deletions(-) diff --git a/scripts/mk-validator-proof.js b/scripts/mk-validator-proof.js index c45f0b1..b92539f 100644 --- a/scripts/mk-validator-proof.js +++ b/scripts/mk-validator-proof.js @@ -33,7 +33,6 @@ import * as Link from 'multiformats/link' // The private key must be the upload-service private key. This makes the // gateway trust attestations issued by the upload-service. - const uploadServiceDIDWeb = process.argv[2] const uploadServicePrivateKey = process.argv[3] const gatewayDIDWeb = process.argv[4] @@ -64,8 +63,8 @@ const audience = DID.parse(gatewayDIDWeb) // and "gateway" is actually the audience (upload service in our case) // The 'with' should be the issuer's DID per colleague's instructions const delegation = await delegate({ - issuer: issuer, - audience: audience, + issuer, + audience, capabilities: [{ can: 'ucan/attest', with: issuer.did() }], expiration: Infinity }) diff --git a/src/middleware/withAuthorizedSpace.js b/src/middleware/withAuthorizedSpace.js index 812543c..cca9af8 100644 --- a/src/middleware/withAuthorizedSpace.js +++ b/src/middleware/withAuthorizedSpace.js @@ -4,7 +4,6 @@ import { resolveDIDKey, getValidatorProofs } from '../server/index.js' import { HttpError } from '@web3-storage/gateway-lib/util' import * as serve from '../capabilities/serve.js' import { SpaceDID } from '@storacha/capabilities/utils' - /** * Extracts a SpaceDID string from various space object formats. @@ -198,7 +197,7 @@ const authorize = async (space, ctx, env) => { proofs: delegationProofs }) .delegate() - + // Load validator proofs and validate the invocation const validatorProofs = await getValidatorProofs(env) const accessResult = await access(invocation, { diff --git a/src/middleware/withGatewayIdentity.js b/src/middleware/withGatewayIdentity.js index d3d8ac3..d2f8fa2 100644 --- a/src/middleware/withGatewayIdentity.js +++ b/src/middleware/withGatewayIdentity.js @@ -20,6 +20,10 @@ export function withGatewayIdentity (handler) { const gatewayIdentity = gatewaySigner.withDID( Schema.DID.from(env.GATEWAY_SERVICE_DID) ) - return handler(req, env, { ...ctx, gatewaySigner, gatewayIdentity }) + return handler(req, env, { + ...ctx, + gatewaySigner: gatewayIdentity, + gatewayIdentity + }) } } diff --git a/src/middleware/withGatewayIdentity.types.ts b/src/middleware/withGatewayIdentity.types.ts index 363bb58..228c918 100644 --- a/src/middleware/withGatewayIdentity.types.ts +++ b/src/middleware/withGatewayIdentity.types.ts @@ -8,6 +8,6 @@ export interface Environment extends MiddlewareEnvironment { } export interface GatewayIdentityContext extends MiddlewareContext { - gatewaySigner: EdSigner + gatewaySigner: Ucanto.Signer gatewayIdentity: Ucanto.Signer } diff --git a/src/middleware/withUcanInvocationHandler.js b/src/middleware/withUcanInvocationHandler.js index 85cab7c..3d9ff65 100644 --- a/src/middleware/withUcanInvocationHandler.js +++ b/src/middleware/withUcanInvocationHandler.js @@ -1,6 +1,7 @@ +import { ed25519 } from '@ucanto/principal' import { createServer } from '../server/index.js' import { createService } from '../server/service.js' - +import { Schema } from '@ucanto/core' /** * @import { Middleware } from '@web3-storage/gateway-lib' * @import { @@ -22,8 +23,16 @@ export function withUcanInvocationHandler (handler) { return handler(request, env, ctx) } - const service = ctx.service ?? await createService(ctx, env) - const server = ctx.server ?? await createServer(ctx, service, env) + const contentServeAuthority = + env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID + ? ed25519.Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( + Schema.DID.from(env.CONTENT_SERVE_AUTHORITY_DID) + ) + : ctx.gatewayIdentity + + const service = + ctx.service ?? (await createService(ctx, contentServeAuthority)) + const server = ctx.server ?? (await createServer(ctx, service, env)) const { headers, body, status } = await server.request({ body: new Uint8Array(await request.arrayBuffer()), diff --git a/src/middleware/withUcanInvocationHandler.types.ts b/src/middleware/withUcanInvocationHandler.types.ts index 2aad1f4..ed81217 100644 --- a/src/middleware/withUcanInvocationHandler.types.ts +++ b/src/middleware/withUcanInvocationHandler.types.ts @@ -5,6 +5,8 @@ import { Service } from '../server/api.types.js' import * as Server from '@ucanto/server' export interface Environment extends MiddlewareEnvironment { GATEWAY_VALIDATOR_PROOF?: string + CONTENT_SERVE_AUTHORITY_PUB_KEY?: string + CONTENT_SERVE_AUTHORITY_DID?: string } export interface Context diff --git a/src/server/index.js b/src/server/index.js index 45234ed..c3d694a 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -9,14 +9,20 @@ import * as Proof from '@storacha/client/proof' */ const knownWebDIDs = { // Production - 'did:web:up.storacha.network': 'did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi', - 'did:web:web3.storage': 'did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi', - 'did:web:w3s.link': 'did:key:z6Mkha3NLZ38QiZXsUHKRHecoumtha3LnbYEL21kXYBFXvo5', - + 'did:web:up.storacha.network': + 'did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi', + 'did:web:web3.storage': + 'did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi', + 'did:web:w3s.link': + 'did:key:z6Mkha3NLZ38QiZXsUHKRHecoumtha3LnbYEL21kXYBFXvo5', + // Staging - 'did:web:staging.up.storacha.network': 'did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn', - 'did:web:staging.web3.storage': 'did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn', - 'did:web:staging.w3s.link': 'did:key:z6MkqK1d4thaCEXSGZ6EchJw3tDPhQriwynWDuR55ayATMNf', + 'did:web:staging.up.storacha.network': + 'did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn', + 'did:web:staging.web3.storage': + 'did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn', + 'did:web:staging.w3s.link': + 'did:key:z6MkqK1d4thaCEXSGZ6EchJw3tDPhQriwynWDuR55ayATMNf' } /** @@ -26,7 +32,7 @@ const knownWebDIDs = { export const resolveDIDKey = async (did) => { if (knownWebDIDs[did]) { const didKey = /** @type {`did:key:${string}`} */ (knownWebDIDs[did]) - return Server.ok([didKey]) // Return array of did:keys + return Server.ok([didKey]) // Return array of did:keys } return Server.error(new DIDResolutionError(did)) } @@ -39,7 +45,7 @@ let cachedValidatorProofs /** * Loads validator proofs from environment variable. * These proofs allow the gateway to validate ucan/attest delegations. - * + * * @param {{ GATEWAY_VALIDATOR_PROOF?: string }} env * @returns {Promise} */ @@ -51,8 +57,12 @@ export const getValidatorProofs = async (env) => { if (env.GATEWAY_VALIDATOR_PROOF) { try { const proof = await Proof.parse(env.GATEWAY_VALIDATOR_PROOF) - const delegation = /** @type {import('@ucanto/interface').Delegation} */ (proof) - console.log(`Gateway validator proof loaded: [issuer: ${delegation.issuer.did()}, audience: ${delegation.audience.did()}]`) + const delegation = /** @type {import('@ucanto/interface').Delegation} */ ( + proof + ) + console.log( + `Gateway validator proof loaded: [issuer: ${delegation.issuer.did()}, audience: ${delegation.audience.did()}]` + ) cachedValidatorProofs = [delegation] } catch (error) { console.error('Failed to parse GATEWAY_VALIDATOR_PROOF:', error) @@ -76,7 +86,10 @@ export async function createServer (ctx, service, env) { console.log('First proof details:', { issuer: proofs[0].issuer.did(), audience: proofs[0].audience.did(), - capabilities: proofs[0].capabilities.map(c => ({ can: c.can, with: c.with })), + capabilities: proofs[0].capabilities.map((c) => ({ + can: c.can, + with: c.with + })), cid: proofs[0].cid.toString() }) } @@ -84,10 +97,10 @@ export async function createServer (ctx, service, env) { id: ctx.gatewaySigner, codec: CAR.inbound, service, - catch: err => console.error(err), + catch: (err) => console.error(err), // TODO: wire into revocations validateAuthorization: () => ({ ok: {} }), resolveDIDKey, - proofs, + proofs }) } diff --git a/src/server/service.js b/src/server/service.js index c34e8f8..5adc021 100644 --- a/src/server/service.js +++ b/src/server/service.js @@ -10,14 +10,17 @@ import { ok } from '@ucanto/client' /** * @template T * @param {import('../middleware/withUcanInvocationHandler.types.js').Context} ctx + * @param {import('@ucanto/interface').Verifier} contentServeAuthority * @returns {import('./api.types.js').Service} */ -export function createService (ctx) { +export function createService (ctx, contentServeAuthority) { return { access: { delegate: UcantoServer.provideAdvanced({ capability: AccessCapabilities.delegate, - audience: Schema.did({ method: 'web' }).or(Schema.did({ method: 'key' })), + audience: Schema.did({ method: 'web' }).or( + Schema.did({ method: 'key' }) + ), handler: async ({ capability, invocation, context }) => { console.log('extracting delegations: ', capability) const result = await extractContentServeDelegations( @@ -31,25 +34,26 @@ export function createService (ctx) { const delegations = result.ok console.log('executing claim for delegations: ', delegations) - const validationResults = await Promise.all( - delegations.map(async (delegation) => { - const validationResult = await claim( - SpaceCapabilities.contentServe, - [delegation], - { - ...context, - authority: ctx.gatewayIdentity, - } - ) - if (validationResult.error) { - console.error( - 'error while validating delegation', - validationResult.error - ) - return validationResult - } + const validationResult = await claim( + SpaceCapabilities.contentServe, + delegations, + { + ...context, + authority: contentServeAuthority + } + ) + if (validationResult.error) { + console.error( + 'error while validating delegation', + validationResult.error + ) + return validationResult + } - const space = capability.with + const space = capability.with + + const validationResults = await Promise.all( + delegations.map((delegation) => { return ctx.delegationsStorage.store(space, delegation) }) ) diff --git a/src/server/utils.js b/src/server/utils.js index 58eb7e8..800643e 100644 --- a/src/server/utils.js +++ b/src/server/utils.js @@ -8,21 +8,24 @@ import { Delegation } from '@ucanto/core' * @param {import('@ucanto/interface').InferInvokedCapability} capability - The capability to validate * @param {import('@ucanto/interface').Invocation} invocation - The invocation containing attached blocks */ -export const extractContentServeDelegations = async (capability, invocation) => { +export const extractContentServeDelegations = async ( + capability, + invocation +) => { console.log('extracting delegations', capability, invocation) /** @type {Map} */ const blocks = new Map() for (const block of invocation.iterateIPLDBlocks()) { blocks.set(block.cid.toString(), block) } - + const delegations = await Promise.all( Object.values(capability.nb.delegations).map(async (cid) => { const block = blocks.get(cid.toString()) if (!block) { throw new Error(`Block not found for delegation CID: ${cid}`) } - + // Try to extract delegation from CAR archive try { const delegation = await Delegation.extract(block.bytes) @@ -37,8 +40,9 @@ export const extractContentServeDelegations = async (capability, invocation) => return Delegation.create({ root: { cid: /** @type {import('@ucanto/interface').UCANLink} */ (cidv1), - bytes: block.bytes, - } + bytes: block.bytes + }, + blocks }) } }) @@ -53,7 +57,7 @@ export const extractContentServeDelegations = async (capability, invocation) => const result = delegation.capabilities.some((c) => { return SpaceCapabilities.contentServe.match({ capability: /** @type {any} */ (c), - delegation, + delegation }).ok }) return result @@ -67,13 +71,12 @@ export const extractContentServeDelegations = async (capability, invocation) => } // Only allow a single delegation reference for now - const nbDelegations = new Set(Object.values(capability.nb.delegations)) - if (nbDelegations.size !== 1) { - return { - error: new InvalidDelegation('nb.delegations has more than one delegation') - } - } + // const nbDelegations = new Set(Object.values(capability.nb.delegations)) + // if (nbDelegations.size !== 1) { + // return { + // error: new InvalidDelegation('nb.delegations has more than one delegation') + // } + // } return { ok: delegations } } - diff --git a/wrangler.toml b/wrangler.toml index 706ceed..472343e 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -176,12 +176,12 @@ account_id = "fffa4b4363a7e5250af8357087263b3a" #] # Copied from STAGING env r2_buckets = [ - { binding = "CARPARK", bucket_name = "carpark-staging-0" } + { binding = "CARPARK", bucket_name = "carpark-staging-0", preview_bucket_name = "carpark-staging-0" } ] kv_namespaces = [ - { binding = "AUTH_TOKEN_METADATA", id = "b618bb05deb8493f944ef4a0f538030c" }, - { binding = "CONTENT_SERVE_DELEGATIONS_STORE", id = "99ae45f8b5b3478a9df09302c27e81a3" }, - { binding = "DAGPB_CONTENT_CACHE", id = "c70a74363e7a4f06ad39fa3022aab7c7" } + { binding = "AUTH_TOKEN_METADATA", id = "b618bb05deb8493f944ef4a0f538030c", preview_id = "6a546b5fc21a423eb4ea07db2d611a91" }, + { binding = "CONTENT_SERVE_DELEGATIONS_STORE", id = "99ae45f8b5b3478a9df09302c27e81a3", preview_id = "ec5c429f8b1849a68d73dee7447b4e30" }, + { binding = "DAGPB_CONTENT_CACHE", id = "c70a74363e7a4f06ad39fa3022aab7c7", preview_id = "fcea72d8d8694b5c831736e1317e9208" } ] [env.fforbeck.observability.logs] @@ -213,10 +213,11 @@ GATEWAY_SERVICE_DID = "did:web:staging.w3s.link" UPLOAD_SERVICE_DID = "did:web:staging.web3.storage" UPLOAD_API_URL = "https://staging.up.web3.storage" # gateway validator proof (gateway -> gateway) -GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" +# GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" # up validator proof (gateway -> upload-service) #GATEWAY_VALIDATOR_PROOF = "mAYIEAJgDOqJlcm9vdHOB2CpYJQABcRIgN620cPucqWQBOpxv6IdEa2h9qIjbLjuVeX5EjKZYntlndmVyc2lvbgGBAgFxEiCZ8EADJu8q8uWgve3nZa/+mzdDfX1IyQ4riBFs3lOiP6dhc1hE7aEDQNWyV/OoO65h7E1sJhiVj/PtTd5rKWmgWRjct8CvfSsoEILycg7GF5CdF7KR/oryw5mjIiYqm1CNmaZr1o7c2ANhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgjZGlkOndlYjpzdGFnaW5nLnVwLnN0b3JhY2hhLm5ldHdvcmtjYXVkVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjZXhw9mNpc3NYIZ0ad2ViOnN0YWdpbmcudXAuc3RvcmFjaGEubmV0d29ya2NwcmaAWQFxEiA3rbRw+5ypZAE6nG/oh0RraH2oiNsuO5V5fkSMplie2aFqdWNhbkAwLjkuMdgqWCUAAXESIJnwQAMm7yry5aC97edlr/6bN0N9fUjJDiuIEWzeU6I/" - +CONTENT_SERVE_AUTHORITY_PUB_KEY = "did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn" +CONTENT_SERVE_AUTHORITY_DID = "did:web:staging.up.storacha.network" [[env.fforbeck.unsafe.bindings]] name = "RATE_LIMITER" type = "ratelimit" From 57ea3306a16fd814ea0bfda2e56bd97fadd07dfd Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Tue, 18 Nov 2025 09:28:37 -0300 Subject: [PATCH 03/13] fix(wip): content retrieval --- src/middleware/withAuthorizedSpace.js | 34 +++++++++++++++++++++++++-- src/middleware/withRateLimit.types.ts | 2 ++ wrangler.toml | 29 ++++++++++++----------- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/src/middleware/withAuthorizedSpace.js b/src/middleware/withAuthorizedSpace.js index cca9af8..28b8a99 100644 --- a/src/middleware/withAuthorizedSpace.js +++ b/src/middleware/withAuthorizedSpace.js @@ -76,14 +76,22 @@ function extractSpaceDID (space) { export function withAuthorizedSpace (handler) { return async (request, env, ctx) => { const { locator, dataCid } = ctx + + console.log(`[withAuthorizedSpace] Locating content: ${dataCid}`) const locRes = await locator.locate(dataCid.multihash) if (locRes.error) { + console.log(`[withAuthorizedSpace] Location failed:`, locRes.error) if (locRes.error.name === 'NotFound') { throw new HttpError('Not Found', { status: 404, cause: locRes.error }) } throw new Error(`failed to locate: ${dataCid}`, { cause: locRes.error }) } + console.log(`[withAuthorizedSpace] Location result:`, { + sites: locRes.ok.site.length, + spaces: locRes.ok.site.map(s => s.space).filter(Boolean) + }) + // Legacy behavior: Site results which have no Space attached are from // before we started authorizing serving content explicitly. For these, we // always serve the content, but only if the request has no authorization @@ -96,6 +104,7 @@ export function withAuthorizedSpace (handler) { ctx.authToken === null if (shouldServeLegacy) { + console.log(`[withAuthorizedSpace] Using legacy path (no space)`) return handler(request, env, ctx) } @@ -113,17 +122,25 @@ export function withAuthorizedSpace (handler) { console.log(`Content found in ${uniqueSpaces.length} different spaces - egress tracking will be skipped`) console.log(`Spaces: ${uniqueSpaces.join(', ')}`) } + + console.log(`[withAuthorizedSpace] Found ${spaces.length} space(s):`, spaces) try { // First space to successfully authorize is the one we'll use. const { space: selectedSpace, delegationProofs } = await Promise.any( spaces.map(async (space) => { + console.log(`[withAuthorizedSpace] Attempting to authorize space: ${space}`) // @ts-ignore const result = await authorize(SpaceDID.from(space), ctx, env) - if (result.error) throw result.error + if (result.error) { + console.log(`[withAuthorizedSpace] Authorization failed for ${space}:`, result.error.message) + throw result.error + } + console.log(`[withAuthorizedSpace] ✅ Authorization succeeded for space: ${space}`) return result.ok }) ) + console.log(`[withAuthorizedSpace] Selected space for egress tracking: ${selectedSpace}`) return handler(request, env, { ...ctx, // Only set space if we're not skipping egress tracking @@ -185,6 +202,19 @@ const authorize = async (space, ctx, env) => { return fail('The gateway is not authorized to serve this content.') } + + // Get the content serve authority (upload service) from environment + // @ts-ignore - env has these properties from wrangler.toml + const contentServeAuthority = + env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID + ? + // @ts-ignore + Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( + // @ts-ignore + env.CONTENT_SERVE_AUTHORITY_DID + ) + : ctx.gatewayIdentity + // Create an invocation of the serve capability. const invocation = await serve.transportHttp .invoke({ @@ -202,7 +232,7 @@ const authorize = async (space, ctx, env) => { const validatorProofs = await getValidatorProofs(env) const accessResult = await access(invocation, { capability: serve.transportHttp, - authority: ctx.gatewayIdentity, + authority: contentServeAuthority, // Use upload service as authority principal: Verifier, proofs: validatorProofs, resolveDIDKey, diff --git a/src/middleware/withRateLimit.types.ts b/src/middleware/withRateLimit.types.ts index dd0fef2..836b26f 100644 --- a/src/middleware/withRateLimit.types.ts +++ b/src/middleware/withRateLimit.types.ts @@ -7,6 +7,8 @@ export interface Environment extends MiddlewareEnvironment { AUTH_TOKEN_METADATA: KVNamespace FF_RATE_LIMITER_ENABLED: string GATEWAY_VALIDATOR_PROOF?: string + CONTENT_SERVE_AUTHORITY_PUB_KEY: string + CONTENT_SERVE_AUTHORITY_DID: string } export interface TokenMetadata { diff --git a/wrangler.toml b/wrangler.toml index 472343e..e77d503 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -195,29 +195,30 @@ FF_EGRESS_TRACKER_ROLLOUT_PERCENTAGE = "100" FF_TELEMETRY_ENABLED = "true" FF_DELEGATIONS_STORAGE_ENABLED = "true" FF_RAMP_UP_PROBABILITY = "100" -FF_DAGPB_CONTENT_CACHE_TTL_SECONDS = 300 +FF_DAGPB_CONTENT_CACHE_TTL_SECONDS = 0 FF_DAGPB_CONTENT_CACHE_MAX_SIZE_MB = 2 -FF_DAGPB_CONTENT_CACHE_ENABLED = "true" +FF_DAGPB_CONTENT_CACHE_ENABLED = "false" TELEMETRY_RATIO = 1.0 -### staging + +### fforbeck-staging CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage" CARPARK_PUBLIC_BUCKET_URL = "https://carpark-staging-0.r2.w3s.link" GATEWAY_SERVICE_DID = "did:web:staging.w3s.link" -UPLOAD_SERVICE_DID = "did:web:staging.up.storacha.network" -UPLOAD_API_URL = "https://staging.up.storacha.network" -INDEXING_SERVICE_URL = "https://staging.indexer.storacha.network/" -### prod -CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage" -# CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link" -GATEWAY_SERVICE_DID = "did:web:staging.w3s.link" UPLOAD_SERVICE_DID = "did:web:staging.web3.storage" UPLOAD_API_URL = "https://staging.up.web3.storage" -# gateway validator proof (gateway -> gateway) -# GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" -# up validator proof (gateway -> upload-service) -#GATEWAY_VALIDATOR_PROOF = "mAYIEAJgDOqJlcm9vdHOB2CpYJQABcRIgN620cPucqWQBOpxv6IdEa2h9qIjbLjuVeX5EjKZYntlndmVyc2lvbgGBAgFxEiCZ8EADJu8q8uWgve3nZa/+mzdDfX1IyQ4riBFs3lOiP6dhc1hE7aEDQNWyV/OoO65h7E1sJhiVj/PtTd5rKWmgWRjct8CvfSsoEILycg7GF5CdF7KR/oryw5mjIiYqm1CNmaZr1o7c2ANhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgjZGlkOndlYjpzdGFnaW5nLnVwLnN0b3JhY2hhLm5ldHdvcmtjYXVkVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjZXhw9mNpc3NYIZ0ad2ViOnN0YWdpbmcudXAuc3RvcmFjaGEubmV0d29ya2NwcmaAWQFxEiA3rbRw+5ypZAE6nG/oh0RraH2oiNsuO5V5fkSMplie2aFqdWNhbkAwLjkuMdgqWCUAAXESIJnwQAMm7yry5aC97edlr/6bN0N9fUjJDiuIEWzeU6I/" +INDEXING_SERVICE_URL = "https://staging.indexer.storacha.network/" CONTENT_SERVE_AUTHORITY_PUB_KEY = "did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn" CONTENT_SERVE_AUTHORITY_DID = "did:web:staging.up.storacha.network" + +### fforbeck-prod +#CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage" +#CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link" +#GATEWAY_SERVICE_DID = "did:web:w3s.link" +#UPLOAD_SERVICE_DID = "did:web:web3.storage" +#UPLOAD_API_URL = "https://up.web3.storage" +#CONTENT_SERVE_AUTHORITY_PUB_KEY = "did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi" +#CONTENT_SERVE_AUTHORITY_DID = "did:web:up.storacha.network" + [[env.fforbeck.unsafe.bindings]] name = "RATE_LIMITER" type = "ratelimit" From 6c2858f67c2da3dde89615820e0d659329697c86 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Wed, 26 Nov 2025 10:21:41 -0300 Subject: [PATCH 04/13] wip --- src/middleware/withAuthorizedSpace.js | 20 +++++++++---------- src/middleware/withRateLimit.types.ts | 2 -- src/middleware/withUcanInvocationHandler.js | 14 ++++++------- .../withUcanInvocationHandler.types.ts | 1 - src/server/service.js | 11 +++++++--- wrangler.toml | 1 + 6 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/middleware/withAuthorizedSpace.js b/src/middleware/withAuthorizedSpace.js index 28b8a99..f3c364d 100644 --- a/src/middleware/withAuthorizedSpace.js +++ b/src/middleware/withAuthorizedSpace.js @@ -205,15 +205,15 @@ const authorize = async (space, ctx, env) => { // Get the content serve authority (upload service) from environment // @ts-ignore - env has these properties from wrangler.toml - const contentServeAuthority = - env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID - ? - // @ts-ignore - Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( - // @ts-ignore - env.CONTENT_SERVE_AUTHORITY_DID - ) - : ctx.gatewayIdentity + // const contentServeAuthority = + // env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID + // ? + // // @ts-ignore + // Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( + // // @ts-ignore + // env.CONTENT_SERVE_AUTHORITY_DID + // ) + // : ctx.gatewayIdentity // Create an invocation of the serve capability. const invocation = await serve.transportHttp @@ -232,7 +232,7 @@ const authorize = async (space, ctx, env) => { const validatorProofs = await getValidatorProofs(env) const accessResult = await access(invocation, { capability: serve.transportHttp, - authority: contentServeAuthority, // Use upload service as authority + authority: ctx.gatewayIdentity, principal: Verifier, proofs: validatorProofs, resolveDIDKey, diff --git a/src/middleware/withRateLimit.types.ts b/src/middleware/withRateLimit.types.ts index 836b26f..dd0fef2 100644 --- a/src/middleware/withRateLimit.types.ts +++ b/src/middleware/withRateLimit.types.ts @@ -7,8 +7,6 @@ export interface Environment extends MiddlewareEnvironment { AUTH_TOKEN_METADATA: KVNamespace FF_RATE_LIMITER_ENABLED: string GATEWAY_VALIDATOR_PROOF?: string - CONTENT_SERVE_AUTHORITY_PUB_KEY: string - CONTENT_SERVE_AUTHORITY_DID: string } export interface TokenMetadata { diff --git a/src/middleware/withUcanInvocationHandler.js b/src/middleware/withUcanInvocationHandler.js index 3d9ff65..d73bfd9 100644 --- a/src/middleware/withUcanInvocationHandler.js +++ b/src/middleware/withUcanInvocationHandler.js @@ -23,15 +23,15 @@ export function withUcanInvocationHandler (handler) { return handler(request, env, ctx) } - const contentServeAuthority = - env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID - ? ed25519.Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( - Schema.DID.from(env.CONTENT_SERVE_AUTHORITY_DID) - ) - : ctx.gatewayIdentity + // const contentServeAuthority = + // env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID + // ? ed25519.Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( + // Schema.DID.from(env.CONTENT_SERVE_AUTHORITY_DID) + // ) + // : ctx.gatewayIdentity const service = - ctx.service ?? (await createService(ctx, contentServeAuthority)) + ctx.service ?? (await createService(ctx, env)) const server = ctx.server ?? (await createServer(ctx, service, env)) const { headers, body, status } = await server.request({ diff --git a/src/middleware/withUcanInvocationHandler.types.ts b/src/middleware/withUcanInvocationHandler.types.ts index ed81217..da84e17 100644 --- a/src/middleware/withUcanInvocationHandler.types.ts +++ b/src/middleware/withUcanInvocationHandler.types.ts @@ -6,7 +6,6 @@ import * as Server from '@ucanto/server' export interface Environment extends MiddlewareEnvironment { GATEWAY_VALIDATOR_PROOF?: string CONTENT_SERVE_AUTHORITY_PUB_KEY?: string - CONTENT_SERVE_AUTHORITY_DID?: string } export interface Context diff --git a/src/server/service.js b/src/server/service.js index 5adc021..bb230f6 100644 --- a/src/server/service.js +++ b/src/server/service.js @@ -6,14 +6,16 @@ import { extractContentServeDelegations } from './utils.js' import { claim, Schema } from '@ucanto/validator' import * as UcantoServer from '@ucanto/server' import { ok } from '@ucanto/client' +import { getValidatorProofs } from './index.js' /** * @template T * @param {import('../middleware/withUcanInvocationHandler.types.js').Context} ctx - * @param {import('@ucanto/interface').Verifier} contentServeAuthority + * @param {import('../middleware/withUcanInvocationHandler.types.js').Environment} env * @returns {import('./api.types.js').Service} */ -export function createService (ctx, contentServeAuthority) { +export function createService (ctx, env) { + return { access: { delegate: UcantoServer.provideAdvanced({ @@ -34,12 +36,15 @@ export function createService (ctx, contentServeAuthority) { const delegations = result.ok console.log('executing claim for delegations: ', delegations) + const validatorProofs = await getValidatorProofs(env) + console.log('validatorProofs: ', validatorProofs) const validationResult = await claim( SpaceCapabilities.contentServe, delegations, { ...context, - authority: contentServeAuthority + authority: ctx.gatewayIdentity, + proofs: [...validatorProofs], } ) if (validationResult.error) { diff --git a/wrangler.toml b/wrangler.toml index e77d503..dde20cf 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -209,6 +209,7 @@ UPLOAD_API_URL = "https://staging.up.web3.storage" INDEXING_SERVICE_URL = "https://staging.indexer.storacha.network/" CONTENT_SERVE_AUTHORITY_PUB_KEY = "did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn" CONTENT_SERVE_AUTHORITY_DID = "did:web:staging.up.storacha.network" +GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" ### fforbeck-prod #CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage" From 5201d92fbdec5972b8f4953409632a4e09d93f8a Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Mon, 8 Dec 2025 09:20:57 -0300 Subject: [PATCH 05/13] fix config --- wrangler.toml | 3 --- 1 file changed, 3 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index dde20cf..f08d0ea 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -184,9 +184,6 @@ kv_namespaces = [ { binding = "DAGPB_CONTENT_CACHE", id = "c70a74363e7a4f06ad39fa3022aab7c7", preview_id = "fcea72d8d8694b5c831736e1317e9208" } ] -[env.fforbeck.observability.logs] -enabled = true - [env.fforbeck.vars] DEBUG = "true" FF_RATE_LIMITER_ENABLED = "false" From 97236b4f97f1caaa0ce845f7f4f9b531952013f8 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Mon, 8 Dec 2025 09:41:18 -0300 Subject: [PATCH 06/13] fix config --- src/middleware/withGatewayIdentity.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middleware/withGatewayIdentity.js b/src/middleware/withGatewayIdentity.js index d2f8fa2..365deea 100644 --- a/src/middleware/withGatewayIdentity.js +++ b/src/middleware/withGatewayIdentity.js @@ -22,7 +22,7 @@ export function withGatewayIdentity (handler) { ) return handler(req, env, { ...ctx, - gatewaySigner: gatewayIdentity, + gatewaySigner, //: gatewayIdentity, gatewayIdentity }) } From c28d53b809adadfca1430c7708f186548b438842 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Mon, 8 Dec 2025 09:44:37 -0300 Subject: [PATCH 07/13] lint fix --- src/middleware/withAuthorizedSpace.js | 15 +++++++-------- src/middleware/withGatewayIdentity.js | 2 +- src/server/service.js | 3 +-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/middleware/withAuthorizedSpace.js b/src/middleware/withAuthorizedSpace.js index f3c364d..57cc7ff 100644 --- a/src/middleware/withAuthorizedSpace.js +++ b/src/middleware/withAuthorizedSpace.js @@ -76,18 +76,18 @@ function extractSpaceDID (space) { export function withAuthorizedSpace (handler) { return async (request, env, ctx) => { const { locator, dataCid } = ctx - + console.log(`[withAuthorizedSpace] Locating content: ${dataCid}`) const locRes = await locator.locate(dataCid.multihash) if (locRes.error) { - console.log(`[withAuthorizedSpace] Location failed:`, locRes.error) + console.log('[withAuthorizedSpace] Location failed:', locRes.error) if (locRes.error.name === 'NotFound') { throw new HttpError('Not Found', { status: 404, cause: locRes.error }) } throw new Error(`failed to locate: ${dataCid}`, { cause: locRes.error }) } - console.log(`[withAuthorizedSpace] Location result:`, { + console.log('[withAuthorizedSpace] Location result:', { sites: locRes.ok.site.length, spaces: locRes.ok.site.map(s => s.space).filter(Boolean) }) @@ -104,7 +104,7 @@ export function withAuthorizedSpace (handler) { ctx.authToken === null if (shouldServeLegacy) { - console.log(`[withAuthorizedSpace] Using legacy path (no space)`) + console.log('[withAuthorizedSpace] Using legacy path (no space)') return handler(request, env, ctx) } @@ -122,7 +122,7 @@ export function withAuthorizedSpace (handler) { console.log(`Content found in ${uniqueSpaces.length} different spaces - egress tracking will be skipped`) console.log(`Spaces: ${uniqueSpaces.join(', ')}`) } - + console.log(`[withAuthorizedSpace] Found ${spaces.length} space(s):`, spaces) try { @@ -202,19 +202,18 @@ const authorize = async (space, ctx, env) => { return fail('The gateway is not authorized to serve this content.') } - // Get the content serve authority (upload service) from environment // @ts-ignore - env has these properties from wrangler.toml // const contentServeAuthority = // env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID - // ? + // ? // // @ts-ignore // Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( // // @ts-ignore // env.CONTENT_SERVE_AUTHORITY_DID // ) // : ctx.gatewayIdentity - + // Create an invocation of the serve capability. const invocation = await serve.transportHttp .invoke({ diff --git a/src/middleware/withGatewayIdentity.js b/src/middleware/withGatewayIdentity.js index 365deea..535467b 100644 --- a/src/middleware/withGatewayIdentity.js +++ b/src/middleware/withGatewayIdentity.js @@ -22,7 +22,7 @@ export function withGatewayIdentity (handler) { ) return handler(req, env, { ...ctx, - gatewaySigner, //: gatewayIdentity, + gatewaySigner, // : gatewayIdentity, gatewayIdentity }) } diff --git a/src/server/service.js b/src/server/service.js index bb230f6..4a14b52 100644 --- a/src/server/service.js +++ b/src/server/service.js @@ -15,7 +15,6 @@ import { getValidatorProofs } from './index.js' * @returns {import('./api.types.js').Service} */ export function createService (ctx, env) { - return { access: { delegate: UcantoServer.provideAdvanced({ @@ -44,7 +43,7 @@ export function createService (ctx, env) { { ...context, authority: ctx.gatewayIdentity, - proofs: [...validatorProofs], + proofs: [...validatorProofs] } ) if (validationResult.error) { From 5b5a046018990e6699bf7e8998d51fa8cbb84236 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Mon, 8 Dec 2025 09:49:32 -0300 Subject: [PATCH 08/13] lint fix --- src/middleware/withUcanInvocationHandler.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/middleware/withUcanInvocationHandler.js b/src/middleware/withUcanInvocationHandler.js index d73bfd9..e2cd690 100644 --- a/src/middleware/withUcanInvocationHandler.js +++ b/src/middleware/withUcanInvocationHandler.js @@ -1,7 +1,6 @@ -import { ed25519 } from '@ucanto/principal' import { createServer } from '../server/index.js' import { createService } from '../server/service.js' -import { Schema } from '@ucanto/core' + /** * @import { Middleware } from '@web3-storage/gateway-lib' * @import { From 7d2d8c6f4555ab00465cb550e26cbe5776868eaf Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Mon, 8 Dec 2025 10:31:27 -0300 Subject: [PATCH 09/13] minor fix --- src/middleware/withGatewayIdentity.js | 2 +- src/server/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/middleware/withGatewayIdentity.js b/src/middleware/withGatewayIdentity.js index 535467b..af6c123 100644 --- a/src/middleware/withGatewayIdentity.js +++ b/src/middleware/withGatewayIdentity.js @@ -22,7 +22,7 @@ export function withGatewayIdentity (handler) { ) return handler(req, env, { ...ctx, - gatewaySigner, // : gatewayIdentity, + gatewaySigner, gatewayIdentity }) } diff --git a/src/server/index.js b/src/server/index.js index c3d694a..d2360f1 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -94,7 +94,7 @@ export async function createServer (ctx, service, env) { }) } return Server.create({ - id: ctx.gatewaySigner, + id: ctx.gatewaySigner.withDID(ctx.gatewayIdentity.did()), codec: CAR.inbound, service, catch: (err) => console.error(err), From cead20905ef52ee5b2ba958ff24d06d5a68ca2ea Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Mon, 8 Dec 2025 10:38:18 -0300 Subject: [PATCH 10/13] clean up comments --- src/middleware/withAuthorizedSpace.js | 12 ------------ src/middleware/withUcanInvocationHandler.js | 7 ------- 2 files changed, 19 deletions(-) diff --git a/src/middleware/withAuthorizedSpace.js b/src/middleware/withAuthorizedSpace.js index 57cc7ff..bf2ea58 100644 --- a/src/middleware/withAuthorizedSpace.js +++ b/src/middleware/withAuthorizedSpace.js @@ -202,18 +202,6 @@ const authorize = async (space, ctx, env) => { return fail('The gateway is not authorized to serve this content.') } - // Get the content serve authority (upload service) from environment - // @ts-ignore - env has these properties from wrangler.toml - // const contentServeAuthority = - // env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID - // ? - // // @ts-ignore - // Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( - // // @ts-ignore - // env.CONTENT_SERVE_AUTHORITY_DID - // ) - // : ctx.gatewayIdentity - // Create an invocation of the serve capability. const invocation = await serve.transportHttp .invoke({ diff --git a/src/middleware/withUcanInvocationHandler.js b/src/middleware/withUcanInvocationHandler.js index e2cd690..acafd44 100644 --- a/src/middleware/withUcanInvocationHandler.js +++ b/src/middleware/withUcanInvocationHandler.js @@ -22,13 +22,6 @@ export function withUcanInvocationHandler (handler) { return handler(request, env, ctx) } - // const contentServeAuthority = - // env.CONTENT_SERVE_AUTHORITY_PUB_KEY && env.CONTENT_SERVE_AUTHORITY_DID - // ? ed25519.Verifier.parse(env.CONTENT_SERVE_AUTHORITY_PUB_KEY).withDID( - // Schema.DID.from(env.CONTENT_SERVE_AUTHORITY_DID) - // ) - // : ctx.gatewayIdentity - const service = ctx.service ?? (await createService(ctx, env)) const server = ctx.server ?? (await createServer(ctx, service, env)) From 2af6f0bb88685df42a87929ec8508b575364633d Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Tue, 9 Dec 2025 10:07:50 -0300 Subject: [PATCH 11/13] cleanup --- package-lock.json | 4 +++- package.json | 5 +++-- src/middleware/withAuthorizedSpace.js | 14 -------------- src/middleware/withUcanInvocationHandler.types.ts | 1 - src/server/utils.js | 9 --------- wrangler.toml | 7 ++----- 6 files changed, 8 insertions(+), 32 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1c6f9e4..14edab3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,12 +9,13 @@ "version": "2.36.0", "license": "Apache-2.0 OR MIT", "dependencies": { + "@ipld/dag-json": "^10.2.5", "@ipld/dag-pb": "^4.1.5", "@microlabs/otel-cf-workers": "^1.0.0-rc.48", "@opentelemetry/api": "^1.9.0", "@opentelemetry/sdk-trace-base": "^1.27.0", "@storacha/capabilities": "^1.10.0", - "@storacha/indexing-service-client": "^2.6.6", + "@storacha/indexing-service-client": "^2.6.9", "@types/node": "^24.10.1", "@ucanto/client": "^9.0.2", "@ucanto/core": "^10.4.5", @@ -31,6 +32,7 @@ "multiformats": "^13.0.1" }, "devDependencies": { + "@ipld/dag-ucan": "^3.4.5", "@storacha/blob-index": "^1.2.4", "@storacha/cli": "^1.6.2", "@storacha/client": "^1.8.2", diff --git a/package.json b/package.json index 93228fb..4fbddac 100644 --- a/package.json +++ b/package.json @@ -36,12 +36,13 @@ "test:unit:only": "npm run build:debug && mocha --experimental-vm-modules" }, "dependencies": { + "@ipld/dag-json": "^10.2.5", "@ipld/dag-pb": "^4.1.5", "@microlabs/otel-cf-workers": "^1.0.0-rc.48", "@opentelemetry/api": "^1.9.0", "@opentelemetry/sdk-trace-base": "^1.27.0", "@storacha/capabilities": "^1.10.0", - "@storacha/indexing-service-client": "^2.6.6", + "@storacha/indexing-service-client": "^2.6.9", "@types/node": "^24.10.1", "@ucanto/client": "^9.0.2", "@ucanto/core": "^10.4.5", @@ -58,8 +59,8 @@ "multiformats": "^13.0.1" }, "devDependencies": { - "@storacha/blob-index": "^1.2.4", "@ipld/dag-ucan": "^3.4.5", + "@storacha/blob-index": "^1.2.4", "@storacha/cli": "^1.6.2", "@storacha/client": "^1.8.2", "@storacha/upload-client": "^1.3.6", diff --git a/src/middleware/withAuthorizedSpace.js b/src/middleware/withAuthorizedSpace.js index bf2ea58..a5e2eb9 100644 --- a/src/middleware/withAuthorizedSpace.js +++ b/src/middleware/withAuthorizedSpace.js @@ -76,22 +76,14 @@ function extractSpaceDID (space) { export function withAuthorizedSpace (handler) { return async (request, env, ctx) => { const { locator, dataCid } = ctx - - console.log(`[withAuthorizedSpace] Locating content: ${dataCid}`) const locRes = await locator.locate(dataCid.multihash) if (locRes.error) { - console.log('[withAuthorizedSpace] Location failed:', locRes.error) if (locRes.error.name === 'NotFound') { throw new HttpError('Not Found', { status: 404, cause: locRes.error }) } throw new Error(`failed to locate: ${dataCid}`, { cause: locRes.error }) } - console.log('[withAuthorizedSpace] Location result:', { - sites: locRes.ok.site.length, - spaces: locRes.ok.site.map(s => s.space).filter(Boolean) - }) - // Legacy behavior: Site results which have no Space attached are from // before we started authorizing serving content explicitly. For these, we // always serve the content, but only if the request has no authorization @@ -123,24 +115,18 @@ export function withAuthorizedSpace (handler) { console.log(`Spaces: ${uniqueSpaces.join(', ')}`) } - console.log(`[withAuthorizedSpace] Found ${spaces.length} space(s):`, spaces) - try { // First space to successfully authorize is the one we'll use. const { space: selectedSpace, delegationProofs } = await Promise.any( spaces.map(async (space) => { - console.log(`[withAuthorizedSpace] Attempting to authorize space: ${space}`) // @ts-ignore const result = await authorize(SpaceDID.from(space), ctx, env) if (result.error) { - console.log(`[withAuthorizedSpace] Authorization failed for ${space}:`, result.error.message) throw result.error } - console.log(`[withAuthorizedSpace] ✅ Authorization succeeded for space: ${space}`) return result.ok }) ) - console.log(`[withAuthorizedSpace] Selected space for egress tracking: ${selectedSpace}`) return handler(request, env, { ...ctx, // Only set space if we're not skipping egress tracking diff --git a/src/middleware/withUcanInvocationHandler.types.ts b/src/middleware/withUcanInvocationHandler.types.ts index da84e17..2aad1f4 100644 --- a/src/middleware/withUcanInvocationHandler.types.ts +++ b/src/middleware/withUcanInvocationHandler.types.ts @@ -5,7 +5,6 @@ import { Service } from '../server/api.types.js' import * as Server from '@ucanto/server' export interface Environment extends MiddlewareEnvironment { GATEWAY_VALIDATOR_PROOF?: string - CONTENT_SERVE_AUTHORITY_PUB_KEY?: string } export interface Context diff --git a/src/server/utils.js b/src/server/utils.js index 800643e..7fcd3a0 100644 --- a/src/server/utils.js +++ b/src/server/utils.js @@ -12,7 +12,6 @@ export const extractContentServeDelegations = async ( capability, invocation ) => { - console.log('extracting delegations', capability, invocation) /** @type {Map} */ const blocks = new Map() for (const block of invocation.iterateIPLDBlocks()) { @@ -70,13 +69,5 @@ export const extractContentServeDelegations = async ( } } - // Only allow a single delegation reference for now - // const nbDelegations = new Set(Object.values(capability.nb.delegations)) - // if (nbDelegations.size !== 1) { - // return { - // error: new InvalidDelegation('nb.delegations has more than one delegation') - // } - // } - return { ok: delegations } } diff --git a/wrangler.toml b/wrangler.toml index f08d0ea..ebe0b57 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -201,11 +201,10 @@ TELEMETRY_RATIO = 1.0 CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage" CARPARK_PUBLIC_BUCKET_URL = "https://carpark-staging-0.r2.w3s.link" GATEWAY_SERVICE_DID = "did:web:staging.w3s.link" -UPLOAD_SERVICE_DID = "did:web:staging.web3.storage" +UPLOAD_SERVICE_DID = "did:web:staging.up.storacha.network" UPLOAD_API_URL = "https://staging.up.web3.storage" INDEXING_SERVICE_URL = "https://staging.indexer.storacha.network/" -CONTENT_SERVE_AUTHORITY_PUB_KEY = "did:key:z6MkhcbEpJpEvNVDd3n5RurquVdqs5dPU16JDU5VZTDtFgnn" -CONTENT_SERVE_AUTHORITY_DID = "did:web:staging.up.storacha.network" +MAX_SHARDS = "825" GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" ### fforbeck-prod @@ -214,8 +213,6 @@ GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wV #GATEWAY_SERVICE_DID = "did:web:w3s.link" #UPLOAD_SERVICE_DID = "did:web:web3.storage" #UPLOAD_API_URL = "https://up.web3.storage" -#CONTENT_SERVE_AUTHORITY_PUB_KEY = "did:key:z6MkqdncRZ1wj8zxCTDUQ8CRT8NQWd63T7mZRvZUX8B7XDFi" -#CONTENT_SERVE_AUTHORITY_DID = "did:web:up.storacha.network" [[env.fforbeck.unsafe.bindings]] name = "RATE_LIMITER" From 8601eb837bd499bcb079d2777ff0d1ca2b0e3638 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Tue, 9 Dec 2025 10:39:05 -0300 Subject: [PATCH 12/13] added missing GATEWAY_VALIDATOR_PROOF for staging and prod --- wrangler.toml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/wrangler.toml b/wrangler.toml index ebe0b57..ba9ae61 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -72,6 +72,7 @@ CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage" CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link" UPLOAD_API_URL = "https://up.web3.storage" INDEXING_SERVICE_URL = "https://indexer.storacha.network/" +GATEWAY_VALIDATOR_PROOF = "mAYIEAPQCOqJlcm9vdHOB2CpYJQABcRIgYOVdOrpLxbw0mELO22krjEVICp5cju4nVChUCL8+i/1ndmVyc2lvbgHdAQFxEiCUjD+aZlOcJxbn3UlM1dqeSYs09dyPrMvVKPmC+TSaoadhc1hE7aEDQPhDJ+yO05lTqYJndtPY9d2WXGtyj90eoL0MBeKylYKMQDnSMBkMAv8XMX+OEZd7QrhvzAktsXMEKY+wPQAbpQRhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHBkaWQ6d2ViOnczcy5saW5rY2F1ZFgZnRp3ZWI6dXAuc3RvcmFjaGEubmV0d29ya2NleHD2Y2lzc06dGndlYjp3M3MubGlua2NwcmaAWQFxEiBg5V06ukvFvDSYQs7baSuMRUgKnlyO7idUKFQIvz6L/aFqdWNhbkAwLjkuMdgqWCUAAXESIJSMP5pmU5wnFufdSUzV2p5JizT13I+sy9Uo+YL5NJqh" [env.production.observability] enabled = true @@ -115,7 +116,7 @@ CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage" UPLOAD_API_URL = "https://staging.up.web3.storage" CARPARK_PUBLIC_BUCKET_URL = "https://carpark-staging-0.r2.w3s.link" INDEXING_SERVICE_URL = "https://staging.indexer.storacha.network/" - +GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" # Test! [env.test] @@ -196,24 +197,15 @@ FF_DAGPB_CONTENT_CACHE_TTL_SECONDS = 0 FF_DAGPB_CONTENT_CACHE_MAX_SIZE_MB = 2 FF_DAGPB_CONTENT_CACHE_ENABLED = "false" TELEMETRY_RATIO = 1.0 - -### fforbeck-staging CONTENT_CLAIMS_SERVICE_URL = "https://staging.claims.web3.storage" CARPARK_PUBLIC_BUCKET_URL = "https://carpark-staging-0.r2.w3s.link" GATEWAY_SERVICE_DID = "did:web:staging.w3s.link" UPLOAD_SERVICE_DID = "did:web:staging.up.storacha.network" -UPLOAD_API_URL = "https://staging.up.web3.storage" +UPLOAD_API_URL = "https://staging.up.storacha.network" INDEXING_SERVICE_URL = "https://staging.indexer.storacha.network/" MAX_SHARDS = "825" GATEWAY_VALIDATOR_PROOF = "mAYIEAI0DOqJlcm9vdHOB2CpYJQABcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqlndmVyc2lvbgH2AQFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOA6dhc1hE7aEDQCEy4qAv4dybXYh/iipwdOLf4DlYjZGFopGvvfDZbVq6Q4MIY5PY9XXHP9D6ELs6qjcvSLfP/nbew/IWbG55RAJhdmUwLjkuMWNhdHSBomNjYW5rdWNhbi9hdHRlc3Rkd2l0aHgYZGlkOndlYjpzdGFnaW5nLnczcy5saW5rY2F1ZFghnRp3ZWI6c3RhZ2luZy51cC5zdG9yYWNoYS5uZXR3b3JrY2V4cPZjaXNzVp0ad2ViOnN0YWdpbmcudzNzLmxpbmtjcHJmgFkBcRIg5a/pcyEeJn5snQZHm+wVBzeFbY8RTlfwETn+nMMiCqmhanVjYW5AMC45LjHYKlglAAFxEiD8LRCZwMMfTVlGtNX85n0aVxjp54oE0/G/WTG/dsOOAw" -### fforbeck-prod -#CONTENT_CLAIMS_SERVICE_URL = "https://claims.web3.storage" -#CARPARK_PUBLIC_BUCKET_URL = "https://carpark-prod-0.r2.w3s.link" -#GATEWAY_SERVICE_DID = "did:web:w3s.link" -#UPLOAD_SERVICE_DID = "did:web:web3.storage" -#UPLOAD_API_URL = "https://up.web3.storage" - [[env.fforbeck.unsafe.bindings]] name = "RATE_LIMITER" type = "ratelimit" From 6dc6687fe58f52110731e5bc3650b764d41916d4 Mon Sep 17 00:00:00 2001 From: Felipe Forbeck Date: Tue, 9 Dec 2025 12:50:09 -0300 Subject: [PATCH 13/13] update blob-fetcher lib --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 14edab3..888a84f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "@ucanto/server": "^11.0.3", "@ucanto/transport": "^9.2.1", "@ucanto/validator": "^10.0.1", - "@web3-storage/blob-fetcher": "^4.2.5", + "@web3-storage/blob-fetcher": "^4.2.6", "@web3-storage/gateway-lib": "^5.2.2", "dagula": "^8.0.0", "http-range-parse": "^1.0.0", @@ -6079,9 +6079,9 @@ "license": "ISC" }, "node_modules/@web3-storage/blob-fetcher": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/@web3-storage/blob-fetcher/-/blob-fetcher-4.2.5.tgz", - "integrity": "sha512-wfs8XNcgp5nE7zFidUG7L/QL5MxCCK1a22ly8meRgHScycSJCu1OXEGGYY9wxOXgkQxZ8S65Kkn/TIh8xGgUZg==", + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/@web3-storage/blob-fetcher/-/blob-fetcher-4.2.6.tgz", + "integrity": "sha512-4svLIamPg7tjIDJK3vKmNq4XagggcfSKNRp3EbVtJA5lRdBYjj7h19242TMB/3iL8rvU36vMZKCFldNGAKnk5Q==", "license": "Apache-2.0 OR MIT", "dependencies": { "@cloudflare/workers-types": "^4.20241022.0", diff --git a/package.json b/package.json index 4fbddac..e628201 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "@ucanto/server": "^11.0.3", "@ucanto/transport": "^9.2.1", "@ucanto/validator": "^10.0.1", - "@web3-storage/blob-fetcher": "^4.2.5", + "@web3-storage/blob-fetcher": "^4.2.6", "@web3-storage/gateway-lib": "^5.2.2", "dagula": "^8.0.0", "http-range-parse": "^1.0.0",