[FIX] Reveal chunk headers to unambiguously handle chunked responses (#90)

* [FIX] Reveal chunk headers to unambiguously handle chunked encoded responses

Author: Scratch-net

proto/api.proto

@@ -25,6 +25,7 @@ enum AttestorVersion {
 	ATTESTOR_VERSION_2_0_1 = 4;
 	ATTESTOR_VERSION_3_0_0 = 5;
 	ATTESTOR_VERSION_3_1_0 = 6;
+	ATTESTOR_VERSION_3_2_0 = 7;
 }
 
 enum ErrorCode {

src/client/create-claim.ts

@@ -29,6 +29,7 @@ import {
 	makeDefaultOPRFOperator,
 	makeHttpResponseParser,
 	preparePacketsForReveal,
+	REDACTION_CHAR_CODE,
 	redactSlices,
 	uint8ArrayToStr,
 	unixTimestampSeconds
@@ -512,9 +513,25 @@ async function _createClaimOnAttestor<N extends ProviderName>(
 
 			revealedPackets.push(...packets.filter(p => p.sender === 'server'))
 		} else {
-			for(const {
-				block, redactedPlaintext, overshotToprfFromPrevBlock, toprfs, oprfRawMarkers
-			} of serverPacketsToReveal) {
+			const revealByBlock = new Map(
+				serverPacketsToReveal.map(r => [r.block, r])
+			)
+			// walk every server block in order: revealed blocks are proven via
+			// zk, fully-redacted blocks get an asterisk placeholder so the local
+			// receipt stays byte-aligned for chunked dechunking
+			for(const sb of serverBlocks) {
+				const reveal = revealByBlock.get(sb)
+				if(!reveal) {
+					revealedPackets.push({
+						sender: 'server',
+						message: new Uint8Array(sb.plaintext.length).fill(REDACTION_CHAR_CODE)
+					})
+					continue
+				}
+
+				const {
+					block, redactedPlaintext, overshotToprfFromPrevBlock, toprfs, oprfRawMarkers
+				} = reveal
 				setRevealOfMessage(block.message, {
 					type: 'zk',
 					redactedPlaintext,

src/config/index.ts

@@ -28,7 +28,7 @@ export const DNS_SERVERS = [
 // 10m
 export const MAX_CLAIM_TIMESTAMP_DIFF_S = 10 * 60
 
-export const CURRENT_ATTESTOR_VERSION = AttestorVersion.ATTESTOR_VERSION_3_1_0
+export const CURRENT_ATTESTOR_VERSION = AttestorVersion.ATTESTOR_VERSION_3_2_0
 
 export const DEFAULT_METADATA: InitRequest = {
 	signatureType: ServiceSignatureType.SERVICE_SIGNATURE_TYPE_ETH,

src/proto/api.ts

@@ -29,6 +29,7 @@ import {
 	findIndexInUint8Array,
 	getHttpRequestDataFromTranscript,
 	logger,
+	makeHttpResponseParser,
 	REDACTION_CHAR,
 	REDACTION_CHAR_CODE,
 	strToUint8Array,
@@ -207,6 +208,8 @@ const HTTP_PROVIDER: Provider<'http'> = {
 			throw new Error('Failed to find response body')
 		}
 
+		const revealFraming = shouldRevealChunkFraming(ctx.version)
+
 		const reveals: RedactedOrHashedArraySlice[] = [
 			{ fromIndex: 0, toIndex: headerEndIndex }
 		]
@@ -230,20 +233,39 @@ const HTTP_PROVIDER: Provider<'http'> = {
 			reveals.push(res.headerIndices['date'])
 		}
 
+		//reveal transfer-encoding header so the verifier can dechunk the body
+		if(revealFraming && res.headerIndices['transfer-encoding']) {
+			reveals.push(res.headerIndices['transfer-encoding'])
+		}
+
 		const body = uint8ArrayToBinaryStr(res.body)
 
 		const redactions: RedactedOrHashedArraySlice[] = []
 		for(const rs of params.responseRedactions || []) {
 			const processor = processRedactionRequest(
-				body, rs, bodyStartIdx, res.chunks
+				body, rs, bodyStartIdx, res.chunks, revealFraming
 			)
 			for(const { reveal, redactions: reds } of processor) {
 				reveals.push(reveal)
 				redactions.push(...reds)
 			}
 		}
 
-		reveals.sort((a, b) => a.toIndex - b.toIndex)
+		//reveal all chunk framing (size lines + terminator) so the verifier
+		//can dechunk; chunk data stays redacted unless a redaction reveals it
+		if(revealFraming && res.chunks?.length) {
+			let prev = res.headerEndIdx + 4
+			for(const chunk of res.chunks) {
+				reveals.push({ fromIndex: prev, toIndex: chunk.fromIndex })
+				prev = chunk.toIndex
+			}
+
+			reveals.push({ fromIndex: prev, toIndex: response.length })
+		}
+
+		//reveals can overlap (e.g. a redaction reveal spanning chunk framing),
+		//so redact the complement of their union
+		reveals.sort((a, b) => a.fromIndex - b.fromIndex)
 
 		if(reveals.length > 1) {
 			let currentIndex = 0
@@ -252,10 +274,12 @@ const HTTP_PROVIDER: Provider<'http'> = {
 					redactions.push({ fromIndex: currentIndex, toIndex: r.fromIndex })
 				}
 
-				currentIndex = r.toIndex
+				currentIndex = Math.max(currentIndex, r.toIndex)
 			}
 
-			redactions.push({ fromIndex: currentIndex, toIndex: response.length })
+			if(currentIndex < response.length) {
+				redactions.push({ fromIndex: currentIndex, toIndex: response.length })
+			}
 		}
 
 		for(const r of reveals) {
@@ -321,11 +345,17 @@ const HTTP_PROVIDER: Provider<'http'> = {
 			throw new Error(`Connection header must be "close", got "${connectionHeader}"`)
 		}
 
-		const serverBlocks = receipt
+		const allServerBlocks = receipt
 			.filter(s => s.sender === 'server')
 			.map((r) => r.message)
-			// filter out fully redacted blocks
-			.filter(b => !b.every(b => b === REDACTION_CHAR_CODE))
+		// drop leading fully-redacted (handshake) blocks so the response starts
+		// at the status line, but keep redacted body records so chunked byte
+		// alignment is preserved for dechunking
+		const firstRealIdx = allServerBlocks
+			.findIndex(b => !b.every(x => x === REDACTION_CHAR_CODE))
+		const serverBlocks = firstRealIdx === -1
+			? []
+			: allServerBlocks.slice(firstRealIdx)
 		const response = concatArrays(...serverBlocks)
 
 		let res: string
@@ -389,12 +419,19 @@ const HTTP_PROVIDER: Provider<'http'> = {
 		}
 
 
-		//remove asterisks to account for chunks in the middle of revealed strings
-		if(!secretParams) {
+		if(shouldRevealChunkFraming(clientVersion)) {
+			//dechunk the body so matches see contiguous content; redaction
+			//asterisks are preserved as boundaries between revealed slices
+			const headersStr = res.slice(0, bodyStart)
+			res = /transfer-encoding:\s*chunked/i.test(headersStr)
+				? headersStr + dechunkResponseBody(res.slice(bodyStart))
+				: res
+		} else if(!secretParams) {
+			//legacy clients redact chunk framing; collapse asterisk runs so
+			//matches can span the redacted framing
 			res = res.slice(bodyStart).replace(/\*{3,}/g, '')
 		}
 
-
 		for(const { type, value, invert } of params.responseMatches || []) {
 			const inv = Boolean(invert) // explicitly cast to boolean
 
@@ -510,6 +547,23 @@ function shouldRevealCrlf({ version }: ProviderCtx) {
 	return version >= AttestorVersion.ATTESTOR_VERSION_2_0_1
 }
 
+// revealing chunk framing (instead of redacting it) + dechunking on the
+// verifier is a breaking change; older clients still redact framing and rely
+// on the asterisk-collapse matching path
+function shouldRevealChunkFraming(version: AttestorVersion) {
+	return version >= AttestorVersion.ATTESTOR_VERSION_3_2_0
+}
+
+// dechunk a revealed body by reusing the parser: wrap it in a minimal
+// (already-validated) header block. streamEnded() skipped to tolerate trailing bytes
+function dechunkResponseBody(body: string) {
+	const parser = makeHttpResponseParser()
+	parser.onChunk(
+		strToUint8Array('HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n' + body)
+	)
+	return uint8ArrayToStr(parser.res.body ?? new Uint8Array())
+}
+
 function getHostPort(params: ProviderParams<'http'>, secretParams: ProviderSecretParams<'http'>) {
 	const { host } = new URL(getURL(params, secretParams))
 	if(!host) {
@@ -550,6 +604,7 @@ function* processRedactionRequest(
 	rs: HTTPResponseRedaction,
 	bodyStartIdx: number,
 	resChunks: ArraySlice[] | undefined,
+	revealFraming: boolean,
 ): Generator<RedactionItem> {
 	let element = body
 	let elementIdx = 0
@@ -665,22 +720,19 @@ function* processRedactionRequest(
 
 	function* addRedaction(
 		hash: RedactedOrHashedArraySlice['hash'] | null = rs.hash,
-		_resChunks = resChunks
 	): Generator<RedactionItem> {
 		if(elementIdx < 0 || !elementLength) {
 			return
 		}
 
 		const reveal = getReveal(elementIdx, elementLength, hash || undefined)
 
-		yield {
-			reveal,
-			redactions: getRedactionsForChunkHeaders(
-				reveal.fromIndex,
-				reveal.toIndex,
-				_resChunks
-			)
-		}
+		// new clients leave chunk framing revealed (verifier dechunks); older
+		// clients redact the framing inside a reveal that spans chunks
+		const redactions = revealFraming
+			? []
+			: getRedactionsForChunkHeaders(reveal.fromIndex, reveal.toIndex, resChunks)
+		yield { reveal, redactions }
 	}
 
 	function getReveal(

src/providers/http/index.ts

@@ -1,4 +1,4 @@
-import { areUint8ArraysEqual, concatenateUint8Arrays } from '@reclaimprotocol/tls'
+import { areUint8ArraysEqual, AUTH_TAG_BYTE_LENGTH, concatenateUint8Arrays, SUPPORTED_CIPHER_SUITE_MAP } from '@reclaimprotocol/tls'
 import type { ZKEngine } from '@reclaimprotocol/zk-symmetric-crypto'
 
 import type {
@@ -457,7 +457,15 @@ export async function decryptTranscript(
 			plaintextLength = plaintext.length
 		} else {
 			plaintext = content
-			plaintextLength = plaintext.length
+			// ciphertext = explicit-nonce + plaintext + auth-tag; strip both so the
+			// reconstructed redacted length matches the true plaintext length.
+			// TLS1.2 AES-GCM prepends an 8-byte explicit nonce; TLS1.3 and
+			// ChaCha20-Poly1305 have none
+			const { cipher } = SUPPORTED_CIPHER_SUITE_MAP[cipherSuite]
+			const explicitNonceLength = tlsVersion !== 'TLS1_3' && cipher.includes('GCM')
+				? 8
+				: 0
+			plaintextLength = content.length - explicitNonceLength - AUTH_TAG_BYTE_LENGTH
 		}
 
 		decryptedTranscript.push({