Skip to content

Commit 32c278b

Browse files
acolytec3acolytec3
authored
Add types and store support for Ephemeral Headers (#778)
* update spec test to latest * spec test runner updates * Handle ephemeral headers FindCONTENT * fix ephemeral headers offer * test fixes * TODO * clean up todo * partial cleanup of spec test runner
1 parent 5060b62 commit 32c278b

8 files changed

Lines changed: 223 additions & 153 deletions

File tree

‎packages/portal-spec-tests‎

Submodule portal-spec-tests updated 29 files

‎packages/portalnetwork/src/networks/history/history.ts‎

Lines changed: 83 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { BlockHeader } from '@ethereumjs/block'
22
import { Block, createBlockHeaderFromRLP } from '@ethereumjs/block'
3-
import { bytesToHex, bytesToInt, concatBytes, equalsBytes, hexToBytes, PrefixedHexString } from '@ethereumjs/util'
3+
import { bytesToHex, bytesToInt, concatBytes, equalsBytes, hexToBytes, type PrefixedHexString } from '@ethereumjs/util'
44
import debug from 'debug'
55

66
import type {
@@ -401,70 +401,47 @@ export class HistoryNetwork extends BaseNetwork {
401401

402402
const contentKey = decodeHistoryNetworkContentKey(decodedContentMessage.contentKey)
403403
let value: Uint8Array | undefined
404-
if (contentKey.contentType === HistoryNetworkContentType.EphemeralHeader) {
405-
if (contentKey.keyOpt.ancestorCount < 0 || contentKey.keyOpt.ancestorCount > 255) {
406-
const errorMessage = `received invalid ephemeral headers request with invalid ancestorCount: expected 0 <= 255, got ${contentKey.keyOpt.ancestorCount}`
404+
if (contentKey.contentType === HistoryNetworkContentType.EphemeralHeaderFindContent) {
405+
const ck = contentKey as { contentType: HistoryNetworkContentType.EphemeralHeaderFindContent, keyOpt: EphemeralHeaderKeyValues }
406+
if (ck.keyOpt.ancestorCount < 0 || ck.keyOpt.ancestorCount > 255) {
407+
const errorMessage = `received invalid ephemeral headers request with invalid ancestorCount: expected 0 <= 255, got ${ck.keyOpt.ancestorCount}`
407408
this.logger.extend('FOUNDCONTENT')(errorMessage)
408409
throw new Error(errorMessage)
409410
}
410411
this.logger.extend('FOUNDCONTENT')(
411-
`Received ephemeral headers request for block ${bytesToHex(contentKey.keyOpt.blockHash)} with ancestorCount ${contentKey.keyOpt.ancestorCount}`,
412+
`Received ephemeral headers request for block ${bytesToHex(ck.keyOpt.blockHash)} with ancestorCount ${ck.keyOpt.ancestorCount}`,
412413
)
413-
// Retrieve the starting header from the FINDCONTENT request
414-
const headerKey = getEphemeralHeaderDbKey(contentKey.keyOpt.blockHash)
415-
const firstHeader = await this.findContentLocally(headerKey)
416-
417-
if (firstHeader === undefined) {
418-
// If we don't have the requested header, send an empty payload
419-
// We never send an ENRs response for ephemeral headers
420-
value = undefined
414+
try {
415+
const payload = await this.assembleEphemeralHeadersPayload(ck.keyOpt.blockHash, ck.keyOpt.ancestorCount)
416+
this.logger.extend('FOUNDCONTENT')(
417+
`Found ${payload.length} headers for ${bytesToHex(ck.keyOpt.blockHash)}, assembling ephemeral headers response to ${shortId(src.nodeId)}`,
418+
)
419+
value = payload
420+
}
421+
catch (err: any) {
422+
if (err.message.includes('Header not found')) {
423+
this.logger.extend('FOUNDCONTENT').extend('EPHEMERALHEADERS')(
424+
`Header not found for ${bytesToHex(ck.keyOpt.blockHash)}, sending empty ephemeral headers response to ${shortId(src.nodeId)}`,
425+
)
426+
} else {
427+
this.logger.extend('FOUNDCONTENT').extend('EPHEMERALHEADERS')(
428+
`Error assembling ephemeral headers response to ${shortId(src.nodeId)}: ${err.message}`,
429+
)
430+
}
421431
const emptyHeaderPayload = EphemeralHeaderPayload.serialize([])
422432
const messagePayload = ContentMessageType.serialize({
423433
selector: FoundContent.CONTENT,
424434
value: emptyHeaderPayload,
425435
})
426-
this.logger.extend('FOUNDCONTENT')(
427-
`Header not found for ${bytesToHex(contentKey.keyOpt.blockHash)}, sending empty ephemeral headers response to ${shortId(src.nodeId)}`,
428-
)
436+
429437
await this.sendResponse(
430438
src,
431439
requestId,
432440
concatBytes(Uint8Array.from([MessageCodes.CONTENT]), messagePayload),
433441
)
434442
return
435-
} else {
436-
this.logger.extend('FOUNDCONTENT')(
437-
`Header found for ${bytesToHex(contentKey.keyOpt.blockHash)}, assembling ephemeral headers response to ${shortId(src.nodeId)}`,
438-
)
439-
// We have the requested header so begin assembling the payload
440-
const headersList = [firstHeader]
441-
const firstHeaderNumber = this.ephemeralHeaderIndex.getByValue(
442-
bytesToHex(contentKey.keyOpt.blockHash),
443-
)
444-
for (let x = 1; x <= contentKey.keyOpt.ancestorCount; x++) {
445-
// Determine if we have the ancestor header at block number `firstHeaderNumber - x`
446-
const ancestorNumber = firstHeaderNumber! - BigInt(x)
447-
const ancestorHash = this.ephemeralHeaderIndex.getByKey(ancestorNumber)
448-
if (ancestorHash === undefined)
449-
break // Stop looking for more ancestors if we don't have the current one in the index
450-
else {
451-
const ancestorKey = getEphemeralHeaderDbKey(hexToBytes(ancestorHash as PrefixedHexString))
452-
const ancestorHeader = await this.findContentLocally(ancestorKey)
453-
if (ancestorHeader === undefined) {
454-
// This would only happen if our index gets out of sync with the DB
455-
// Stop looking for more ancestors if we don't have the current one in the DB
456-
this.ephemeralHeaderIndex.delete(ancestorNumber)
457-
break
458-
} else {
459-
headersList.push(ancestorHeader)
460-
}
461-
}
462-
}
463-
this.logger.extend('FOUNDCONTENT')(
464-
`found ${headersList.length - 1} ancestor headers for ${bytesToHex(contentKey.keyOpt.blockHash)}`,
465-
)
466-
value = EphemeralHeaderPayload.serialize(headersList)
467443
}
444+
468445
} else {
469446
value = await this.findContentLocally(decodedContentMessage.contentKey)
470447
}
@@ -476,10 +453,10 @@ export class HistoryNetwork extends BaseNetwork {
476453
) {
477454
this.logger.extend('FOUNDCONTENT')(
478455
'Found value for requested content ' +
479-
bytesToHex(decodedContentMessage.contentKey) +
480-
' ' +
481-
bytesToHex(value.slice(0, 10)) +
482-
'...',
456+
bytesToHex(decodedContentMessage.contentKey) +
457+
' ' +
458+
bytesToHex(value.slice(0, 10)) +
459+
'...',
483460
)
484461
const payload = ContentMessageType.serialize({
485462
selector: FoundContent.CONTENT,
@@ -591,7 +568,7 @@ export class HistoryNetwork extends BaseNetwork {
591568
break
592569
}
593570

594-
case HistoryNetworkContentType.EphemeralHeader: {
571+
case HistoryNetworkContentType.EphemeralHeaderFindContent: {
595572
const payload = EphemeralHeaderPayload.deserialize(value)
596573
if (payload.length === 0) {
597574
this.logger.extend('STORE')('Received empty ephemeral header payload')
@@ -639,6 +616,23 @@ export class HistoryNetwork extends BaseNetwork {
639616
return
640617
}
641618
}
619+
case HistoryNetworkContentType.EphemeralHeaderOffer: {
620+
const payload = EphemeralHeaderPayload.deserialize(value)
621+
if (payload.length === 0) {
622+
this.logger.extend('STORE')('Received empty ephemeral header payload')
623+
return
624+
}
625+
const header = createBlockHeaderFromRLP(payload[0], { setHardfork: true })
626+
// Check if we already have this header
627+
if (this.ephemeralHeaderIndex.getByValue(bytesToHex(header.hash())) !== undefined) {
628+
this.logger.extend('STORE')(`Ephemeral header ${bytesToHex(header.hash())} already exists`)
629+
return
630+
}
631+
const hashKey = getEphemeralHeaderDbKey(header.hash())
632+
await this.put(hashKey, bytesToHex(payload[0]))
633+
this.ephemeralHeaderIndex.set(header.number, bytesToHex(header.hash()))
634+
break
635+
}
642636
}
643637

644638
this.emit('ContentAdded', contentKey, value)
@@ -649,8 +643,7 @@ export class HistoryNetwork extends BaseNetwork {
649643
}
650644
}
651645
this.logger(
652-
`${HistoryNetworkContentType[contentType]} added for ${
653-
keyOpt instanceof Uint8Array ? bytesToHex(keyOpt) : keyOpt
646+
`${HistoryNetworkContentType[contentType]} added for ${keyOpt instanceof Uint8Array ? bytesToHex(keyOpt) : keyOpt
654647
}`,
655648
)
656649
}
@@ -710,4 +703,39 @@ export class HistoryNetwork extends BaseNetwork {
710703
}
711704
return block.header.stateRoot
712705
}
706+
707+
/**
708+
* Assembles an ephemeral header FINDCONTENT payload for a given block hash and ancestor count
709+
* @param blockHash - The hash of the block to assemble the payload for
710+
* @param ancestorCount - The number of ancestor headers to include in the payload
711+
* @returns The assembled ephemeral header payload
712+
*/
713+
public async assembleEphemeralHeadersPayload(blockHash: Uint8Array, ancestorCount: number): Promise<Uint8Array> {
714+
const headers: Uint8Array[] = []
715+
const header = await this.get(getEphemeralHeaderDbKey(blockHash))
716+
if (header === undefined) {
717+
throw new Error('Header not found')
718+
}
719+
this.logger.extend('FOUNDCONTENT').extend('EPHEMERALHEADERS')(`Found requested header for ${bytesToHex(blockHash)}`)
720+
headers.push(hexToBytes(header as PrefixedHexString))
721+
if (ancestorCount === 0) {
722+
return EphemeralHeaderPayload.serialize(headers)
723+
}
724+
let ancestorNumber = this.ephemeralHeaderIndex.getByValue(bytesToHex(blockHash))
725+
for (let i = 0; i < ancestorCount; i++) {
726+
if (ancestorNumber === undefined) {
727+
break
728+
}
729+
ancestorNumber--
730+
// TODO: Decide if this is safe or if we should retrieve each header from the DB and step back using the parent hash (which would be less efficient)
731+
const ancestorHashKey = getEphemeralHeaderDbKey(hexToBytes(this.ephemeralHeaderIndex.getByKey(ancestorNumber)! as PrefixedHexString))
732+
const ancestorHeader = await this.get(ancestorHashKey)
733+
if (ancestorHeader === undefined) {
734+
break
735+
}
736+
headers.push(hexToBytes(ancestorHeader as PrefixedHexString))
737+
}
738+
this.logger.extend('FOUNDCONTENT').extend('EPHEMERALHEADERS')(`Found ${headers.length - 1} ancestor headers out of ${ancestorCount} requested for ${bytesToHex(blockHash)}`)
739+
return EphemeralHeaderPayload.serialize(headers)
740+
}
713741
}

‎packages/portalnetwork/src/networks/history/types.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,9 @@ export enum HistoryNetworkContentType {
3737
BlockBody = 1,
3838
Receipt = 2,
3939
BlockHeaderByNumber = 3,
40-
EphemeralHeader = 4,
40+
EphemeralHeaderFindContent = 4,
41+
EphemeralHeaderOffer = 5,
42+
EphemeralHeader = 99, // using an arbitrarily high number to avoid potential conflicts with future content types
4143
}
4244
export enum HistoryNetworkRetrievalMechanism {
4345
BlockHeaderByHash = 0,
@@ -224,7 +226,7 @@ export const HistoricalSummariesBlockProofDeneb = new ContainerType({
224226
})
225227

226228
/** Ephemeral header types */
227-
export const EphemeralHeaderKey = new ContainerType({
229+
export const EphemeralHeaderFindContentKey = new ContainerType({
228230
blockHash: Bytes32Type,
229231
ancestorCount: new UintNumberType(1),
230232
})
@@ -239,3 +241,7 @@ export type EphemeralHeaderKeyValues = {
239241
blockHash: Uint8Array
240242
ancestorCount: number
241243
}
244+
245+
export const EphemeralHeaderOfferKey = new ContainerType({ blockHash: Bytes32Type })
246+
247+
export const EphemeralHeaderOfferPayload = new ContainerType({ header: BlockHeader })

‎packages/portalnetwork/src/networks/history/util.ts‎

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ import {
1717
BlockHeaderWithProof,
1818
BlockNumberKey,
1919
CAPELLA_ERA,
20-
EphemeralHeaderKey,
20+
EphemeralHeaderFindContentKey,
21+
EphemeralHeaderOfferKey,
2122
EpochAccumulator,
2223
HistoryNetworkContentType,
2324
MERGE_BLOCK,
@@ -86,18 +87,25 @@ export const getContentKey = (
8687
encodedKey = BlockHeaderByNumberKey(key)
8788
break
8889
}
89-
case HistoryNetworkContentType.EphemeralHeader: {
90+
case HistoryNetworkContentType.EphemeralHeaderFindContent: {
9091
if (typeof key !== 'object' || !('blockHash' in key) || !('ancestorCount' in key))
9192
throw new Error('block hash and ancestor count are required to generate contentKey')
9293
encodedKey = Uint8Array.from([
9394
contentType,
94-
...EphemeralHeaderKey.serialize({
95+
...EphemeralHeaderFindContentKey.serialize({
9596
blockHash: key.blockHash,
9697
ancestorCount: key.ancestorCount,
9798
}),
9899
])
99100
break
100101
}
102+
case HistoryNetworkContentType.EphemeralHeaderOffer: {
103+
encodedKey = Uint8Array.from([
104+
contentType,
105+
...EphemeralHeaderOfferKey.serialize({ blockHash: key as Uint8Array }),
106+
])
107+
break
108+
}
101109
default:
102110
throw new Error('unsupported content type')
103111
}
@@ -127,20 +135,25 @@ export const decodeHistoryNetworkContentKey = (
127135
contentKey: Uint8Array,
128136
):
129137
| {
130-
contentType:
131-
| HistoryNetworkContentType.BlockHeader
132-
| HistoryNetworkContentType.BlockBody
133-
| HistoryNetworkContentType.Receipt
134-
keyOpt: Uint8Array
135-
}
138+
contentType:
139+
| HistoryNetworkContentType.BlockHeader
140+
| HistoryNetworkContentType.BlockBody
141+
| HistoryNetworkContentType.Receipt
142+
| HistoryNetworkContentType.EphemeralHeaderOffer
143+
keyOpt: Uint8Array
144+
}
136145
| {
137-
contentType: HistoryNetworkContentType.BlockHeaderByNumber
138-
keyOpt: bigint
139-
}
146+
contentType: HistoryNetworkContentType.BlockHeaderByNumber
147+
keyOpt: bigint
148+
}
140149
| {
141-
contentType: HistoryNetworkContentType.EphemeralHeader
142-
keyOpt: EphemeralHeaderKeyValues
143-
} => {
150+
contentType: HistoryNetworkContentType.EphemeralHeaderFindContent
151+
keyOpt: EphemeralHeaderKeyValues
152+
| {
153+
contentType: HistoryNetworkContentType.EphemeralHeaderOffer
154+
keyOpt: Uint8Array
155+
}
156+
} => {
144157
const contentType: HistoryNetworkContentType = contentKey[0]
145158
switch (contentType) {
146159
case HistoryNetworkContentType.BlockHeaderByNumber: {
@@ -150,13 +163,14 @@ export const decodeHistoryNetworkContentKey = (
150163
keyOpt: blockNumber,
151164
}
152165
}
153-
case HistoryNetworkContentType.EphemeralHeader: {
154-
const key = EphemeralHeaderKey.deserialize(contentKey.slice(1))
166+
case HistoryNetworkContentType.EphemeralHeaderFindContent: {
167+
const key = EphemeralHeaderFindContentKey.deserialize(contentKey.slice(1))
155168
return {
156169
contentType,
157170
keyOpt: key,
158171
}
159172
}
173+
case HistoryNetworkContentType.EphemeralHeader: throw new Error('EphemeralHeader is only for internal use')
160174
default: {
161175
const blockHash = contentKey.slice(1)
162176
return {

‎packages/portalnetwork/src/networks/network.ts‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import {
99
concatBytes,
1010
fromAscii,
1111
hexToBytes,
12-
PrefixedHexString,
12+
type PrefixedHexString,
1313
randomBytes,
1414
} from '@ethereumjs/util'
1515
import { EventEmitter } from 'eventemitter3'
@@ -637,11 +637,11 @@ export abstract class BaseNetwork extends EventEmitter {
637637
const requestedKeys: Uint8Array[] =
638638
version === 0
639639
? contentKeys.filter(
640-
(n, idx) => (<AcceptMessage<0>>msg).contentKeys.get(idx) === true,
641-
)
640+
(n, idx) => (<AcceptMessage<0>>msg).contentKeys.get(idx) === true,
641+
)
642642
: contentKeys.filter(
643-
(n, idx) => (<AcceptMessage<1>>msg).contentKeys[idx] === AcceptCode.ACCEPT,
644-
)
643+
(n, idx) => (<AcceptMessage<1>>msg).contentKeys[idx] === AcceptCode.ACCEPT,
644+
)
645645
if (requestedKeys.length === 0) {
646646
// Don't start uTP stream if no content ACCEPTed
647647
this.logger.extend('ACCEPT')(`No content ACCEPTed by ${shortId(enr.nodeId)}`)
@@ -705,8 +705,7 @@ export abstract class BaseNetwork extends EventEmitter {
705705
version: Version,
706706
) => {
707707
this.logger.extend('OFFER')(
708-
`Received from ${shortId(src.nodeId, this.routingTable)} with ${
709-
msg.contentKeys.length
708+
`Received from ${shortId(src.nodeId, this.routingTable)} with ${msg.contentKeys.length
710709
} pieces of content.`,
711710
)
712711
switch (version) {
@@ -881,8 +880,7 @@ export abstract class BaseNetwork extends EventEmitter {
881880
}
882881
await this.sendResponse(src, requestId, encodedPayload)
883882
this.logger.extend('ACCEPT')(
884-
`Sent to ${shortId(src.nodeId, this.routingTable)} for ${
885-
desiredContentKeys.length
883+
`Sent to ${shortId(src.nodeId, this.routingTable)} for ${desiredContentKeys.length
886884
} pieces of content. connectionId: ${id}`,
887885
)
888886
const enr = this.findEnr(src.nodeId) ?? src
@@ -918,10 +916,10 @@ export abstract class BaseNetwork extends EventEmitter {
918916
) {
919917
this.logger(
920918
'Found value for requested content ' +
921-
bytesToHex(decodedContentMessage.contentKey) +
922-
' ' +
923-
bytesToHex(value.slice(0, 10)) +
924-
'...',
919+
bytesToHex(decodedContentMessage.contentKey) +
920+
' ' +
921+
bytesToHex(value.slice(0, 10)) +
922+
'...',
925923
)
926924
const payload = ContentMessageType.serialize({
927925
selector: 1,
@@ -995,7 +993,7 @@ export abstract class BaseNetwork extends EventEmitter {
995993
while (
996994
encodedEnrs.length > 0 &&
997995
arrayByteLength(encodedEnrs) >
998-
MAX_UDP_PACKET_SIZE - getTalkReqOverhead(hexToBytes(this.networkId).byteLength)
996+
MAX_UDP_PACKET_SIZE - getTalkReqOverhead(hexToBytes(this.networkId).byteLength)
999997
) {
1000998
// Remove ENRs until total ENRs less than 1200 bytes
1001999
encodedEnrs.pop()

0 commit comments

Comments
 (0)