Skip to content

Commit 0a896f2

Browse files
g11techg11tech
committed
add data column to types repo and network
1 parent 04400ae commit 0a896f2

23 files changed

+510
-10
lines changed

‎packages/beacon-node/src/chain/errors/dataColumnSidecarError.ts

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import {Slot} from "@lodestar/types";
2+
import {GossipActionError} from "./gossipValidation.js";
3+
4+
export enum DataColumnSidecarErrorCode {
5+
INVALID_INDEX = "BLOB_SIDECAR_ERROR_INVALID_INDEX",
6+
7+
// following errors are adapted from the block errors
8+
FUTURE_SLOT = "BLOB_SIDECAR_ERROR_FUTURE_SLOT",
9+
}
10+
11+
export type DataColumnSidecarErrorType =
12+
| {code: DataColumnSidecarErrorCode.INVALID_INDEX; columnIndex: number; gossipIndex: number}
13+
| {code: DataColumnSidecarErrorCode.FUTURE_SLOT; blockSlot: Slot; currentSlot: Slot};
14+
15+
export class DataColumnSidecarGossipError extends GossipActionError<DataColumnSidecarErrorType> {}

‎packages/beacon-node/src/chain/validation/dataColumnSidecar.ts

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import {
2+
KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH,
3+
KZG_COMMITMENTS_SUBTREE_INDEX,
4+
DATA_COLUMN_SIDECAR_SUBNET_COUNT,
5+
} from "lodestar/params";
6+
import {electra, ssz} from "@lodestar/types";
7+
import {verifyMerkleBranch} from "@lodestar/utils";
8+
9+
import {DataColumnSidecarGossipError, DataColumnSidecarErrorCode} from "../errors/dataColumnSidecarError.js";
10+
import {GossipAction} from "../errors/gossipValidation.js";
11+
import {IBeaconChain} from "../interface.js";
12+
13+
export async function validateGossipDataColumnSidecar(
14+
chain: IBeaconChain,
15+
dataColumnSideCar: electra.DataColumnSidecar,
16+
gossipIndex: number
17+
): Promise<void> {
18+
const dataColumnSlot = dataColumnSideCar.signedBlockHeader.message.slot;
19+
20+
if (dataColumnSideCar.index % DATA_COLUMN_SIDECAR_SUBNET_COUNT !== gossipIndex) {
21+
throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
22+
code: DataColumnSidecarErrorCode.INVALID_INDEX,
23+
columnIndex: dataColumnSideCar.index,
24+
gossipIndex,
25+
});
26+
}
27+
28+
// [IGNORE] The sidecar is not from a future slot (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) --
29+
// i.e. validate that sidecar.slot <= current_slot (a client MAY queue future blocks for processing at
30+
// the appropriate slot).
31+
const currentSlotWithGossipDisparity = chain.clock.currentSlotWithGossipDisparity;
32+
if (currentSlotWithGossipDisparity < dataColumnSlot) {
33+
throw new DataColumnSidecarGossipError(GossipAction.IGNORE, {
34+
code: DataColumnSidecarErrorCode.FUTURE_SLOT,
35+
currentSlot: currentSlotWithGossipDisparity,
36+
blockSlot: dataColumnSlot,
37+
});
38+
}
39+
40+
validateInclusionProof(dataColumnSideCar);
41+
}
42+
43+
function validateInclusionProof(dataColumnSideCar: electra.DataColumnSidecar): boolean {
44+
return verifyMerkleBranch(
45+
ssz.deneb.BlobKzgCommitments.hashTreeRoot(dataColumnSideCar.kzgCommitments),
46+
dataColumnSideCar.kzgCommitmentsInclusionProof,
47+
KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH,
48+
KZG_COMMITMENTS_SUBTREE_INDEX,
49+
dataColumnSideCar.signedBlockHeader.message.bodyRoot
50+
);
51+
}

‎packages/beacon-node/src/db/buckets.ts

+3
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ export enum Bucket {
6161
// 54 was for bestPartialLightClientUpdate, allocate a fresh one
6262
// lightClient_bestLightClientUpdate = 55, // SyncPeriod -> LightClientUpdate // DEPRECATED on v1.5.0
6363
lightClient_bestLightClientUpdate = 56, // SyncPeriod -> [Slot, LightClientUpdate]
64+
65+
allForks_dataColumnSidecars = 57, // ELECTRA BeaconBlockRoot -> DataColumnSidecars
66+
allForks_dataColumnSidecarsArchive = 58, // ELECTRA BeaconBlockSlot -> DataColumnSidecars
6467
}
6568

6669
export function getBucketNameByValue<T extends Bucket>(enumValue: T): keyof typeof Bucket {

‎packages/beacon-node/src/db/repositories/dataColumnSidecars.ts

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import {ValueOf, ContainerType, ByteVectorType} from "@chainsafe/ssz";
2+
import {ChainForkConfig} from "@lodestar/config";
3+
import {Db, Repository} from "@lodestar/db";
4+
import {ssz} from "@lodestar/types";
5+
import {MAX_BLOB_COMMITMENTS_PER_BLOCK, NUMBER_OF_COLUMNS} from "@lodestar/params";
6+
7+
import {Bucket, getBucketNameByValue} from "../buckets.js";
8+
9+
export const dataColumnSidecarsWrapperSsz = new ContainerType(
10+
{
11+
blockRoot: ssz.Root,
12+
slot: ssz.Slot,
13+
columnsSize: ssz.Uint8,
14+
// each byte[i] tells what index (1 based) the column i is stored, 0 means not custodied
15+
custodyColumns: new ByteVectorType(NUMBER_OF_COLUMNS),
16+
dataColumnSidecars: ssz.electra.DataColumnSidecars,
17+
},
18+
{typeName: "DataColumnSidecarsWrapper", jsonCase: "eth2"}
19+
);
20+
21+
export type DataColumnSidecarsWrapper = ValueOf<typeof dataColumnSidecarsWrapperSsz>;
22+
export const COLUMN_SIZE_IN_WRAPPER_INDEX = 44;
23+
export const CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX = 45;
24+
export const DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX = 46 + MAX_BLOB_COMMITMENTS_PER_BLOCK;
25+
26+
/**
27+
* dataColumnSidecarsWrapper by block root (= hash_tree_root(SignedBeaconBlock.message))
28+
*
29+
* Used to store unfinalized DataColumnSidecars
30+
*/
31+
export class DataColumnSidecarsRepository extends Repository<Uint8Array, DataColumnSidecarsWrapper> {
32+
constructor(config: ChainForkConfig, db: Db) {
33+
const bucket = Bucket.allForks_dataColumnSidecars;
34+
super(config, db, bucket, dataColumnSidecarsWrapperSsz, getBucketNameByValue(bucket));
35+
}
36+
37+
/**
38+
* Id is hashTreeRoot of unsigned BeaconBlock
39+
*/
40+
getId(value: DataColumnSidecarsWrapper): Uint8Array {
41+
const {blockRoot} = value;
42+
return blockRoot;
43+
}
44+
}

‎packages/beacon-node/src/db/repositories/dataColumnSidecarsArchive.ts

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
import {ChainForkConfig} from "@lodestar/config";
2+
import {Db, Repository} from "@lodestar/db";
3+
import {Slot} from "@lodestar/types";
4+
import {bytesToInt} from "@lodestar/utils";
5+
import {Bucket, getBucketNameByValue} from "../buckets.js";
6+
import {dataColumnSidecarsWrapperSsz, DataColumnSidecarsWrapper} from "./dataColumnSidecars.js";
7+
8+
/**
9+
* dataColumnSidecarsWrapper by slot
10+
*
11+
* Used to store finalized DataColumnSidecars
12+
*/
13+
export class BlobSidecarsArchiveRepository extends Repository<Slot, DataColumnSidecarsWrapper> {
14+
constructor(config: ChainForkConfig, db: Db) {
15+
const bucket = Bucket.allForks_dataColumnSidecarsArchive;
16+
super(config, db, bucket, dataColumnSidecarsWrapperSsz, getBucketNameByValue(bucket));
17+
}
18+
19+
// Handle key as slot
20+
21+
getId(value: DataColumnSidecarsWrapper): Slot {
22+
return value.slot;
23+
}
24+
25+
decodeKey(data: Uint8Array): number {
26+
return bytesToInt(super.decodeKey(data) as unknown as Uint8Array, "be");
27+
}
28+
}

‎packages/beacon-node/src/network/gossip/interface.ts

+5-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ import {Libp2p} from "libp2p";
22
import {Message, TopicValidatorResult} from "@libp2p/interface";
33
import {PeerIdStr} from "@chainsafe/libp2p-gossipsub/types";
44
import {ForkName} from "@lodestar/params";
5-
import {allForks, altair, capella, deneb, phase0, Slot} from "@lodestar/types";
5+
import {allForks, altair, capella, deneb, phase0, Slot, electra} from "@lodestar/types";
66
import {BeaconConfig} from "@lodestar/config";
77
import {Logger} from "@lodestar/utils";
88
import {IBeaconChain} from "../../chain/index.js";
@@ -13,6 +13,7 @@ import {GossipActionError} from "../../chain/errors/gossipValidation.js";
1313
export enum GossipType {
1414
beacon_block = "beacon_block",
1515
blob_sidecar = "blob_sidecar",
16+
data_column_sidecar = "data_column_sidecar",
1617
beacon_aggregate_and_proof = "beacon_aggregate_and_proof",
1718
beacon_attestation = "beacon_attestation",
1819
voluntary_exit = "voluntary_exit",
@@ -41,6 +42,7 @@ export interface IGossipTopic {
4142
export type GossipTopicTypeMap = {
4243
[GossipType.beacon_block]: {type: GossipType.beacon_block};
4344
[GossipType.blob_sidecar]: {type: GossipType.blob_sidecar; index: number};
45+
[GossipType.data_column_sidecar]: {type: GossipType.data_column_sidecar; index: number};
4446
[GossipType.beacon_aggregate_and_proof]: {type: GossipType.beacon_aggregate_and_proof};
4547
[GossipType.beacon_attestation]: {type: GossipType.beacon_attestation; subnet: number};
4648
[GossipType.voluntary_exit]: {type: GossipType.voluntary_exit};
@@ -71,6 +73,7 @@ export type SSZTypeOfGossipTopic<T extends GossipTopic> = T extends {type: infer
7173
export type GossipTypeMap = {
7274
[GossipType.beacon_block]: allForks.SignedBeaconBlock;
7375
[GossipType.blob_sidecar]: deneb.BlobSidecar;
76+
[GossipType.data_column_sidecar]: electra.DataColumnSidecar;
7477
[GossipType.beacon_aggregate_and_proof]: phase0.SignedAggregateAndProof;
7578
[GossipType.beacon_attestation]: phase0.Attestation;
7679
[GossipType.voluntary_exit]: phase0.SignedVoluntaryExit;
@@ -86,6 +89,7 @@ export type GossipTypeMap = {
8689
export type GossipFnByType = {
8790
[GossipType.beacon_block]: (signedBlock: allForks.SignedBeaconBlock) => Promise<void> | void;
8891
[GossipType.blob_sidecar]: (blobSidecar: deneb.BlobSidecar) => Promise<void> | void;
92+
[GossipType.data_column_sidecar]: (blobSidecar: electra.DataColumnSidecar) => Promise<void> | void;
8993
[GossipType.beacon_aggregate_and_proof]: (aggregateAndProof: phase0.SignedAggregateAndProof) => Promise<void> | void;
9094
[GossipType.beacon_attestation]: (attestation: phase0.Attestation) => Promise<void> | void;
9195
[GossipType.voluntary_exit]: (voluntaryExit: phase0.SignedVoluntaryExit) => Promise<void> | void;

‎packages/beacon-node/src/network/gossip/topic.ts

+20
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
SYNC_COMMITTEE_SUBNET_COUNT,
88
isForkLightClient,
99
MAX_BLOBS_PER_BLOCK,
10+
DATA_COLUMN_SIDECAR_SUBNET_COUNT,
1011
} from "@lodestar/params";
1112

1213
import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
@@ -75,6 +76,8 @@ function stringifyGossipTopicType(topic: GossipTopic): string {
7576
return `${topic.type}_${topic.subnet}`;
7677
case GossipType.blob_sidecar:
7778
return `${topic.type}_${topic.index}`;
79+
case GossipType.data_column_sidecar:
80+
return `${topic.type}_${topic.index}`;
7881
}
7982
}
8083

@@ -86,6 +89,8 @@ export function getGossipSSZType(topic: GossipTopic) {
8689
return ssz[topic.fork].SignedBeaconBlock;
8790
case GossipType.blob_sidecar:
8891
return ssz.deneb.BlobSidecar;
92+
case GossipType.data_column_sidecar:
93+
return ssz.electra.DataColumnSidecar;
8994
case GossipType.beacon_aggregate_and_proof:
9095
return ssz.phase0.SignedAggregateAndProof;
9196
case GossipType.beacon_attestation:
@@ -189,6 +194,13 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr:
189194
return {type: GossipType.blob_sidecar, index, fork, encoding};
190195
}
191196

197+
if (gossipTypeStr.startsWith(GossipType.data_column_sidecar)) {
198+
const indexStr = gossipTypeStr.slice(GossipType.data_column_sidecar.length + 1); // +1 for '_' concatenating the topic name and the index
199+
const index = parseInt(indexStr, 10);
200+
if (Number.isNaN(index)) throw Error(`index ${indexStr} is not a number`);
201+
return {type: GossipType.data_column_sidecar, index, fork, encoding};
202+
}
203+
192204
throw Error(`Unknown gossip type ${gossipTypeStr}`);
193205
} catch (e) {
194206
(e as Error).message = `Invalid gossip topic ${topicStr}: ${(e as Error).message}`;
@@ -212,6 +224,13 @@ export function getCoreTopicsAtFork(
212224
{type: GossipType.attester_slashing},
213225
];
214226

227+
// After Electra also track data_column_sidecar_{index}
228+
if (ForkSeq[fork] >= ForkSeq.electra) {
229+
for (let index = 0; index < DATA_COLUMN_SIDECAR_SUBNET_COUNT; index++) {
230+
topics.push({type: GossipType.data_column_sidecar, index});
231+
}
232+
}
233+
215234
// After Deneb also track blob_sidecar_{index}
216235
if (ForkSeq[fork] >= ForkSeq.deneb) {
217236
for (let index = 0; index < MAX_BLOBS_PER_BLOCK; index++) {
@@ -262,6 +281,7 @@ function parseEncodingStr(encodingStr: string): GossipEncoding {
262281
export const gossipTopicIgnoreDuplicatePublishError: Record<GossipType, boolean> = {
263282
[GossipType.beacon_block]: true,
264283
[GossipType.blob_sidecar]: true,
284+
[GossipType.data_column_sidecar]: true,
265285
[GossipType.beacon_aggregate_and_proof]: true,
266286
[GossipType.beacon_attestation]: true,
267287
[GossipType.voluntary_exit]: true,

‎packages/beacon-node/src/network/interface.ts

+10-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import {
1616
import type {AddressManager, ConnectionManager, Registrar, TransportManager} from "@libp2p/interface-internal";
1717
import type {Datastore} from "interface-datastore";
1818
import {Identify} from "@chainsafe/libp2p-identify";
19-
import {Slot, SlotRootHex, allForks, altair, capella, deneb, phase0} from "@lodestar/types";
19+
import {Slot, SlotRootHex, allForks, altair, capella, deneb, phase0, electra} from "@lodestar/types";
2020
import {PeerIdStr} from "../util/peerId.js";
2121
import {INetworkEventBus} from "./events.js";
2222
import {INetworkCorePublic} from "./core/types.js";
@@ -57,10 +57,19 @@ export interface INetwork extends INetworkCorePublic {
5757
): Promise<WithBytes<allForks.SignedBeaconBlock>[]>;
5858
sendBlobSidecarsByRange(peerId: PeerIdStr, request: deneb.BlobSidecarsByRangeRequest): Promise<deneb.BlobSidecar[]>;
5959
sendBlobSidecarsByRoot(peerId: PeerIdStr, request: deneb.BlobSidecarsByRootRequest): Promise<deneb.BlobSidecar[]>;
60+
sendDataColumnSidecarsByRange(
61+
peerId: PeerIdStr,
62+
request: electra.DataColumnSidecarsByRangeRequest
63+
): Promise<electra.DataColumnSidecar[]>;
64+
sendDataColumnSidecarsByRoot(
65+
peerId: PeerIdStr,
66+
request: electra.DataColumnsSidecarByRootRequest
67+
): Promise<electra.DataColumnSidecar[]>;
6068

6169
// Gossip
6270
publishBeaconBlock(signedBlock: allForks.SignedBeaconBlock): Promise<number>;
6371
publishBlobSidecar(blobSidecar: deneb.BlobSidecar): Promise<number>;
72+
publishDataColumnSidecar(dataColumnSideCar: electra.DataColumnSidecar): Promise<number>;
6473
publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise<number>;
6574
publishBeaconAttestation(attestation: phase0.Attestation, subnet: number): Promise<number>;
6675
publishVoluntaryExit(voluntaryExit: phase0.SignedVoluntaryExit): Promise<number>;

‎packages/beacon-node/src/network/network.ts

+25-2
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ import {BeaconConfig} from "@lodestar/config";
55
import {sleep} from "@lodestar/utils";
66
import {LoggerNode} from "@lodestar/logger/node";
77
import {computeStartSlotAtEpoch, computeTimeAtSlot} from "@lodestar/state-transition";
8-
import {phase0, allForks, deneb, altair, Root, capella, SlotRootHex} from "@lodestar/types";
8+
import {phase0, allForks, deneb, altair, Root, capella, SlotRootHex, electra} from "@lodestar/types";
99
import {routes} from "@lodestar/api";
1010
import {ResponseIncoming} from "@lodestar/reqresp";
11-
import {ForkSeq, MAX_BLOBS_PER_BLOCK} from "@lodestar/params";
11+
import {ForkSeq, MAX_BLOBS_PER_BLOCK, MAX_DATA_COLUMNS_PER_BLOCK} from "@lodestar/params";
1212
import {Metrics, RegistryMetricCreator} from "../metrics/index.js";
1313
import {IBeaconChain} from "../chain/index.js";
1414
import {IBeaconDb} from "../db/interface.js";
@@ -502,6 +502,29 @@ export class Network implements INetwork {
502502
);
503503
}
504504

505+
async sendDataColumnSidecarsByRange(
506+
peerId: PeerIdStr,
507+
request: electra.DataColumnSidecarsByRangeRequest
508+
): Promise<electra.DataColumnSidecar[]> {
509+
return collectMaxResponseTyped(
510+
this.sendReqRespRequest(peerId, ReqRespMethod.DataColumnSidecarsByRange, [Version.V1], request),
511+
// request's count represent the slots, so the actual max count received could be slots * blobs per slot
512+
request.count * MAX_DATA_COLUMNS_PER_BLOCK,
513+
responseSszTypeByMethod[ReqRespMethod.DataColumnSidecarsByRange]
514+
);
515+
}
516+
517+
async sendDataColumnSidecarsByRoot(
518+
peerId: PeerIdStr,
519+
request: electra.BlobSidecarsByRootRequest
520+
): Promise<electra.DataColumnSidecar[]> {
521+
return collectMaxResponseTyped(
522+
this.sendReqRespRequest(peerId, ReqRespMethod.DataColumnSidecarsByRoot, [Version.V1], request),
523+
request.length,
524+
responseSszTypeByMethod[ReqRespMethod.DataColumnSidecarsByRoot]
525+
);
526+
}
527+
505528
private sendReqRespRequest<Req>(
506529
peerId: PeerIdStr,
507530
method: ReqRespMethod,

‎packages/beacon-node/src/network/processor/extractSlotRootFns.ts

+9
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
getSlotFromSignedAggregateAndProofSerialized,
77
getSlotFromBlobSidecarSerialized,
88
getSlotFromSignedBeaconBlockSerialized,
9+
getSlotFromDataColumnSidecarSerialized,
910
} from "../../util/sszBytes.js";
1011
import {GossipType} from "../gossip/index.js";
1112
import {ExtractSlotRootFns} from "./types.js";
@@ -45,6 +46,14 @@ export function createExtractBlockSlotRootFns(): ExtractSlotRootFns {
4546
[GossipType.blob_sidecar]: (data: Uint8Array): SlotOptionalRoot | null => {
4647
const slot = getSlotFromBlobSidecarSerialized(data);
4748

49+
if (slot === null) {
50+
return null;
51+
}
52+
return {slot};
53+
},
54+
[GossipType.data_column_sidecar]: (data: Uint8Array): SlotOptionalRoot | null => {
55+
const slot = getSlotFromDataColumnSidecarSerialized(data);
56+
4857
if (slot === null) {
4958
return null;
5059
}

‎packages/beacon-node/src/network/processor/gossipHandlers.ts

+7
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,13 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler
408408
}
409409
},
410410

411+
[GossipType.data_column_sidecar]: async ({
412+
gossipData,
413+
topic,
414+
peerIdStr,
415+
seenTimestampSec,
416+
}: GossipHandlerParamGeneric<GossipType.data_column_sidecar>) => {},
417+
411418
[GossipType.beacon_aggregate_and_proof]: async ({
412419
gossipData,
413420
topic,

‎packages/beacon-node/src/network/processor/gossipQueues/index.ts

+5
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ const defaultGossipQueueOpts: {
3939
type: QueueType.FIFO,
4040
dropOpts: {type: DropType.count, count: 1},
4141
},
42+
[GossipType.data_column_sidecar]: {
43+
maxLength: 4096,
44+
type: QueueType.FIFO,
45+
dropOpts: {type: DropType.count, count: 1},
46+
},
4247
// lighthoue has aggregate_queue 4096 and unknown_block_aggregate_queue 1024, we use single queue
4348
[GossipType.beacon_aggregate_and_proof]: {
4449
maxLength: 5120,

‎packages/beacon-node/src/network/processor/index.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ type WorkOpts = {
6565
const executeGossipWorkOrderObj: Record<GossipType, WorkOpts> = {
6666
[GossipType.beacon_block]: {bypassQueue: true},
6767
[GossipType.blob_sidecar]: {bypassQueue: true},
68+
[GossipType.data_column_sidecar]: {bypassQueue: true},
6869
[GossipType.beacon_aggregate_and_proof]: {},
6970
[GossipType.voluntary_exit]: {},
7071
[GossipType.bls_to_execution_change]: {},
@@ -267,7 +268,12 @@ export class NetworkProcessor {
267268
});
268269
return;
269270
}
270-
if (slot === clockSlot && (topicType === GossipType.beacon_block || topicType === GossipType.blob_sidecar)) {
271+
if (
272+
slot === clockSlot &&
273+
(topicType === GossipType.beacon_block ||
274+
topicType === GossipType.blob_sidecar ||
275+
topicType === GossipType.data_column_sidecar)
276+
) {
271277
// in the worse case if the current slot block is not valid, this will be reset in the next slot
272278
this.isProcessingCurrentSlotBlock = true;
273279
}

0 commit comments

Comments
 (0)