From 5317b6100d1b8a7193f26d11d6c7dcb56a23e8a2 Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:11:10 +0200 Subject: [PATCH 01/13] feat!(share/proto): define proto --- share/shwap/pb/shwap.pb.go | 1346 +++++++++++++++++++++++++++++++----- share/shwap/pb/shwap.proto | 24 +- 2 files changed, 1193 insertions(+), 177 deletions(-) diff --git a/share/shwap/pb/shwap.pb.go b/share/shwap/pb/shwap.pb.go index 000bf78ca7..0c15cc07cd 100644 --- a/share/shwap/pb/shwap.pb.go +++ b/share/shwap/pb/shwap.pb.go @@ -186,8 +186,8 @@ func (m *Sample) GetProofType() AxisType { } type RowNamespaceData struct { - Shares []*Share `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` - Proof *pb.Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` + Shares []*Share `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` + Proof *Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` } func (m *RowNamespaceData) Reset() { *m = RowNamespaceData{} } @@ -230,13 +230,109 @@ func (m *RowNamespaceData) GetShares() []*Share { return nil } -func (m *RowNamespaceData) GetProof() *pb.Proof { +func (m *RowNamespaceData) GetProof() *Proof { if m != nil { return m.Proof } return nil } +type NamespaceData struct { + NamespaceData []*RowNamespaceData `protobuf:"bytes,1,rep,name=namespaceData,proto3" json:"namespaceData,omitempty"` +} + +func (m *NamespaceData) Reset() { *m = NamespaceData{} } +func (m *NamespaceData) String() string { return proto.CompactTextString(m) } +func (*NamespaceData) ProtoMessage() {} +func (*NamespaceData) Descriptor() ([]byte, []int) { + return fileDescriptor_9431653f3c9f0bcb, []int{3} +} +func (m *NamespaceData) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *NamespaceData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_NamespaceData.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *NamespaceData) XXX_Merge(src proto.Message) { + xxx_messageInfo_NamespaceData.Merge(m, src) +} +func (m *NamespaceData) XXX_Size() int { + return m.Size() +} +func (m *NamespaceData) XXX_DiscardUnknown() { + xxx_messageInfo_NamespaceData.DiscardUnknown(m) +} + +var xxx_messageInfo_NamespaceData proto.InternalMessageInfo + +func (m *NamespaceData) GetNamespaceData() []*RowNamespaceData { + if m != nil { + return m.NamespaceData + } + return nil +} + +type RangeNamespaceData struct { + Start int32 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` + RangeNamespaceData *NamespaceData `protobuf:"bytes,3,opt,name=rangeNamespaceData,proto3" json:"rangeNamespaceData,omitempty"` +} + +func (m *RangeNamespaceData) Reset() { *m = RangeNamespaceData{} } +func (m *RangeNamespaceData) String() string { return proto.CompactTextString(m) } +func (*RangeNamespaceData) ProtoMessage() {} +func (*RangeNamespaceData) Descriptor() ([]byte, []int) { + return fileDescriptor_9431653f3c9f0bcb, []int{4} +} +func (m *RangeNamespaceData) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RangeNamespaceData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RangeNamespaceData.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RangeNamespaceData) XXX_Merge(src proto.Message) { + xxx_messageInfo_RangeNamespaceData.Merge(m, src) +} +func (m *RangeNamespaceData) XXX_Size() int { + return m.Size() +} +func (m *RangeNamespaceData) XXX_DiscardUnknown() { + xxx_messageInfo_RangeNamespaceData.DiscardUnknown(m) +} + +var xxx_messageInfo_RangeNamespaceData proto.InternalMessageInfo + +func (m *RangeNamespaceData) GetStart() int32 { + if m != nil { + return m.Start + } + return 0 +} + +func (m *RangeNamespaceData) GetRangeNamespaceData() *NamespaceData { + if m != nil { + return m.RangeNamespaceData + } + return nil +} + type Share struct { Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` } @@ -245,7 +341,7 @@ func (m *Share) Reset() { *m = Share{} } func (m *Share) String() string { return proto.CompactTextString(m) } func (*Share) ProtoMessage() {} func (*Share) Descriptor() ([]byte, []int) { - return fileDescriptor_9431653f3c9f0bcb, []int{3} + return fileDescriptor_9431653f3c9f0bcb, []int{5} } func (m *Share) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -281,43 +377,185 @@ func (m *Share) GetData() []byte { return nil } +type RowRootProof struct { + Total int64 `protobuf:"varint,1,opt,name=total,proto3" json:"total,omitempty"` + Index int64 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"` + LeafHash []byte `protobuf:"bytes,3,opt,name=leafHash,proto3" json:"leafHash,omitempty"` + Aunts [][]byte `protobuf:"bytes,4,rep,name=aunts,proto3" json:"aunts,omitempty"` +} + +func (m *RowRootProof) Reset() { *m = RowRootProof{} } +func (m *RowRootProof) String() string { return proto.CompactTextString(m) } +func (*RowRootProof) ProtoMessage() {} +func (*RowRootProof) Descriptor() ([]byte, []int) { + return fileDescriptor_9431653f3c9f0bcb, []int{6} +} +func (m *RowRootProof) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RowRootProof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RowRootProof.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RowRootProof) XXX_Merge(src proto.Message) { + xxx_messageInfo_RowRootProof.Merge(m, src) +} +func (m *RowRootProof) XXX_Size() int { + return m.Size() +} +func (m *RowRootProof) XXX_DiscardUnknown() { + xxx_messageInfo_RowRootProof.DiscardUnknown(m) +} + +var xxx_messageInfo_RowRootProof proto.InternalMessageInfo + +func (m *RowRootProof) GetTotal() int64 { + if m != nil { + return m.Total + } + return 0 +} + +func (m *RowRootProof) GetIndex() int64 { + if m != nil { + return m.Index + } + return 0 +} + +func (m *RowRootProof) GetLeafHash() []byte { + if m != nil { + return m.LeafHash + } + return nil +} + +func (m *RowRootProof) GetAunts() [][]byte { + if m != nil { + return m.Aunts + } + return nil +} + +type Proof struct { + SharesProof *pb.Proof `protobuf:"bytes,1,opt,name=sharesProof,proto3" json:"sharesProof,omitempty"` + RowRootProof *RowRootProof `protobuf:"bytes,2,opt,name=rowRootProof,proto3" json:"rowRootProof,omitempty"` + Root []byte `protobuf:"bytes,3,opt,name=root,proto3" json:"root,omitempty"` +} + +func (m *Proof) Reset() { *m = Proof{} } +func (m *Proof) String() string { return proto.CompactTextString(m) } +func (*Proof) ProtoMessage() {} +func (*Proof) Descriptor() ([]byte, []int) { + return fileDescriptor_9431653f3c9f0bcb, []int{7} +} +func (m *Proof) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Proof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Proof.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Proof) XXX_Merge(src proto.Message) { + xxx_messageInfo_Proof.Merge(m, src) +} +func (m *Proof) XXX_Size() int { + return m.Size() +} +func (m *Proof) XXX_DiscardUnknown() { + xxx_messageInfo_Proof.DiscardUnknown(m) +} + +var xxx_messageInfo_Proof proto.InternalMessageInfo + +func (m *Proof) GetSharesProof() *pb.Proof { + if m != nil { + return m.SharesProof + } + return nil +} + +func (m *Proof) GetRowRootProof() *RowRootProof { + if m != nil { + return m.RowRootProof + } + return nil +} + +func (m *Proof) GetRoot() []byte { + if m != nil { + return m.Root + } + return nil +} + func init() { proto.RegisterEnum("shwap.AxisType", AxisType_name, AxisType_value) proto.RegisterEnum("shwap.Row_HalfSide", Row_HalfSide_name, Row_HalfSide_value) proto.RegisterType((*Row)(nil), "shwap.Row") proto.RegisterType((*Sample)(nil), "shwap.Sample") proto.RegisterType((*RowNamespaceData)(nil), "shwap.RowNamespaceData") + proto.RegisterType((*NamespaceData)(nil), "shwap.NamespaceData") + proto.RegisterType((*RangeNamespaceData)(nil), "shwap.RangeNamespaceData") proto.RegisterType((*Share)(nil), "shwap.Share") + proto.RegisterType((*RowRootProof)(nil), "shwap.RowRootProof") + proto.RegisterType((*Proof)(nil), "shwap.Proof") } func init() { proto.RegisterFile("share/shwap/pb/shwap.proto", fileDescriptor_9431653f3c9f0bcb) } var fileDescriptor_9431653f3c9f0bcb = []byte{ - // 381 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x4f, 0x6b, 0xe2, 0x40, - 0x18, 0xc6, 0x33, 0x1b, 0xe3, 0xc6, 0x57, 0xd1, 0x30, 0x7b, 0x09, 0xee, 0x92, 0x95, 0xb0, 0x0b, - 0xb2, 0x60, 0xb2, 0xe8, 0x27, 0xd8, 0xbf, 0xb5, 0x60, 0x6b, 0x19, 0x85, 0x42, 0x2f, 0x61, 0x62, - 0x46, 0x13, 0x88, 0x9d, 0x21, 0x49, 0x49, 0x3d, 0xf7, 0xd0, 0x6b, 0x3f, 0x56, 0x8f, 0x1e, 0x7b, - 0x2c, 0xfa, 0x45, 0x4a, 0x26, 0xb1, 0x14, 0xda, 0x43, 0x6f, 0xbf, 0xcc, 0xf3, 0xcc, 0xbc, 0xcf, - 0x13, 0x5e, 0xe8, 0xa6, 0x21, 0x4d, 0x98, 0x9b, 0x86, 0x39, 0x15, 0xae, 0xf0, 0x4b, 0x70, 0x44, - 0xc2, 0x33, 0x8e, 0x35, 0xf9, 0xd1, 0x6d, 0x0b, 0xdf, 0x15, 0x09, 0xe7, 0xcb, 0xf2, 0xd8, 0xbe, - 0x45, 0xa0, 0x12, 0x9e, 0xe3, 0x01, 0x34, 0xe5, 0xe5, 0xd4, 0x0b, 0x69, 0xbc, 0x34, 0x51, 0x4f, - 0xed, 0x37, 0x87, 0x2d, 0xa7, 0x7c, 0x61, 0x56, 0x28, 0x04, 0x4a, 0xc3, 0x98, 0xc6, 0x4b, 0xfc, - 0x13, 0x1a, 0x85, 0xcf, 0x4b, 0xa3, 0x80, 0x99, 0x1f, 0x7a, 0xa8, 0xdf, 0x1e, 0x7e, 0xaa, 0xcc, - 0x84, 0xe7, 0x4e, 0xe1, 0x99, 0x45, 0x01, 0x23, 0x7a, 0x58, 0x91, 0xfd, 0x15, 0xf4, 0xc3, 0x29, - 0xd6, 0xa1, 0x36, 0xf9, 0xf7, 0x7f, 0x6e, 0x28, 0xb8, 0x01, 0x1a, 0x39, 0x3e, 0x1a, 0xcf, 0x0d, - 0x64, 0xdf, 0x20, 0xa8, 0xcf, 0xe8, 0x5a, 0xc4, 0x0c, 0xdb, 0xa0, 0xc9, 0x59, 0x26, 0xea, 0xa1, - 0x57, 0x31, 0x4a, 0x09, 0x7f, 0x07, 0x4d, 0xf6, 0x90, 0xd3, 0x9b, 0xc3, 0x8e, 0x53, 0xb5, 0xf2, - 0x9d, 0xb3, 0x02, 0x48, 0xa9, 0x62, 0x07, 0x40, 0x82, 0x97, 0x6d, 0x04, 0x33, 0x55, 0x99, 0xb4, - 0x53, 0xbd, 0xf7, 0xeb, 0x3a, 0x4a, 0xe7, 0x1b, 0xc1, 0x48, 0x43, 0x5a, 0x0a, 0xb4, 0x3d, 0x30, - 0x08, 0xcf, 0x4f, 0xe9, 0x9a, 0xa5, 0x82, 0x2e, 0xd8, 0x5f, 0x9a, 0x51, 0xfc, 0x0d, 0xea, 0x65, - 0xf5, 0x37, 0x7f, 0x4b, 0xa5, 0xbd, 0x33, 0x90, 0xfd, 0x19, 0x34, 0x79, 0x0f, 0x63, 0xa8, 0x05, - 0x34, 0xa3, 0xb2, 0x63, 0x8b, 0x48, 0xfe, 0xf1, 0x05, 0xf4, 0x43, 0x28, 0xfc, 0x11, 0x54, 0x32, - 0x3d, 0x37, 0x94, 0x02, 0xfe, 0x4c, 0x27, 0x06, 0xfa, 0x7d, 0x72, 0xbf, 0xb3, 0xd0, 0x76, 0x67, - 0xa1, 0xc7, 0x9d, 0x85, 0xee, 0xf6, 0x96, 0xb2, 0xdd, 0x5b, 0xca, 0xc3, 0xde, 0x52, 0x2e, 0x46, - 0xab, 0x28, 0x0b, 0xaf, 0x7c, 0x67, 0xc1, 0xd7, 0xee, 0x82, 0xc5, 0x2c, 0xcd, 0x22, 0xca, 0x93, - 0xd5, 0x33, 0x0f, 0x2e, 0x79, 0x50, 0xec, 0xc5, 0xcb, 0xed, 0xf0, 0xeb, 0x72, 0x03, 0x46, 0x4f, - 0x01, 0x00, 0x00, 0xff, 0xff, 0x67, 0xb6, 0xc0, 0x8b, 0x36, 0x02, 0x00, 0x00, + // 541 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x53, 0x41, 0x8f, 0xd2, 0x40, + 0x14, 0x66, 0x2c, 0x45, 0x78, 0x74, 0x77, 0xc9, 0xb8, 0x89, 0x04, 0x4d, 0x25, 0x8d, 0x26, 0xc4, + 0x64, 0x8b, 0xb2, 0x07, 0x4f, 0x1e, 0xd4, 0x55, 0x31, 0x59, 0x77, 0xcd, 0x83, 0xc4, 0xc4, 0x98, + 0x90, 0x01, 0x06, 0xda, 0xa4, 0x30, 0x4d, 0x3b, 0x1b, 0x76, 0xcf, 0x1e, 0xbc, 0xfa, 0xb3, 0x3c, + 0xee, 0xd1, 0xa3, 0x81, 0x3f, 0x62, 0x66, 0xa6, 0x60, 0x1b, 0xf6, 0xf6, 0x7d, 0xef, 0x7d, 0x7d, + 0xef, 0x7b, 0xaf, 0x6f, 0xa0, 0x95, 0x06, 0x2c, 0xe1, 0xdd, 0x34, 0x58, 0xb1, 0xb8, 0x1b, 0x8f, + 0x0d, 0xf0, 0xe3, 0x44, 0x48, 0x41, 0x6d, 0x4d, 0x5a, 0x87, 0xf1, 0xb8, 0x1b, 0x27, 0x42, 0xcc, + 0x4c, 0xd8, 0xfb, 0x49, 0xc0, 0x42, 0xb1, 0xa2, 0x27, 0x50, 0xd7, 0x1f, 0xa7, 0xa3, 0x80, 0x45, + 0xb3, 0x26, 0x69, 0x5b, 0x9d, 0x7a, 0xcf, 0xf1, 0x4d, 0x85, 0x81, 0xca, 0x20, 0x18, 0x41, 0x9f, + 0x45, 0x33, 0xfa, 0x02, 0x6a, 0x4a, 0x37, 0x4a, 0xc3, 0x29, 0x6f, 0xde, 0x6b, 0x93, 0xce, 0x61, + 0xef, 0x41, 0x26, 0x46, 0xb1, 0xf2, 0x95, 0x66, 0x10, 0x4e, 0x39, 0x56, 0x83, 0x0c, 0x79, 0x4f, + 0xa0, 0xba, 0x8d, 0xd2, 0x2a, 0x94, 0xcf, 0xdf, 0x7f, 0x18, 0x36, 0x4a, 0xb4, 0x06, 0x36, 0x7e, + 0xfa, 0xd8, 0x1f, 0x36, 0x88, 0xf7, 0x83, 0x40, 0x65, 0xc0, 0x16, 0x71, 0xc4, 0xa9, 0x07, 0xb6, + 0xee, 0xd5, 0x24, 0x6d, 0xb2, 0x67, 0xc3, 0xa4, 0xe8, 0x33, 0xb0, 0xf5, 0x1c, 0xba, 0x7b, 0xbd, + 0x77, 0xe4, 0x67, 0x53, 0x8d, 0xfd, 0x2f, 0x0a, 0xa0, 0xc9, 0x52, 0x1f, 0x40, 0x83, 0x91, 0xbc, + 0x89, 0x79, 0xd3, 0xd2, 0x4e, 0x8f, 0xb2, 0x7a, 0x6f, 0xae, 0xc3, 0x74, 0x78, 0x13, 0x73, 0xac, + 0x69, 0x89, 0x82, 0xde, 0x77, 0x68, 0xa0, 0x58, 0x5d, 0xb0, 0x05, 0x4f, 0x63, 0x36, 0xe1, 0x67, + 0x4c, 0x32, 0xfa, 0x14, 0x2a, 0x66, 0xf4, 0x3b, 0xd7, 0x92, 0xe5, 0x94, 0xe9, 0xbc, 0xa1, 0xad, + 0x28, 0xef, 0xc6, 0xbb, 0x80, 0x83, 0x62, 0xe9, 0xd7, 0x70, 0xb0, 0xcc, 0x07, 0xb2, 0x0e, 0x0f, + 0xff, 0xef, 0xb2, 0xa0, 0xc7, 0xa2, 0xda, 0x8b, 0x81, 0x22, 0x5b, 0xce, 0x79, 0xb1, 0xe8, 0x31, + 0xd8, 0xa9, 0x64, 0x89, 0xd4, 0xeb, 0xb3, 0xd1, 0x10, 0x7a, 0x06, 0x34, 0xd9, 0xd3, 0xea, 0x8d, + 0xd4, 0x7b, 0xc7, 0x59, 0xbf, 0x62, 0xb3, 0x3b, 0xf4, 0xde, 0x23, 0xb0, 0xf5, 0xd8, 0x94, 0x42, + 0x79, 0x6a, 0x0c, 0x93, 0x8e, 0x83, 0x1a, 0x7b, 0x11, 0x38, 0x28, 0x56, 0x28, 0x84, 0xd4, 0x53, + 0x2b, 0x23, 0x52, 0x48, 0x16, 0x69, 0x91, 0x85, 0x86, 0xa8, 0x68, 0xb8, 0x9c, 0xf2, 0x6b, 0xbd, + 0x28, 0x0b, 0x0d, 0xa1, 0x2d, 0xa8, 0x46, 0x9c, 0xcd, 0xfa, 0x2c, 0x0d, 0xb4, 0x29, 0x07, 0x77, + 0x5c, 0x7d, 0xc1, 0xae, 0x96, 0x32, 0x6d, 0x96, 0xdb, 0x56, 0xc7, 0x41, 0x43, 0xd4, 0xe9, 0xda, + 0xa6, 0xcf, 0xcb, 0xed, 0xf1, 0x6a, 0x9a, 0x5d, 0xcd, 0xde, 0x45, 0xe4, 0x35, 0xf4, 0x15, 0x38, + 0x49, 0xce, 0x6a, 0xf6, 0xd3, 0x72, 0x37, 0xbc, 0x4b, 0x61, 0x41, 0xa8, 0xe6, 0x4e, 0x84, 0x90, + 0x99, 0x47, 0x8d, 0x9f, 0x3f, 0x86, 0xea, 0xf6, 0x96, 0xe8, 0x7d, 0xb0, 0xf0, 0xf2, 0x6b, 0xa3, + 0xa4, 0xc0, 0xbb, 0xcb, 0xf3, 0x06, 0x79, 0xfb, 0xf9, 0xf7, 0xda, 0x25, 0xb7, 0x6b, 0x97, 0xfc, + 0x5d, 0xbb, 0xe4, 0xd7, 0xc6, 0x2d, 0xdd, 0x6e, 0xdc, 0xd2, 0x9f, 0x8d, 0x5b, 0xfa, 0x76, 0x3a, + 0x0f, 0x65, 0x70, 0x35, 0xf6, 0x27, 0x62, 0xd1, 0x9d, 0xf0, 0x88, 0xa7, 0x32, 0x64, 0x22, 0x99, + 0xef, 0xf0, 0xc9, 0x52, 0x4c, 0xd5, 0x73, 0xce, 0x3f, 0xea, 0x71, 0x45, 0x3f, 0xdc, 0xd3, 0x7f, + 0x01, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xd2, 0x0a, 0xe0, 0xed, 0x03, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -463,6 +701,83 @@ func (m *RowNamespaceData) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *NamespaceData) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NamespaceData) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *NamespaceData) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.NamespaceData) > 0 { + for iNdEx := len(m.NamespaceData) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.NamespaceData[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwap(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *RangeNamespaceData) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RangeNamespaceData) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RangeNamespaceData) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.RangeNamespaceData != nil { + { + size, err := m.RangeNamespaceData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwap(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + if m.Start != 0 { + i = encodeVarintShwap(dAtA, i, uint64(m.Start)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *Share) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -493,44 +808,147 @@ func (m *Share) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func encodeVarintShwap(dAtA []byte, offset int, v uint64) int { - offset -= sovShwap(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ +func (m *RowRootProof) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } - dAtA[offset] = uint8(v) - return base + return dAtA[:n], nil } -func (m *Row) Size() (n int) { - if m == nil { - return 0 - } + +func (m *RowRootProof) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RowRootProof) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l - if len(m.SharesHalf) > 0 { - for _, e := range m.SharesHalf { - l = e.Size() - n += 1 + l + sovShwap(uint64(l)) + if len(m.Aunts) > 0 { + for iNdEx := len(m.Aunts) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Aunts[iNdEx]) + copy(dAtA[i:], m.Aunts[iNdEx]) + i = encodeVarintShwap(dAtA, i, uint64(len(m.Aunts[iNdEx]))) + i-- + dAtA[i] = 0x22 } } - if m.HalfSide != 0 { - n += 1 + sovShwap(uint64(m.HalfSide)) + if len(m.LeafHash) > 0 { + i -= len(m.LeafHash) + copy(dAtA[i:], m.LeafHash) + i = encodeVarintShwap(dAtA, i, uint64(len(m.LeafHash))) + i-- + dAtA[i] = 0x1a } - return n + if m.Index != 0 { + i = encodeVarintShwap(dAtA, i, uint64(m.Index)) + i-- + dAtA[i] = 0x10 + } + if m.Total != 0 { + i = encodeVarintShwap(dAtA, i, uint64(m.Total)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil } -func (m *Sample) Size() (n int) { - if m == nil { - return 0 +func (m *Proof) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } + return dAtA[:n], nil +} + +func (m *Proof) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Proof) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l - if m.Share != nil { - l = m.Share.Size() - n += 1 + l + sovShwap(uint64(l)) + if len(m.Root) > 0 { + i -= len(m.Root) + copy(dAtA[i:], m.Root) + i = encodeVarintShwap(dAtA, i, uint64(len(m.Root))) + i-- + dAtA[i] = 0x1a + } + if m.RowRootProof != nil { + { + size, err := m.RowRootProof.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwap(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + if m.SharesProof != nil { + { + size, err := m.SharesProof.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwap(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintShwap(dAtA []byte, offset int, v uint64) int { + offset -= sovShwap(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Row) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.SharesHalf) > 0 { + for _, e := range m.SharesHalf { + l = e.Size() + n += 1 + l + sovShwap(uint64(l)) + } + } + if m.HalfSide != 0 { + n += 1 + sovShwap(uint64(m.HalfSide)) + } + return n +} + +func (m *Sample) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Share != nil { + l = m.Share.Size() + n += 1 + l + sovShwap(uint64(l)) } if m.Proof != nil { l = m.Proof.Size() @@ -561,6 +979,37 @@ func (m *RowNamespaceData) Size() (n int) { return n } +func (m *NamespaceData) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.NamespaceData) > 0 { + for _, e := range m.NamespaceData { + l = e.Size() + n += 1 + l + sovShwap(uint64(l)) + } + } + return n +} + +func (m *RangeNamespaceData) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Start != 0 { + n += 1 + sovShwap(uint64(m.Start)) + } + if m.RangeNamespaceData != nil { + l = m.RangeNamespaceData.Size() + n += 1 + l + sovShwap(uint64(l)) + } + return n +} + func (m *Share) Size() (n int) { if m == nil { return 0 @@ -571,16 +1020,510 @@ func (m *Share) Size() (n int) { if l > 0 { n += 1 + l + sovShwap(uint64(l)) } - return n -} + return n +} + +func (m *RowRootProof) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Total != 0 { + n += 1 + sovShwap(uint64(m.Total)) + } + if m.Index != 0 { + n += 1 + sovShwap(uint64(m.Index)) + } + l = len(m.LeafHash) + if l > 0 { + n += 1 + l + sovShwap(uint64(l)) + } + if len(m.Aunts) > 0 { + for _, b := range m.Aunts { + l = len(b) + n += 1 + l + sovShwap(uint64(l)) + } + } + return n +} + +func (m *Proof) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.SharesProof != nil { + l = m.SharesProof.Size() + n += 1 + l + sovShwap(uint64(l)) + } + if m.RowRootProof != nil { + l = m.RowRootProof.Size() + n += 1 + l + sovShwap(uint64(l)) + } + l = len(m.Root) + if l > 0 { + n += 1 + l + sovShwap(uint64(l)) + } + return n +} + +func sovShwap(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozShwap(x uint64) (n int) { + return sovShwap(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Row) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Row: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Row: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SharesHalf", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SharesHalf = append(m.SharesHalf, &Share{}) + if err := m.SharesHalf[len(m.SharesHalf)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field HalfSide", wireType) + } + m.HalfSide = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.HalfSide |= Row_HalfSide(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShwap(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwap + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Sample) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Sample: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Sample: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Share", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Share == nil { + m.Share = &Share{} + } + if err := m.Share.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Proof", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Proof == nil { + m.Proof = &pb.Proof{} + } + if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProofType", wireType) + } + m.ProofType = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProofType |= AxisType(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipShwap(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwap + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RowNamespaceData) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RowNamespaceData: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RowNamespaceData: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Shares", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Shares = append(m.Shares, &Share{}) + if err := m.Shares[len(m.Shares)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Proof", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Proof == nil { + m.Proof = &Proof{} + } + if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShwap(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwap + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NamespaceData) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NamespaceData: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NamespaceData: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field NamespaceData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.NamespaceData = append(m.NamespaceData, &RowNamespaceData{}) + if err := m.NamespaceData[len(m.NamespaceData)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShwap(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwap + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } -func sovShwap(x uint64) (n int) { - return (math_bits.Len64(x|1) + 6) / 7 -} -func sozShwap(x uint64) (n int) { - return sovShwap(uint64((x << 1) ^ uint64((int64(x) >> 63)))) + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil } -func (m *Row) Unmarshal(dAtA []byte) error { +func (m *RangeNamespaceData) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -603,15 +1546,34 @@ func (m *Row) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Row: wiretype end group for non-group") + return fmt.Errorf("proto: RangeNamespaceData: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Row: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: RangeNamespaceData: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) + } + m.Start = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Start |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SharesHalf", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RangeNamespaceData", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -638,16 +1600,68 @@ func (m *Row) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.SharesHalf = append(m.SharesHalf, &Share{}) - if err := m.SharesHalf[len(m.SharesHalf)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if m.RangeNamespaceData == nil { + m.RangeNamespaceData = &NamespaceData{} + } + if err := m.RangeNamespaceData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field HalfSide", wireType) + default: + iNdEx = preIndex + skippy, err := skipShwap(dAtA[iNdEx:]) + if err != nil { + return err } - m.HalfSide = 0 + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwap + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Share) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Share: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Share: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShwap @@ -657,11 +1671,26 @@ func (m *Row) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.HalfSide |= Row_HalfSide(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } + if byteLen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipShwap(dAtA[iNdEx:]) @@ -683,7 +1712,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { } return nil } -func (m *Sample) Unmarshal(dAtA []byte) error { +func (m *RowRootProof) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -706,17 +1735,17 @@ func (m *Sample) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Sample: wiretype end group for non-group") + return fmt.Errorf("proto: RowRootProof: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Sample: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: RowRootProof: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Share", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Total", wireType) } - var msglen int + m.Total = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShwap @@ -726,33 +1755,35 @@ func (m *Sample) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + m.Total |= int64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { - return ErrInvalidLengthShwap - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthShwap - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Share == nil { - m.Share = &Share{} + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType) } - if err := m.Share.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err + m.Index = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Index |= int64(b&0x7F) << shift + if b < 0x80 { + break + } } - iNdEx = postIndex - case 2: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Proof", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field LeafHash", wireType) } - var msglen int + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShwap @@ -762,33 +1793,31 @@ func (m *Sample) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + if byteLen < 0 { return ErrInvalidLengthShwap } - postIndex := iNdEx + msglen + postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthShwap } if postIndex > l { return io.ErrUnexpectedEOF } - if m.Proof == nil { - m.Proof = &pb.Proof{} - } - if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err + m.LeafHash = append(m.LeafHash[:0], dAtA[iNdEx:postIndex]...) + if m.LeafHash == nil { + m.LeafHash = []byte{} } iNdEx = postIndex - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ProofType", wireType) + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Aunts", wireType) } - m.ProofType = 0 + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowShwap @@ -798,11 +1827,24 @@ func (m *Sample) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ProofType |= AxisType(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } + if byteLen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Aunts = append(m.Aunts, make([]byte, postIndex-iNdEx)) + copy(m.Aunts[len(m.Aunts)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipShwap(dAtA[iNdEx:]) @@ -824,7 +1866,7 @@ func (m *Sample) Unmarshal(dAtA []byte) error { } return nil } -func (m *RowNamespaceData) Unmarshal(dAtA []byte) error { +func (m *Proof) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -847,15 +1889,15 @@ func (m *RowNamespaceData) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: RowNamespaceData: wiretype end group for non-group") + return fmt.Errorf("proto: Proof: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: RowNamespaceData: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: Proof: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Shares", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field SharesProof", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -882,14 +1924,16 @@ func (m *RowNamespaceData) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Shares = append(m.Shares, &Share{}) - if err := m.Shares[len(m.Shares)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if m.SharesProof == nil { + m.SharesProof = &pb.Proof{} + } + if err := m.SharesProof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Proof", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RowRootProof", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -916,66 +1960,16 @@ func (m *RowNamespaceData) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.Proof == nil { - m.Proof = &pb.Proof{} + if m.RowRootProof == nil { + m.RowRootProof = &RowRootProof{} } - if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.RowRootProof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipShwap(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthShwap - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *Share) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShwap - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Share: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Share: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: + case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Root", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -1002,9 +1996,9 @@ func (m *Share) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) - if m.Data == nil { - m.Data = []byte{} + m.Root = append(m.Root[:0], dAtA[iNdEx:postIndex]...) + if m.Root == nil { + m.Root = []byte{} } iNdEx = postIndex default: diff --git a/share/shwap/pb/shwap.proto b/share/shwap/pb/shwap.proto index d7daea568a..8d40168ac9 100644 --- a/share/shwap/pb/shwap.proto +++ b/share/shwap/pb/shwap.proto @@ -23,7 +23,16 @@ message Sample { message RowNamespaceData { repeated Share shares = 1; - proof.pb.Proof proof = 2; + Proof proof = 2; +} + +message NamespaceData { + repeated RowNamespaceData namespaceData = 1; +} + +message RangeNamespaceData { + int32 start = 1; + NamespaceData rangeNamespaceData = 3; } message Share { @@ -34,3 +43,16 @@ enum AxisType { ROW = 0; COL = 1; } + +message RowRootProof { + int64 total = 1; + int64 index = 2; + bytes leafHash = 3; + repeated bytes aunts = 4; +} + +message Proof { + proof.pb.Proof sharesProof = 1; + RowRootProof rowRootProof = 2; + bytes root = 3; +} From d815b236f1a1a27e02122c14ce9376621752393a Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:27:57 +0200 Subject: [PATCH 02/13] feat(share/shwap): add proof type --- share/shwap/proof.go | 220 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 share/shwap/proof.go diff --git a/share/shwap/proof.go b/share/shwap/proof.go new file mode 100644 index 0000000000..051ae56118 --- /dev/null +++ b/share/shwap/proof.go @@ -0,0 +1,220 @@ +package shwap + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/tendermint/tendermint/crypto/merkle" + + libshare "github.com/celestiaorg/go-square/v2/share" + "github.com/celestiaorg/nmt" + nmt_pb "github.com/celestiaorg/nmt/pb" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/shwap/pb" +) + +// Proof represents a proof that a data share is included in a +// committed data root. +// It consists of multiple components to support verification: +// * shareProof: A nmt proof that verifies the inclusion of a specific data share within a row of the datasquare. +// * rowRootProof: A Merkle proof that verifies the inclusion of the row root within the final data root. +// * root: The row root against which the proof is verified +type Proof struct { + shareProof *nmt.Proof + rowRootProof *merkle.Proof + root []byte +} + +func NewProof(rowIndex int, sharesProofs *nmt.Proof, root *share.AxisRoots) *Proof { + proof := &Proof{shareProof: sharesProofs} + roots := append(root.RowRoots, root.ColumnRoots...) //nolint: gocritic + _, proofs := merkle.ProofsFromByteSlices(roots) + + proof.rowRootProof = proofs[rowIndex] + proof.root = roots[rowIndex] + return proof +} + +// SharesProof returns a shares inclusion proof. +func (p *Proof) SharesProof() *nmt.Proof { + return p.shareProof +} + +// RowRootProof returns a merkle proof of the row root to the data root. +func (p *Proof) RowRootProof() *merkle.Proof { + return p.rowRootProof +} + +// VerifyInclusion verifies the inclusion of the shares to the data root. +func (p *Proof) VerifyInclusion(shares []libshare.Share, namespace libshare.Namespace, dataRoot []byte) error { + outside, err := share.IsOutsideRange(namespace, p.root, p.root) + if err != nil { + return err + } + if outside { + return fmt.Errorf("namespace out of range") + } + + isValid := p.shareProof.VerifyInclusion( + share.NewSHA256Hasher(), + namespace.Bytes(), + libshare.ToBytes(shares), + p.root, + ) + if !isValid { + return errors.New("failed to verify nmt proof") + } + + err = p.rowRootProof.Verify(dataRoot, p.root) + if err != nil { + return err + } + return nil +} + +// VerifyNamespace verifies the whole namespace of the shares to the data root. +func (p *Proof) VerifyNamespace(shrs []libshare.Share, namespace libshare.Namespace, dataRoot []byte) error { + outside, err := share.IsOutsideRange(namespace, p.root, p.root) + if err != nil { + return err + } + if outside { + return fmt.Errorf("namespace out of range") + } + + leaves := make([][]byte, 0, len(shrs)) + for _, sh := range shrs { + namespaceBytes := sh.Namespace().Bytes() + leave := make([]byte, len(sh.ToBytes())+len(namespaceBytes)) + copy(leave, namespaceBytes) + copy(leave[len(namespaceBytes):], sh.ToBytes()) + leaves = append(leaves, leave) + } + + valid := p.shareProof.VerifyNamespace( + share.NewSHA256Hasher(), + namespace.Bytes(), + leaves, + p.root, + ) + if !valid { + return fmt.Errorf("failed to verify namespace for the shares in row") + } + + err = p.rowRootProof.Verify(dataRoot, p.root) + if err != nil { + return err + } + return nil +} + +// Start returns that start index of the proof +func (p *Proof) Start() int { + return p.shareProof.Start() +} + +// End returns an *inclusive* end index of the proof. +func (p *Proof) End() int { + return p.shareProof.End() - 1 +} + +func (p *Proof) IsOfAbsence() bool { + return p.shareProof.IsOfAbsence() +} + +func (p *Proof) MarshalJSON() ([]byte, error) { + temp := struct { + ShareProof *nmt.Proof `json:"share_proof"` + RowRootProof *merkle.Proof `json:"row_root_proof"` + Roots []byte `json:"root"` + }{ + ShareProof: p.shareProof, + RowRootProof: p.rowRootProof, + Roots: p.root, + } + return json.Marshal(temp) +} + +func (p *Proof) UnmarshalJSON(data []byte) error { + temp := struct { + ShareProof *nmt.Proof `json:"share_proof"` + RowRootProof *merkle.Proof `json:"row_root_proof"` + Root []byte `json:"root"` + }{} + err := json.Unmarshal(data, &temp) + if err != nil { + return err + } + p.shareProof = temp.ShareProof + p.rowRootProof = temp.RowRootProof + p.root = temp.Root + return nil +} + +func (p *Proof) ToProto() *pb.Proof { + nmtProof := &nmt_pb.Proof{ + Start: int64(p.shareProof.Start()), + End: int64(p.shareProof.End()), + Nodes: p.shareProof.Nodes(), + LeafHash: p.shareProof.LeafHash(), + IsMaxNamespaceIgnored: p.shareProof.IsMaxNamespaceIDIgnored(), + } + + rowRootProofs := &pb.RowRootProof{ + Total: p.rowRootProof.Total, + Index: p.rowRootProof.Index, + LeafHash: p.rowRootProof.LeafHash, + Aunts: p.rowRootProof.Aunts, + } + + return &pb.Proof{ + SharesProof: nmtProof, + RowRootProof: rowRootProofs, + Root: p.root, + } +} + +func ProofFromProto(p *pb.Proof) (*Proof, error) { + var proof nmt.Proof + if p.GetSharesProof().GetLeafHash() != nil { + proof = nmt.NewAbsenceProof( + int(p.GetSharesProof().GetStart()), + int(p.GetSharesProof().GetEnd()), + p.GetSharesProof().GetNodes(), + p.GetSharesProof().GetLeafHash(), + p.GetSharesProof().GetIsMaxNamespaceIgnored(), + ) + } else { + proof = nmt.NewInclusionProof( + int(p.GetSharesProof().GetStart()), + int(p.GetSharesProof().GetEnd()), + p.GetSharesProof().GetNodes(), + p.GetSharesProof().GetIsMaxNamespaceIgnored(), + ) + } + + rowRootProof := &merkle.Proof{ + Total: p.GetRowRootProof().GetTotal(), + Index: p.GetRowRootProof().GetIndex(), + LeafHash: p.GetRowRootProof().GetLeafHash(), + Aunts: p.GetRowRootProof().GetAunts(), + } + + return &Proof{ + shareProof: &proof, + rowRootProof: rowRootProof, + root: p.GetRoot(), + }, nil +} + +// IsEmptyProof checks if the proof is empty or not. It returns *false* +// if at least one of the condition is not met. +func (p *Proof) IsEmptyProof() bool { + return p == nil || + p.shareProof == nil || + p.shareProof.IsEmptyProof() || + p.rowRootProof == nil || + p.root == nil +} From 3d2e1168e437b46d2b6096e029c09e45cfa2c948 Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:29:31 +0200 Subject: [PATCH 03/13] feat(share/shwap): add range namespace data with id --- share/shwap/range_namespace_data.go | 188 ++++++++++++++++++++ share/shwap/range_namespace_data_id.go | 179 +++++++++++++++++++ share/shwap/range_namespace_data_id_test.go | 59 ++++++ share/shwap/range_namespace_data_test.go | 154 ++++++++++++++++ 4 files changed, 580 insertions(+) create mode 100644 share/shwap/range_namespace_data.go create mode 100644 share/shwap/range_namespace_data_id.go create mode 100644 share/shwap/range_namespace_data_id_test.go create mode 100644 share/shwap/range_namespace_data_test.go diff --git a/share/shwap/range_namespace_data.go b/share/shwap/range_namespace_data.go new file mode 100644 index 0000000000..95b2c9f8ae --- /dev/null +++ b/share/shwap/range_namespace_data.go @@ -0,0 +1,188 @@ +package shwap + +import ( + "fmt" + "github.com/pkg/errors" + + "github.com/celestiaorg/celestia-app/v3/pkg/wrapper" + libshare "github.com/celestiaorg/go-square/v2/share" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/shwap/pb" +) + +// RangeNamespaceData embeds `NamespaceData` and contains a contiguous range of shares +// along with proofs for these shares. +type RangeNamespaceData struct { + Start int `json:"start"` + NamespaceData `json:"namespace_data"` +} + +// RangedNamespaceDataFromShares builds a range of namespaced data for the given coordinates: +// shares is a list of shares (grouped by the rows) relative to the data square and +// needed to build the range; +// namespace is the target namespace for the built range; +// from is the coordinates of the first share of the range within the EDS. +// to is the coordinates of the last inclusive share of the range within the EDS. +func RangedNamespaceDataFromShares( + shares [][]libshare.Share, + namespace libshare.Namespace, + roots *share.AxisRoots, + from, to SampleCoords, +) (RangeNamespaceData, error) { + if len(shares) == 0 { + return RangeNamespaceData{}, fmt.Errorf("empty share list") + } + + odsSize := len(shares[0]) / 2 + nsData := make([]RowNamespaceData, 0, len(shares)) + rngData := RangeNamespaceData{Start: from.Row} + for i, row := 0, from.Row; i < len(shares); i++ { + rowShares := shares[i] + // end index will be explicitly set only for the last row in range. + // in other cases, it will be equal to the odsSize. + exclusiveEnd := odsSize + if i == len(shares)-1 { + // `to.Col` is an inclusive index + exclusiveEnd = to.Col + 1 + } + + // ensure that all shares from the range belong to the requested namespace + for offset := range rowShares[from.Col:exclusiveEnd] { + if !namespace.Equals(rowShares[from.Col+offset].Namespace()) { + return RangeNamespaceData{}, + fmt.Errorf("targeted namespace was not found in share at {Row: %d, Col: %d}", + row, from.Col+offset, + ) + } + } + + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(odsSize), uint(row)) + + for _, shr := range rowShares { + if err := tree.Push(shr.ToBytes()); err != nil { + return RangeNamespaceData{}, fmt.Errorf("failed to build tree for row %d: %w", row, err) + } + } + + root, err := tree.Root() + if err != nil { + return RangeNamespaceData{}, fmt.Errorf("failed to get root for row %d: %w", row, err) + } + + outside, err := share.IsOutsideRange(namespace, root, root) + if err != nil { + return RangeNamespaceData{}, err + } + if outside { + return RangeNamespaceData{}, ErrNamespaceOutsideRange + } + + proof, err := tree.ProveRange(from.Col, exclusiveEnd) + if err != nil { + return RangeNamespaceData{}, err + } + dataRootProof := NewProof(row, &proof, roots) + nsData = append(nsData, + RowNamespaceData{Shares: rowShares[from.Col:exclusiveEnd], Proof: dataRootProof}, + ) + + // reset from.Col as we are moving to the next row. + from.Col = 0 + row++ + } + + rngData.NamespaceData = nsData + return rngData, nil +} + +// Verify performs a basic validation of the incoming data. It ensures that the response data +// correspond to the request. +func (rngdata *RangeNamespaceData) Verify( + namespace libshare.Namespace, + from SampleCoords, + to SampleCoords, + dataHash []byte, +) error { + if rngdata.IsEmpty() { + return errors.New("empty data") + } + if rngdata.NamespaceData[0].Proof.IsEmptyProof() { + return errors.New("empty proof") + } + + if from.Row != rngdata.Start { + return fmt.Errorf("mismatched row: wanted: %d, got: %d", rngdata.Start, from.Row) + } + if from.Col != rngdata.NamespaceData[0].Proof.Start() { + return fmt.Errorf("mismatched col: wanted: %d, got: %d", rngdata.NamespaceData[0].Proof.Start(), from.Col) + } + if to.Col != rngdata.NamespaceData[len(rngdata.NamespaceData)-1].Proof.End() { + return fmt.Errorf( + "mismatched col: wanted: %d, got: %d", + rngdata.NamespaceData[len(rngdata.NamespaceData)-1].Proof.End(), to.Col, + ) + } + + for i, nsData := range rngdata.NamespaceData { + if nsData.Proof.IsEmptyProof() { + return fmt.Errorf("nil proof for row: %d", rngdata.Start+i) + } + if nsData.Proof.shareProof.IsOfAbsence() { + return fmt.Errorf("absence proof for row: %d", rngdata.Start+i) + } + + err := nsData.Proof.VerifyInclusion(nsData.Shares, namespace, dataHash) + if err != nil { + return fmt.Errorf("%w for row: %d, %w", ErrFailedVerification, rngdata.Start+i, err) + } + } + return nil +} + +func (rngdata *RangeNamespaceData) IsEmpty() bool { + return len(rngdata.NamespaceData) == 0 +} + +func (rngdata *RangeNamespaceData) ToProto() *pb.RangeNamespaceData { + return &pb.RangeNamespaceData{ + Start: int32(rngdata.Start), + RangeNamespaceData: rngdata.NamespaceData.ToProto(), + } +} + +func (rngdata *RangeNamespaceData) CleanupData() { + for i := range rngdata.NamespaceData { + rngdata.NamespaceData[i].Shares = nil + } +} + +func RangeNamespaceDataFromProto(nd *pb.RangeNamespaceData) (*RangeNamespaceData, error) { + data, err := NamespaceDataFromProto(nd.RangeNamespaceData) + if err != nil { + return nil, err + } + return &RangeNamespaceData{Start: int(nd.Start), NamespaceData: data}, nil +} + +// RangeCoordsFromIdx accepts the start index and the length of the range and +// computes `shwap.SampleCoords` of the first and the last sample of this range. +// * edsIndex is the index of the first sample inside the eds; +// * length is the amount of *ORIGINAL* samples that are expected to get(including the first sample); +func RangeCoordsFromIdx(edsIndex, length, edsSize int) (SampleCoords, SampleCoords, error) { + from, err := SampleCoordsFrom1DIndex(edsIndex, edsSize) + if err != nil { + return SampleCoords{}, SampleCoords{}, err + } + odsIndex, err := SampleCoordsAs1DIndex(from, edsSize/2) + if err != nil { + return SampleCoords{}, SampleCoords{}, err + } + + toInclusive := odsIndex + length - 1 + toCoords, err := SampleCoordsFrom1DIndex(toInclusive, edsSize/2) + if err != nil { + return SampleCoords{}, SampleCoords{}, err + } + return from, toCoords, nil +} diff --git a/share/shwap/range_namespace_data_id.go b/share/shwap/range_namespace_data_id.go new file mode 100644 index 0000000000..03157970bd --- /dev/null +++ b/share/shwap/range_namespace_data_id.go @@ -0,0 +1,179 @@ +package shwap + +import ( + "encoding/binary" + "fmt" + "io" + + libshare "github.com/celestiaorg/go-square/v2/share" +) + +// RangeNamespaceDataIDSize defines the size of the RangeNamespaceDataIDSize in bytes, +// combining SampleID size, Namespace size, 4 additional bytes +// for the end coordinates of share of the range and uint representation of bool flag. +const RangeNamespaceDataIDSize = EdsIDSize + libshare.NamespaceSize + 10 + +// RangeNamespaceDataID identifies the continuous range of shares in the DataSquare(EDS), +// starting from the given `SampleID` and contains `Length` number of shares. +type RangeNamespaceDataID struct { + EdsID + // coordinates from the first share of the range + From SampleCoords + // coordinates from the last share of the range + To SampleCoords + // DataNamespace is a string representation of the namespace of the requested range. + DataNamespace libshare.Namespace + + ProofsOnly bool +} + +func NewRangeNamespaceDataID( + edsID EdsID, + namespace libshare.Namespace, + from SampleCoords, + to SampleCoords, + edsSize int, + proofsOnly bool, +) (RangeNamespaceDataID, error) { + rngid := RangeNamespaceDataID{ + EdsID: edsID, + From: from, + DataNamespace: namespace, + To: to, + ProofsOnly: proofsOnly, + } + + err := rngid.Verify(edsSize) + if err != nil { + return RangeNamespaceDataID{}, fmt.Errorf("verifying range id: %w", err) + } + return rngid, nil +} + +// Verify validates the RangeNamespaceDataID fields and verifies that number of the requested shares +// does not exceed the number of shares inside the ODS. +func (rngid RangeNamespaceDataID) Verify(edsSize int) error { + err := rngid.EdsID.Validate() + if err != nil { + return fmt.Errorf("invalid EdsID: %w", err) + } + err = rngid.DataNamespace.ValidateForData() + if err != nil { + return err + } + _, err = SampleCoordsAs1DIndex(rngid.From, edsSize) + if err != nil { + return err + } + // verify that to is not exceed that edsSize + _, err = SampleCoordsAs1DIndex(rngid.To, edsSize) + if err != nil { + return err + } + return nil +} + +// Validate performs basic fields validation. +func (rngid RangeNamespaceDataID) Validate() error { + return rngid.DataNamespace.ValidateForData() +} + +// ReadFrom reads the binary form of RangeNamespaceDataID from the provided reader. +func (rngid *RangeNamespaceDataID) ReadFrom(r io.Reader) (int64, error) { + data := make([]byte, RangeNamespaceDataIDSize) + n, err := io.ReadFull(r, data) + if err != nil { + return int64(n), err + } + if n != RangeNamespaceDataIDSize { + return int64(n), fmt.Errorf("RangeNamespaceDataID: expected %d bytes, got %d", RangeNamespaceDataIDSize, n) + } + + id, err := RangeNamespaceDataIDFromBinary(data) + if err != nil { + return int64(n), fmt.Errorf("RangeNamespaceDataIDFromBinary: %w", err) + } + *rngid = id + return int64(n), nil +} + +// WriteTo writes the binary form of RangeNamespaceDataID to the provided writer. +func (rngid RangeNamespaceDataID) WriteTo(w io.Writer) (int64, error) { + data, err := rngid.MarshalBinary() + if err != nil { + return 0, err + } + n, err := w.Write(data) + return int64(n), err +} + +// Equals checks equality of RangeNamespaceDataID. +func (rngid *RangeNamespaceDataID) Equals(other RangeNamespaceDataID) bool { + return rngid.DataNamespace.Equals(other.DataNamespace) && rngid.From == other.From && + rngid.To == other.To +} + +// RangeNamespaceDataIDFromBinary deserializes a RangeNamespaceDataID from its binary form. +func RangeNamespaceDataIDFromBinary(data []byte) (RangeNamespaceDataID, error) { + if len(data) != RangeNamespaceDataIDSize { + return RangeNamespaceDataID{}, fmt.Errorf( + "invalid RangeNamespaceDataID data length: expected %d, got %d", RangeNamespaceDataIDSize, len(data), + ) + } + + edsID, err := EdsIDFromBinary(data[:EdsIDSize]) + if err != nil { + return RangeNamespaceDataID{}, err + } + + fromCoords := SampleCoords{ + Row: int(binary.BigEndian.Uint16(data[EdsIDSize : EdsIDSize+2])), + Col: int(binary.BigEndian.Uint16(data[EdsIDSize+2 : EdsIDSize+4])), + } + toCoords := SampleCoords{ + Row: int(binary.BigEndian.Uint16(data[EdsIDSize+4 : EdsIDSize+6])), + Col: int(binary.BigEndian.Uint16(data[EdsIDSize+6 : EdsIDSize+8])), + } + + ns, err := libshare.NewNamespaceFromBytes(data[EdsIDSize+8 : EdsIDSize+8+libshare.NamespaceSize]) + if err != nil { + return RangeNamespaceDataID{}, fmt.Errorf("converting namespace from binary: %w", err) + } + + var proofsOnly bool + if data[RangeNamespaceDataIDSize-1] == 1 { + proofsOnly = true + } + + rngID := RangeNamespaceDataID{ + EdsID: edsID, + From: fromCoords, + DataNamespace: ns, + To: toCoords, + ProofsOnly: proofsOnly, + } + return rngID, rngID.Validate() +} + +// MarshalBinary encodes RangeNamespaceDataID into binary form. +func (rngid RangeNamespaceDataID) MarshalBinary() ([]byte, error) { + data := make([]byte, 0, RangeNamespaceDataIDSize) + return rngid.appendTo(data), nil +} + +// appendTo helps in constructing the binary representation of RangeNamespaceDataID +// by appending all encoded fields. +func (rngid RangeNamespaceDataID) appendTo(data []byte) []byte { + data = rngid.EdsID.appendTo(data) + data = binary.BigEndian.AppendUint16(data, uint16(rngid.From.Row)) + data = binary.BigEndian.AppendUint16(data, uint16(rngid.From.Col)) + data = binary.BigEndian.AppendUint16(data, uint16(rngid.To.Row)) + data = binary.BigEndian.AppendUint16(data, uint16(rngid.To.Col)) + data = append(data, rngid.DataNamespace.Bytes()...) + if rngid.ProofsOnly { + data = binary.BigEndian.AppendUint16(data, 1) + } else { + data = binary.BigEndian.AppendUint16(data, 0) + } + return data +} diff --git a/share/shwap/range_namespace_data_id_test.go b/share/shwap/range_namespace_data_id_test.go new file mode 100644 index 0000000000..a05b59a8fd --- /dev/null +++ b/share/shwap/range_namespace_data_id_test.go @@ -0,0 +1,59 @@ +package shwap + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + libshare "github.com/celestiaorg/go-square/v2/share" +) + +func TestNewRangeNamespaceDataID(t *testing.T) { + edsSize := 16 + ns := libshare.RandomNamespace() + + rngid, err := NewRangeNamespaceDataID( + EdsID{1}, + ns, + SampleCoords{Row: 5, Col: 10}, + SampleCoords{7, 12}, + edsSize, + true, + ) + require.NoError(t, err) + + bin, err := rngid.MarshalBinary() + require.NoError(t, err) + + rngidOut, err := RangeNamespaceDataIDFromBinary(bin) + require.NoError(t, err) + assert.EqualValues(t, rngid, rngidOut) + + err = rngid.Validate() + require.NoError(t, err) +} + +func TestRangeNamespaceDataIDReaderWriter(t *testing.T) { + edsSize := 32 + ns := libshare.RandomNamespace() + to, err := SampleCoordsFrom1DIndex(10, edsSize) + require.NoError(t, err) + rngid, err := NewRangeNamespaceDataID(EdsID{1}, ns, SampleCoords{Row: 0, Col: 1}, to, edsSize, false) + require.NoError(t, err) + + buf := bytes.NewBuffer(nil) + n, err := rngid.WriteTo(buf) + require.NoError(t, err) + require.Equal(t, int64(RangeNamespaceDataIDSize), n) + + rngidOut := RangeNamespaceDataID{} + n, err = rngidOut.ReadFrom(buf) + require.NoError(t, err) + require.Equal(t, int64(RangeNamespaceDataIDSize), n) + require.EqualValues(t, rngid, rngidOut) + + err = rngidOut.Validate() + require.NoError(t, err) +} diff --git a/share/shwap/range_namespace_data_test.go b/share/shwap/range_namespace_data_test.go new file mode 100644 index 0000000000..a3b300180a --- /dev/null +++ b/share/shwap/range_namespace_data_test.go @@ -0,0 +1,154 @@ +package shwap_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + libshare "github.com/celestiaorg/go-square/v2/share" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/shwap" +) + +func TestRangeNamespaceData(t *testing.T) { + const ( + odsSize = 16 + sharesAmount = odsSize * odsSize + ) + + ns := libshare.RandomNamespace() + square, root := edstest.RandEDSWithNamespace(t, ns, odsSize, odsSize) + + nsRowStart := -1 + for i, row := range root.RowRoots { + outside, err := share.IsOutsideRange(ns, row, row) + require.NoError(t, err) + if !outside { + nsRowStart = i + break + } + } + assert.Greater(t, nsRowStart, -1) + + extended := &eds.Rsmt2D{ExtendedDataSquare: square} + + nsData, err := extended.RowNamespaceData(context.Background(), ns, nsRowStart) + require.NoError(t, err) + col := nsData.Proof.Start() + + for i := 1; i <= odsSize; i++ { + t.Run(fmt.Sprintf("range of %d shares", i), func(t *testing.T) { + toRow, toCol := nsRowStart, col+i-1 + for toCol >= odsSize { + toRow++ + toCol -= odsSize + } + + to := shwap.SampleCoords{Row: toRow, Col: toCol} + dataID, err := shwap.NewRangeNamespaceDataID( + shwap.EdsID{Height: 1}, + ns, + shwap.SampleCoords{Row: nsRowStart, Col: col}, + to, + sharesAmount, + false, + ) + require.NoError(t, err) + + rngdata, err := extended.RangeNamespaceData( + context.Background(), + dataID.DataNamespace, + shwap.SampleCoords{Row: dataID.From.Row, Col: dataID.From.Col}, + to, + ) + require.NoError(t, err) + err = rngdata.Verify(ns, dataID.From, dataID.To, root.Hash()) + require.NoError(t, err) + }) + } +} + +func TestRangeCoordsFromIdx(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + const ( + odsSize = 4 + edsSize = odsSize * 2 + ) + + ns := libshare.RandomNamespace() + square, _ := edstest.RandEDSWithNamespace(t, ns, odsSize*odsSize, odsSize) + rngLengths := []int{2, 7, 11, 15} + extended := &eds.Rsmt2D{ExtendedDataSquare: square} + for _, length := range rngLengths { + from, to, err := shwap.RangeCoordsFromIdx(0, length, edsSize) + require.NoError(t, err) + rngData, err := extended.RangeNamespaceData(ctx, ns, from, to) + require.NoError(t, err) + require.Equal(t, length, len(rngData.Flatten())) + } +} + +func FuzzRangeCoordsFromIdx(f *testing.F) { + if testing.Short() { + f.Skip() + } + + const ( + odsSize = 16 + edsSize = odsSize * 2 + ) + + square := edstest.RandEDS(f, odsSize) + shrs := square.FlattenedODS() + assert.Equal(f, len(shrs), odsSize*odsSize) + + f.Add(0, 3, edsSize) + f.Add(10, 14, edsSize) + f.Add(23, 30, edsSize) + f.Add(62, 3, edsSize) + + f.Fuzz(func(t *testing.T, edsIndex, length, size int) { + if edsIndex < 0 || edsIndex >= edsSize*edsSize { + return + } + + coords, err := shwap.SampleCoordsFrom1DIndex(edsIndex, edsSize) + require.NoError(t, err) + if coords.Row >= odsSize || coords.Col >= odsSize { + return + } + + odsIndexStart := coords.Row*odsSize + coords.Col + if odsIndexStart+length >= odsSize*odsSize { + return + } + + if length <= 0 || length >= odsSize*odsSize { + return + } + if size != edsSize { + return + } + + from, to, err := shwap.RangeCoordsFromIdx(edsIndex, length, size) + require.NoError(t, err) + edsStartShare := square.GetCell(uint(from.Row), uint(from.Col)) + edsEndShare := square.GetCell(uint(to.Row), uint(to.Col)) + + odsIndexStart = from.Row*odsSize + from.Col + odsIndexEnd := to.Row*odsSize + to.Col + + odsStartShare := shrs[odsIndexStart] + odsEndShare := shrs[odsIndexEnd] + require.Equal(t, edsStartShare, odsStartShare) + require.Equal(t, edsEndShare, odsEndShare) + }) +} From 9542c57a174704ea1e968b055105bbb5a8ba82e2 Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:30:57 +0200 Subject: [PATCH 04/13] feat(share/p2p/bitswap): add range namespace data block --- .../p2p/bitswap/range_namespace_data_block.go | 149 ++++++++++++++++++ .../range_namespace_data_block_test.go | 41 +++++ 2 files changed, 190 insertions(+) create mode 100644 share/shwap/p2p/bitswap/range_namespace_data_block.go create mode 100644 share/shwap/p2p/bitswap/range_namespace_data_block_test.go diff --git a/share/shwap/p2p/bitswap/range_namespace_data_block.go b/share/shwap/p2p/bitswap/range_namespace_data_block.go new file mode 100644 index 0000000000..1257e1b1db --- /dev/null +++ b/share/shwap/p2p/bitswap/range_namespace_data_block.go @@ -0,0 +1,149 @@ +package bitswap + +import ( + "context" + "fmt" + + "github.com/ipfs/go-cid" + + libshare "github.com/celestiaorg/go-square/v2/share" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/shwap" + shwappb "github.com/celestiaorg/celestia-node/share/shwap/pb" +) + +const ( + // rangeNamespaceDataCodec is a CID codec used for data Bitswap requests over Namespaced Merkle Tree. + rangeNamespaceDataCodec = 0x7830 + + // rangeNamespaceDataMultihashCode is the multihash code for data multihash function. + rangeNamespaceDataMultihashCode = 0x7831 +) + +// maxRangeSize is the maximum size of the RangeNamespaceDataBlock. +// It is calculated as a maxRoxSize multiplied by the half of the square size. +var maxRangeSize = maxRowSize * share.MaxSquareSize / 2 + +func init() { + registerBlock( + rangeNamespaceDataMultihashCode, + rangeNamespaceDataCodec, + maxRangeSize, + shwap.RangeNamespaceDataIDSize, + func(cid cid.Cid) (Block, error) { + return EmptyRangeNamespaceDataBlockFromCID(cid) + }, + ) +} + +// RangeNamespaceDataBlock is a Bitswap compatible block for Shwap's RangeNamespaceData container. +type RangeNamespaceDataBlock struct { + ID shwap.RangeNamespaceDataID + + Container shwap.RangeNamespaceData +} + +// NewEmptyRangeNamespaceDataBlock constructs a new empty RangeNamespaceDataBlock. +func NewEmptyRangeNamespaceDataBlock( + height uint64, + namespace libshare.Namespace, + from shwap.SampleCoords, + to shwap.SampleCoords, + edsSize int, + proofsOnly bool, +) (*RangeNamespaceDataBlock, error) { + id, err := shwap.NewRangeNamespaceDataID(shwap.EdsID{Height: height}, namespace, from, to, edsSize, proofsOnly) + if err != nil { + return nil, err + } + + return &RangeNamespaceDataBlock{ID: id}, nil +} + +// EmptyRangeNamespaceDataBlockFromCID constructs an empty RangeNamespaceDataBlock out of the CID. +func EmptyRangeNamespaceDataBlockFromCID(cid cid.Cid) (*RangeNamespaceDataBlock, error) { + rngidData, err := extractFromCID(cid) + if err != nil { + return nil, err + } + + rndid, err := shwap.RangeNamespaceDataIDFromBinary(rngidData) + if err != nil { + return nil, fmt.Errorf("unmarhalling RangeNamespaceDataBlock: %w", err) + } + return &RangeNamespaceDataBlock{ID: rndid}, nil +} + +func (rndb *RangeNamespaceDataBlock) CID() cid.Cid { + return encodeToCID(rndb.ID, rangeNamespaceDataMultihashCode, rangeNamespaceDataCodec) +} + +func (rndb *RangeNamespaceDataBlock) Height() uint64 { + return rndb.ID.Height +} + +func (rndb *RangeNamespaceDataBlock) Marshal() ([]byte, error) { + if empty := rndb.Container.IsEmpty(); empty { + return nil, fmt.Errorf("cannot marshal empty RangeNamespaceDataBlock") + } + container := rndb.Container.ToProto() + containerData, err := container.Marshal() + if err != nil { + return nil, fmt.Errorf("marshaling RangeNamespaceDataBlock container: %w", err) + } + return containerData, nil +} + +func (rndb *RangeNamespaceDataBlock) Populate(ctx context.Context, eds eds.Accessor) error { + rnd, err := eds.RangeNamespaceData( + ctx, + rndb.ID.DataNamespace, + rndb.ID.From, + rndb.ID.To, + ) + if err != nil { + return fmt.Errorf("accessing RangeNamespaceData: %w", err) + } + + if rndb.ID.ProofsOnly { + rnd.CleanupData() + } + rndb.Container = rnd + return nil +} + +func (rndb *RangeNamespaceDataBlock) UnmarshalFn(root *share.AxisRoots) UnmarshalFn { + return func(cntrData, idData []byte) error { + if empty := rndb.Container.IsEmpty(); !empty { + return nil + } + + rndid, err := shwap.RangeNamespaceDataIDFromBinary(idData) + if err != nil { + return fmt.Errorf("unmarhaling RangeNamespaceDataID: %w", err) + } + + if !rndb.ID.Equals(rndid) { + return fmt.Errorf("requested %+v doesnt match given %+v", rndb.ID, rndid) + } + + var rnd shwappb.RangeNamespaceData + if err := rnd.Unmarshal(cntrData); err != nil { + return fmt.Errorf("unmarshaling RangeNamespaceData for %+v: %w", rndb.ID, err) + } + + rangeNsData, err := shwap.RangeNamespaceDataFromProto(&rnd) + if err != nil { + return fmt.Errorf("unmarshaling RangeNamespaceData for %+v: %w", rndb.ID, err) + } + if !rndb.ID.ProofsOnly { + if err := rangeNsData.Verify(rndid.DataNamespace, rndid.From, rndid.To, root.Hash()); err != nil { + return fmt.Errorf("validating RangeNamespaceData for %+v: %w", rndb.ID, err) + } + } + rndb.Container = *rangeNsData + return nil + } +} diff --git a/share/shwap/p2p/bitswap/range_namespace_data_block_test.go b/share/shwap/p2p/bitswap/range_namespace_data_block_test.go new file mode 100644 index 0000000000..4a6a4672cf --- /dev/null +++ b/share/shwap/p2p/bitswap/range_namespace_data_block_test.go @@ -0,0 +1,41 @@ +package bitswap + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + libshare "github.com/celestiaorg/go-square/v2/share" + + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/shwap" +) + +func TestRangeNamespaceData_FetchRoundtrip(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() + + namespace := libshare.RandomNamespace() + eds, root := edstest.RandEDSWithNamespace(t, namespace, 64, 8) + exchange := newExchangeOverEDS(ctx, t, eds) + blk, err := NewEmptyRangeNamespaceDataBlock(1, namespace, + shwap.SampleCoords{Row: 0, Col: 0}, + shwap.SampleCoords{Row: 2, Col: 2}, + 16, + false, + ) + require.NoError(t, err) + + err = Fetch(ctx, exchange, root, []Block{blk}) + require.NoError(t, err) + + err = blk.Container.Verify( + namespace, + shwap.SampleCoords{Row: 0, Col: 0}, + shwap.SampleCoords{Row: 2, Col: 2}, + root.Hash(), + ) + require.NoError(t, err) +} From 89d44789330c340c1079039566923278cdebc3d8 Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:33:40 +0200 Subject: [PATCH 05/13] refacor!(nodebuilde/share): update api --- nodebuilder/share/mocks/api.go | 19 ++++++------ nodebuilder/share/share.go | 53 ++++++++++++++-------------------- 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index d392f39f5b..561811a8b3 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -9,9 +9,8 @@ import ( reflect "reflect" header "github.com/celestiaorg/celestia-node/header" - share "github.com/celestiaorg/celestia-node/nodebuilder/share" shwap "github.com/celestiaorg/celestia-node/share/shwap" - share0 "github.com/celestiaorg/go-square/v2/share" + share "github.com/celestiaorg/go-square/v2/share" rsmt2d "github.com/celestiaorg/rsmt2d" gomock "github.com/golang/mock/gomock" ) @@ -55,7 +54,7 @@ func (mr *MockModuleMockRecorder) GetEDS(arg0, arg1 interface{}) *gomock.Call { } // GetNamespaceData mocks base method. -func (m *MockModule) GetNamespaceData(arg0 context.Context, arg1 uint64, arg2 share0.Namespace) (shwap.NamespaceData, error) { +func (m *MockModule) GetNamespaceData(arg0 context.Context, arg1 uint64, arg2 share.Namespace) (shwap.NamespaceData, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetNamespaceData", arg0, arg1, arg2) ret0, _ := ret[0].(shwap.NamespaceData) @@ -70,18 +69,18 @@ func (mr *MockModuleMockRecorder) GetNamespaceData(arg0, arg1, arg2 interface{}) } // GetRange mocks base method. -func (m *MockModule) GetRange(arg0 context.Context, arg1 uint64, arg2, arg3 int) (*share.GetRangeResult, error) { +func (m *MockModule) GetRange(arg0 context.Context, arg1 share.Namespace, arg2 uint64, arg3, arg4 shwap.SampleCoords, arg5 bool) (shwap.RangeNamespaceData, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetRange", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(*share.GetRangeResult) + ret := m.ctrl.Call(m, "GetRange", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(shwap.RangeNamespaceData) ret1, _ := ret[1].(error) return ret0, ret1 } // GetRange indicates an expected call of GetRange. -func (mr *MockModuleMockRecorder) GetRange(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockModuleMockRecorder) GetRange(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRange", reflect.TypeOf((*MockModule)(nil).GetRange), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRange", reflect.TypeOf((*MockModule)(nil).GetRange), arg0, arg1, arg2, arg3, arg4, arg5) } // GetRow mocks base method. @@ -115,10 +114,10 @@ func (mr *MockModuleMockRecorder) GetSamples(arg0, arg1, arg2 interface{}) *gomo } // GetShare mocks base method. -func (m *MockModule) GetShare(arg0 context.Context, arg1 uint64, arg2, arg3 int) (share0.Share, error) { +func (m *MockModule) GetShare(arg0 context.Context, arg1 uint64, arg2, arg3 int) (share.Share, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetShare", arg0, arg1, arg2, arg3) - ret0, _ := ret[0].(share0.Share) + ret0, _ := ret[0].(share.Share) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/nodebuilder/share/share.go b/nodebuilder/share/share.go index 46df1e2379..7039729de2 100644 --- a/nodebuilder/share/share.go +++ b/nodebuilder/share/share.go @@ -3,27 +3,17 @@ package share import ( "context" - "github.com/tendermint/tendermint/types" - libshare "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" headerServ "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/shwap" ) var _ Module = (*API)(nil) -// GetRangeResult wraps the return value of the GetRange endpoint -// because Json-RPC doesn't support more than two return values. -type GetRangeResult struct { - Shares []libshare.Share - Proof *types.ShareProof -} - // Module provides access to any data square or block share on the network. // // All Get methods provided on Module follow the following flow: @@ -58,7 +48,9 @@ type Module interface { ctx context.Context, height uint64, namespace libshare.Namespace, ) (shwap.NamespaceData, error) // GetRange gets a list of shares and their corresponding proof. - GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error) + GetRange( + ctx context.Context, ns libshare.Namespace, height uint64, from, to shwap.SampleCoords, proofsOnly bool, + ) (shwap.RangeNamespaceData, error) } // API is a wrapper around Module for the RPC. @@ -91,9 +83,11 @@ type API struct { ) (shwap.NamespaceData, error) `perm:"read"` GetRange func( ctx context.Context, + ns libshare.Namespace, height uint64, - start, end int, - ) (*GetRangeResult, error) `perm:"read"` + from, to shwap.SampleCoords, + proofsOnly bool, + ) (shwap.RangeNamespaceData, error) `perm:"read"` } } @@ -119,8 +113,10 @@ func (api *API) GetRow(ctx context.Context, height uint64, rowIdx int) (shwap.Ro return api.Internal.GetRow(ctx, height, rowIdx) } -func (api *API) GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error) { - return api.Internal.GetRange(ctx, height, start, end) +func (api *API) GetRange( + ctx context.Context, ns libshare.Namespace, height uint64, from, to shwap.SampleCoords, proofsOnly bool, +) (shwap.RangeNamespaceData, error) { + return api.Internal.GetRange(ctx, ns, height, from, to, proofsOnly) } func (api *API) GetNamespaceData( @@ -175,25 +171,18 @@ func (m module) SharesAvailable(ctx context.Context, height uint64) error { return m.avail.SharesAvailable(ctx, header) } -func (m module) GetRange(ctx context.Context, height uint64, start, end int) (*GetRangeResult, error) { - extendedDataSquare, err := m.GetEDS(ctx, height) - if err != nil { - return nil, err - } - - proof, err := eds.ProveShares(extendedDataSquare, start, end) - if err != nil { - return nil, err - } - - shares, err := libshare.FromBytes(extendedDataSquare.FlattenedODS()[start:end]) +func (m module) GetRange( + ctx context.Context, + ns libshare.Namespace, + height uint64, + from, to shwap.SampleCoords, + proofsOnly bool, +) (shwap.RangeNamespaceData, error) { + header, err := m.hs.GetByHeight(ctx, height) if err != nil { - return nil, err + return shwap.RangeNamespaceData{}, err } - return &GetRangeResult{ - Shares: shares, - Proof: proof, - }, nil + return m.getter.GetRangeNamespaceData(ctx, header, ns, from, to, proofsOnly) } func (m module) GetNamespaceData( From 64ba201ba0283bcd2be21ffe21c2ac0c9681276d Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:35:06 +0200 Subject: [PATCH 06/13] swamp(share): add GetRange test --- nodebuilder/tests/share_test.go | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/nodebuilder/tests/share_test.go b/nodebuilder/tests/share_test.go index 551b0d9c69..13d690a086 100644 --- a/nodebuilder/tests/share_test.go +++ b/nodebuilder/tests/share_test.go @@ -23,7 +23,7 @@ import ( ) func TestShareModule(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Minute) t.Cleanup(cancel) sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second*1)) blobSize := 128 @@ -44,6 +44,7 @@ func TestShareModule(t *testing.T) { require.NoError(t, fullNode.Start(ctx)) addrsFull, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(fullNode.Host)) + require.NoError(t, err) lightCfg := sw.DefaultTestConfig(node.Light) @@ -207,7 +208,30 @@ func TestShareModule(t *testing.T) { require.NoError(t, err) // compare commitments require.Equal(t, nodeBlob[0].Commitment, blb[0].Commitment) + } + }, + }, + { + name: "GetRangeNamespaceData", + doFn: func(t *testing.T) { + dah := hdr.DAH + blobLength, err := sampledBlob.Length() + require.NoError(t, err) + from, to, err := shwap.RangeCoordsFromIdx(sampledBlob.Index(), blobLength, len(dah.RowRoots)) + require.NoError(t, err) + for _, client := range clients { + rng, err := client.Share.GetRange(ctx, nodeBlob[0].Namespace(), height, from, to, false) + require.NoError(t, err) + err = rng.Verify(nodeBlob[0].Namespace(), from, to, dah.Hash()) + require.NoError(t, err) + shrs := rng.Flatten() + blbs, err := libshare.ParseBlobs(shrs) + require.NoError(t, err) + + parsedBlob, err := blob.ToNodeBlobs(blbs...) + require.NoError(t, err) + require.Equal(t, nodeBlob[0].Commitment, parsedBlob[0].Commitment) } }, }, From 53ecfcfb129a0bbb28fb3628a2b44d6a6124e55c Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:42:54 +0200 Subject: [PATCH 07/13] chore(share/getters): udpate getters --- share/availability/light/availability_test.go | 10 +++++ share/shwap/getter.go | 9 +++++ share/shwap/getters/cascade.go | 25 ++++++++++++ share/shwap/getters/mock/getter.go | 15 ++++++++ share/shwap/getters/testing.go | 10 +++++ share/shwap/p2p/bitswap/getter.go | 38 +++++++++++++++++++ share/shwap/p2p/shrex/shrex_getter/shrex.go | 10 +++++ store/getter.go | 24 ++++++++++++ 8 files changed, 141 insertions(+) diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index df441c46a1..2635882d38 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -321,6 +321,16 @@ func (g successGetter) GetNamespaceData( panic("not implemented") } +func (g successGetter) GetRangeNamespaceData( + _ context.Context, + _ *header.ExtendedHeader, + _ libshare.Namespace, + _, _ shwap.SampleCoords, + _ bool, +) (shwap.RangeNamespaceData, error) { + panic("not implemented") +} + func TestPruneAll(t *testing.T) { const size = 8 ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) diff --git a/share/shwap/getter.go b/share/shwap/getter.go index f56c8edc63..a3561c25be 100644 --- a/share/shwap/getter.go +++ b/share/shwap/getter.go @@ -47,4 +47,13 @@ type Getter interface { // If no shares are found for target namespace non-inclusion could be also verified by calling // Verify method. GetNamespaceData(context.Context, *header.ExtendedHeader, libshare.Namespace) (NamespaceData, error) + // GetNamespaceData gets all namespaced shares from an EDS within a given range. + // Shares are returned in a row-by-row order if the range spans multiple rows. + GetRangeNamespaceData( + _ context.Context, + _ *header.ExtendedHeader, + _ libshare.Namespace, + from, to SampleCoords, + _ bool, + ) (RangeNamespaceData, error) } diff --git a/share/shwap/getters/cascade.go b/share/shwap/getters/cascade.go index c9ac091ddd..f11abbb4dc 100644 --- a/share/shwap/getters/cascade.go +++ b/share/shwap/getters/cascade.go @@ -3,6 +3,7 @@ package getters import ( "context" "errors" + "fmt" "time" logging "github.com/ipfs/go-log/v2" @@ -102,6 +103,30 @@ func (cg *CascadeGetter) GetNamespaceData( return cascadeGetters(ctx, cg.getters, get) } +func (cg *CascadeGetter) GetRangeNamespaceData( + ctx context.Context, + header *header.ExtendedHeader, + namespace libshare.Namespace, + from, to shwap.SampleCoords, + proofsOnly bool, +) (shwap.RangeNamespaceData, error) { + ctx, span := tracer.Start(ctx, "cascade/get-shares-range", trace.WithAttributes( + attribute.String("namespace", namespace.String()), + )) + defer span.End() + + if from.Row > to.Row { + return shwap.RangeNamespaceData{}, + fmt.Errorf("start row must not be bigger to end row: %d-%d", from.Row, to.Row) + } + + get := func(ctx context.Context, get shwap.Getter) (shwap.RangeNamespaceData, error) { + return get.GetRangeNamespaceData(ctx, header, namespace, from, to, proofsOnly) + } + + return cascadeGetters(ctx, cg.getters, get) +} + // cascade implements a cascading retry algorithm for getting a value from multiple sources. // Cascading implies trying the sources one-by-one in the given order with the // given interval until either: diff --git a/share/shwap/getters/mock/getter.go b/share/shwap/getters/mock/getter.go index ade35b39f8..87f9b4a690 100644 --- a/share/shwap/getters/mock/getter.go +++ b/share/shwap/getters/mock/getter.go @@ -68,6 +68,21 @@ func (mr *MockGetterMockRecorder) GetNamespaceData(arg0, arg1, arg2 interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetNamespaceData", reflect.TypeOf((*MockGetter)(nil).GetNamespaceData), arg0, arg1, arg2) } +// GetRangeNamespaceData mocks base method. +func (m *MockGetter) GetRangeNamespaceData(arg0 context.Context, arg1 *header.ExtendedHeader, arg2 share.Namespace, arg3, arg4 shwap.SampleCoords, arg5 bool) (shwap.RangeNamespaceData, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRangeNamespaceData", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(shwap.RangeNamespaceData) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRangeNamespaceData indicates an expected call of GetRangeNamespaceData. +func (mr *MockGetterMockRecorder) GetRangeNamespaceData(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRangeNamespaceData", reflect.TypeOf((*MockGetter)(nil).GetRangeNamespaceData), arg0, arg1, arg2, arg3, arg4, arg5) +} + // GetRow mocks base method. func (m *MockGetter) GetRow(arg0 context.Context, arg1 *header.ExtendedHeader, arg2 int) (shwap.Row, error) { m.ctrl.T.Helper() diff --git a/share/shwap/getters/testing.go b/share/shwap/getters/testing.go index c244204aba..b67bc6dd4f 100644 --- a/share/shwap/getters/testing.go +++ b/share/shwap/getters/testing.go @@ -93,6 +93,16 @@ func (seg *SingleEDSGetter) GetNamespaceData(context.Context, *header.ExtendedHe panic("SingleEDSGetter: GetNamespaceData is not implemented") } +func (seg *SingleEDSGetter) GetRangeNamespaceData( + _ context.Context, + _ *header.ExtendedHeader, + _ libshare.Namespace, + _, _ shwap.SampleCoords, + _ bool, +) (shwap.RangeNamespaceData, error) { + panic("SingleEDSGetter: GetRangeNamespaceData is not implemented") +} + func (seg *SingleEDSGetter) checkRoots(roots *share.AxisRoots) error { dah, err := da.NewDataAvailabilityHeader(seg.EDS.ExtendedDataSquare) if err != nil { diff --git a/share/shwap/p2p/bitswap/getter.go b/share/shwap/p2p/bitswap/getter.go index 8033d84b76..aa841c4aca 100644 --- a/share/shwap/p2p/bitswap/getter.go +++ b/share/shwap/p2p/bitswap/getter.go @@ -268,6 +268,44 @@ func (g *Getter) GetNamespaceData( return nsShrs, nil } +func (g *Getter) GetRangeNamespaceData( + ctx context.Context, + hdr *header.ExtendedHeader, + ns libshare.Namespace, + from, to shwap.SampleCoords, + proofsOnly bool, +) (shwap.RangeNamespaceData, error) { + if err := ns.ValidateForData(); err != nil { + return shwap.RangeNamespaceData{}, err + } + + ctx, span := tracer.Start(ctx, "get-shares-range") + defer span.End() + + rangeDataBlock, err := NewEmptyRangeNamespaceDataBlock( + hdr.Height(), ns, from, to, len(hdr.DAH.RowRoots), proofsOnly, + ) + if err != nil { + return shwap.RangeNamespaceData{}, err + } + + isArchival := g.isArchival(hdr) + span.SetAttributes(attribute.Bool("is_archival", isArchival)) + + ses, release := g.getSession(isArchival) + defer release() + + blks := []Block{rangeDataBlock} + + err = Fetch(ctx, g.exchange, hdr.DAH, blks, WithFetcher(ses)) + if err != nil { + span.RecordError(err) + span.SetStatus(codes.Error, "Fetch") + return shwap.RangeNamespaceData{}, err + } + return blks[0].(*RangeNamespaceDataBlock).Container, nil +} + // isArchival reports whether the header is for archival data func (g *Getter) isArchival(hdr *header.ExtendedHeader) bool { return !availability.IsWithinWindow(hdr.Time(), g.availWndw) diff --git a/share/shwap/p2p/shrex/shrex_getter/shrex.go b/share/shwap/p2p/shrex/shrex_getter/shrex.go index 142b3f3ab2..f137935d49 100644 --- a/share/shwap/p2p/shrex/shrex_getter/shrex.go +++ b/share/shwap/p2p/shrex/shrex_getter/shrex.go @@ -306,6 +306,16 @@ func (sg *Getter) GetNamespaceData( } } +func (sg *Getter) GetRangeNamespaceData( + _ context.Context, + _ *header.ExtendedHeader, + _ libshare.Namespace, + _, _ shwap.SampleCoords, + _ bool, +) (shwap.RangeNamespaceData, error) { + return shwap.RangeNamespaceData{}, errors.New("not supported") +} + func (sg *Getter) getPeer( ctx context.Context, header *header.ExtendedHeader, diff --git a/store/getter.go b/store/getter.go index 7a94c408ac..fac973936d 100644 --- a/store/getter.go +++ b/store/getter.go @@ -110,3 +110,27 @@ func (g *Getter) GetNamespaceData( } return nd, nil } + +func (g *Getter) GetRangeNamespaceData( + ctx context.Context, + h *header.ExtendedHeader, + ns libshare.Namespace, + from, to shwap.SampleCoords, + proofsOnly bool, +) (shwap.RangeNamespaceData, error) { + acc, err := g.store.GetByHeight(ctx, h.Height()) + if err != nil { + if errors.Is(err, ErrNotFound) { + return shwap.RangeNamespaceData{}, shwap.ErrNotFound + } + return shwap.RangeNamespaceData{}, fmt.Errorf("getting accessor from store:%w", err) + } + rngData, err := acc.RangeNamespaceData(ctx, ns, from, to) + if err != nil { + return shwap.RangeNamespaceData{}, fmt.Errorf("getting range from accessor:%w", err) + } + if proofsOnly { + rngData.CleanupData() + } + return rngData, nil +} From 811875a25e52a0f92613ace0b98c7208ba7f3e9f Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Fri, 7 Mar 2025 13:30:55 +0200 Subject: [PATCH 08/13] update proto --- share/shwap/pb/shwap.pb.go | 374 ++++++++++++++++++++++++++++++------- share/shwap/pb/shwap.proto | 9 +- 2 files changed, 317 insertions(+), 66 deletions(-) diff --git a/share/shwap/pb/shwap.pb.go b/share/shwap/pb/shwap.pb.go index 0c15cc07cd..46b1d0ce17 100644 --- a/share/shwap/pb/shwap.pb.go +++ b/share/shwap/pb/shwap.pb.go @@ -186,8 +186,8 @@ func (m *Sample) GetProofType() AxisType { } type RowNamespaceData struct { - Shares []*Share `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` - Proof *Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` + Shares []*Share `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` + Proof *pb.Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` } func (m *RowNamespaceData) Reset() { *m = RowNamespaceData{} } @@ -230,7 +230,7 @@ func (m *RowNamespaceData) GetShares() []*Share { return nil } -func (m *RowNamespaceData) GetProof() *Proof { +func (m *RowNamespaceData) GetProof() *pb.Proof { if m != nil { return m.Proof } @@ -281,16 +281,61 @@ func (m *NamespaceData) GetNamespaceData() []*RowNamespaceData { return nil } +type RowShares struct { + Shares []*Share `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` +} + +func (m *RowShares) Reset() { *m = RowShares{} } +func (m *RowShares) String() string { return proto.CompactTextString(m) } +func (*RowShares) ProtoMessage() {} +func (*RowShares) Descriptor() ([]byte, []int) { + return fileDescriptor_9431653f3c9f0bcb, []int{4} +} +func (m *RowShares) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *RowShares) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_RowShares.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *RowShares) XXX_Merge(src proto.Message) { + xxx_messageInfo_RowShares.Merge(m, src) +} +func (m *RowShares) XXX_Size() int { + return m.Size() +} +func (m *RowShares) XXX_DiscardUnknown() { + xxx_messageInfo_RowShares.DiscardUnknown(m) +} + +var xxx_messageInfo_RowShares proto.InternalMessageInfo + +func (m *RowShares) GetShares() []*Share { + if m != nil { + return m.Shares + } + return nil +} + type RangeNamespaceData struct { - Start int32 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` - RangeNamespaceData *NamespaceData `protobuf:"bytes,3,opt,name=rangeNamespaceData,proto3" json:"rangeNamespaceData,omitempty"` + Start int32 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` + Shares []*RowShares `protobuf:"bytes,2,rep,name=shares,proto3" json:"shares,omitempty"` + Proofs []*Proof `protobuf:"bytes,3,rep,name=proofs,proto3" json:"proofs,omitempty"` } func (m *RangeNamespaceData) Reset() { *m = RangeNamespaceData{} } func (m *RangeNamespaceData) String() string { return proto.CompactTextString(m) } func (*RangeNamespaceData) ProtoMessage() {} func (*RangeNamespaceData) Descriptor() ([]byte, []int) { - return fileDescriptor_9431653f3c9f0bcb, []int{4} + return fileDescriptor_9431653f3c9f0bcb, []int{5} } func (m *RangeNamespaceData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -326,9 +371,16 @@ func (m *RangeNamespaceData) GetStart() int32 { return 0 } -func (m *RangeNamespaceData) GetRangeNamespaceData() *NamespaceData { +func (m *RangeNamespaceData) GetShares() []*RowShares { + if m != nil { + return m.Shares + } + return nil +} + +func (m *RangeNamespaceData) GetProofs() []*Proof { if m != nil { - return m.RangeNamespaceData + return m.Proofs } return nil } @@ -341,7 +393,7 @@ func (m *Share) Reset() { *m = Share{} } func (m *Share) String() string { return proto.CompactTextString(m) } func (*Share) ProtoMessage() {} func (*Share) Descriptor() ([]byte, []int) { - return fileDescriptor_9431653f3c9f0bcb, []int{5} + return fileDescriptor_9431653f3c9f0bcb, []int{6} } func (m *Share) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -388,7 +440,7 @@ func (m *RowRootProof) Reset() { *m = RowRootProof{} } func (m *RowRootProof) String() string { return proto.CompactTextString(m) } func (*RowRootProof) ProtoMessage() {} func (*RowRootProof) Descriptor() ([]byte, []int) { - return fileDescriptor_9431653f3c9f0bcb, []int{6} + return fileDescriptor_9431653f3c9f0bcb, []int{7} } func (m *RowRootProof) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -455,7 +507,7 @@ func (m *Proof) Reset() { *m = Proof{} } func (m *Proof) String() string { return proto.CompactTextString(m) } func (*Proof) ProtoMessage() {} func (*Proof) Descriptor() ([]byte, []int) { - return fileDescriptor_9431653f3c9f0bcb, []int{7} + return fileDescriptor_9431653f3c9f0bcb, []int{8} } func (m *Proof) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -512,6 +564,7 @@ func init() { proto.RegisterType((*Sample)(nil), "shwap.Sample") proto.RegisterType((*RowNamespaceData)(nil), "shwap.RowNamespaceData") proto.RegisterType((*NamespaceData)(nil), "shwap.NamespaceData") + proto.RegisterType((*RowShares)(nil), "shwap.RowShares") proto.RegisterType((*RangeNamespaceData)(nil), "shwap.RangeNamespaceData") proto.RegisterType((*Share)(nil), "shwap.Share") proto.RegisterType((*RowRootProof)(nil), "shwap.RowRootProof") @@ -521,41 +574,42 @@ func init() { func init() { proto.RegisterFile("share/shwap/pb/shwap.proto", fileDescriptor_9431653f3c9f0bcb) } var fileDescriptor_9431653f3c9f0bcb = []byte{ - // 541 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x53, 0x41, 0x8f, 0xd2, 0x40, - 0x14, 0x66, 0x2c, 0x45, 0x78, 0x74, 0x77, 0xc9, 0xb8, 0x89, 0x04, 0x4d, 0x25, 0x8d, 0x26, 0xc4, - 0x64, 0x8b, 0xb2, 0x07, 0x4f, 0x1e, 0xd4, 0x55, 0x31, 0x59, 0x77, 0xcd, 0x83, 0xc4, 0xc4, 0x98, - 0x90, 0x01, 0x06, 0xda, 0xa4, 0x30, 0x4d, 0x3b, 0x1b, 0x76, 0xcf, 0x1e, 0xbc, 0xfa, 0xb3, 0x3c, - 0xee, 0xd1, 0xa3, 0x81, 0x3f, 0x62, 0x66, 0xa6, 0x60, 0x1b, 0xf6, 0xf6, 0x7d, 0xef, 0x7d, 0x7d, - 0xef, 0x7b, 0xaf, 0x6f, 0xa0, 0x95, 0x06, 0x2c, 0xe1, 0xdd, 0x34, 0x58, 0xb1, 0xb8, 0x1b, 0x8f, - 0x0d, 0xf0, 0xe3, 0x44, 0x48, 0x41, 0x6d, 0x4d, 0x5a, 0x87, 0xf1, 0xb8, 0x1b, 0x27, 0x42, 0xcc, - 0x4c, 0xd8, 0xfb, 0x49, 0xc0, 0x42, 0xb1, 0xa2, 0x27, 0x50, 0xd7, 0x1f, 0xa7, 0xa3, 0x80, 0x45, - 0xb3, 0x26, 0x69, 0x5b, 0x9d, 0x7a, 0xcf, 0xf1, 0x4d, 0x85, 0x81, 0xca, 0x20, 0x18, 0x41, 0x9f, - 0x45, 0x33, 0xfa, 0x02, 0x6a, 0x4a, 0x37, 0x4a, 0xc3, 0x29, 0x6f, 0xde, 0x6b, 0x93, 0xce, 0x61, - 0xef, 0x41, 0x26, 0x46, 0xb1, 0xf2, 0x95, 0x66, 0x10, 0x4e, 0x39, 0x56, 0x83, 0x0c, 0x79, 0x4f, - 0xa0, 0xba, 0x8d, 0xd2, 0x2a, 0x94, 0xcf, 0xdf, 0x7f, 0x18, 0x36, 0x4a, 0xb4, 0x06, 0x36, 0x7e, - 0xfa, 0xd8, 0x1f, 0x36, 0x88, 0xf7, 0x83, 0x40, 0x65, 0xc0, 0x16, 0x71, 0xc4, 0xa9, 0x07, 0xb6, - 0xee, 0xd5, 0x24, 0x6d, 0xb2, 0x67, 0xc3, 0xa4, 0xe8, 0x33, 0xb0, 0xf5, 0x1c, 0xba, 0x7b, 0xbd, - 0x77, 0xe4, 0x67, 0x53, 0x8d, 0xfd, 0x2f, 0x0a, 0xa0, 0xc9, 0x52, 0x1f, 0x40, 0x83, 0x91, 0xbc, - 0x89, 0x79, 0xd3, 0xd2, 0x4e, 0x8f, 0xb2, 0x7a, 0x6f, 0xae, 0xc3, 0x74, 0x78, 0x13, 0x73, 0xac, - 0x69, 0x89, 0x82, 0xde, 0x77, 0x68, 0xa0, 0x58, 0x5d, 0xb0, 0x05, 0x4f, 0x63, 0x36, 0xe1, 0x67, - 0x4c, 0x32, 0xfa, 0x14, 0x2a, 0x66, 0xf4, 0x3b, 0xd7, 0x92, 0xe5, 0x94, 0xe9, 0xbc, 0xa1, 0xad, - 0x28, 0xef, 0xc6, 0xbb, 0x80, 0x83, 0x62, 0xe9, 0xd7, 0x70, 0xb0, 0xcc, 0x07, 0xb2, 0x0e, 0x0f, - 0xff, 0xef, 0xb2, 0xa0, 0xc7, 0xa2, 0xda, 0x8b, 0x81, 0x22, 0x5b, 0xce, 0x79, 0xb1, 0xe8, 0x31, - 0xd8, 0xa9, 0x64, 0x89, 0xd4, 0xeb, 0xb3, 0xd1, 0x10, 0x7a, 0x06, 0x34, 0xd9, 0xd3, 0xea, 0x8d, - 0xd4, 0x7b, 0xc7, 0x59, 0xbf, 0x62, 0xb3, 0x3b, 0xf4, 0xde, 0x23, 0xb0, 0xf5, 0xd8, 0x94, 0x42, - 0x79, 0x6a, 0x0c, 0x93, 0x8e, 0x83, 0x1a, 0x7b, 0x11, 0x38, 0x28, 0x56, 0x28, 0x84, 0xd4, 0x53, - 0x2b, 0x23, 0x52, 0x48, 0x16, 0x69, 0x91, 0x85, 0x86, 0xa8, 0x68, 0xb8, 0x9c, 0xf2, 0x6b, 0xbd, - 0x28, 0x0b, 0x0d, 0xa1, 0x2d, 0xa8, 0x46, 0x9c, 0xcd, 0xfa, 0x2c, 0x0d, 0xb4, 0x29, 0x07, 0x77, - 0x5c, 0x7d, 0xc1, 0xae, 0x96, 0x32, 0x6d, 0x96, 0xdb, 0x56, 0xc7, 0x41, 0x43, 0xd4, 0xe9, 0xda, - 0xa6, 0xcf, 0xcb, 0xed, 0xf1, 0x6a, 0x9a, 0x5d, 0xcd, 0xde, 0x45, 0xe4, 0x35, 0xf4, 0x15, 0x38, - 0x49, 0xce, 0x6a, 0xf6, 0xd3, 0x72, 0x37, 0xbc, 0x4b, 0x61, 0x41, 0xa8, 0xe6, 0x4e, 0x84, 0x90, - 0x99, 0x47, 0x8d, 0x9f, 0x3f, 0x86, 0xea, 0xf6, 0x96, 0xe8, 0x7d, 0xb0, 0xf0, 0xf2, 0x6b, 0xa3, - 0xa4, 0xc0, 0xbb, 0xcb, 0xf3, 0x06, 0x79, 0xfb, 0xf9, 0xf7, 0xda, 0x25, 0xb7, 0x6b, 0x97, 0xfc, - 0x5d, 0xbb, 0xe4, 0xd7, 0xc6, 0x2d, 0xdd, 0x6e, 0xdc, 0xd2, 0x9f, 0x8d, 0x5b, 0xfa, 0x76, 0x3a, - 0x0f, 0x65, 0x70, 0x35, 0xf6, 0x27, 0x62, 0xd1, 0x9d, 0xf0, 0x88, 0xa7, 0x32, 0x64, 0x22, 0x99, - 0xef, 0xf0, 0xc9, 0x52, 0x4c, 0xd5, 0x73, 0xce, 0x3f, 0xea, 0x71, 0x45, 0x3f, 0xdc, 0xd3, 0x7f, - 0x01, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xd2, 0x0a, 0xe0, 0xed, 0x03, 0x00, 0x00, + // 559 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4f, 0x6f, 0xda, 0x4e, + 0x10, 0x65, 0x63, 0xcc, 0x0f, 0x06, 0x27, 0xb1, 0xf6, 0x57, 0xa9, 0x16, 0xad, 0x5c, 0x64, 0xa5, + 0x12, 0xaa, 0x14, 0xd3, 0x90, 0x43, 0x4f, 0x3d, 0xf4, 0x3f, 0x95, 0xd2, 0xa4, 0x5a, 0x90, 0x2a, + 0xf5, 0x82, 0x16, 0x58, 0xb0, 0x25, 0xc3, 0x5a, 0xde, 0x8d, 0x9c, 0xf4, 0xda, 0x43, 0xaf, 0xfd, + 0x58, 0x3d, 0xe6, 0xd8, 0x63, 0x05, 0x5f, 0xa4, 0xda, 0x5d, 0x43, 0xb0, 0xd2, 0x43, 0x6e, 0x6f, + 0x76, 0xde, 0xce, 0xbc, 0x37, 0x3b, 0x36, 0xb4, 0x44, 0x44, 0x33, 0xd6, 0x15, 0x51, 0x4e, 0xd3, + 0x6e, 0x3a, 0x36, 0x20, 0x4c, 0x33, 0x2e, 0x39, 0xb6, 0x75, 0xd0, 0x3a, 0x48, 0xc7, 0xdd, 0x34, + 0xe3, 0x7c, 0x66, 0x8e, 0x83, 0x1f, 0x08, 0x2c, 0xc2, 0x73, 0x7c, 0x0c, 0x4d, 0x7d, 0x59, 0x8c, + 0x22, 0x9a, 0xcc, 0x3c, 0xd4, 0xb6, 0x3a, 0xcd, 0x9e, 0x13, 0x9a, 0x0a, 0x03, 0x95, 0x21, 0x60, + 0x08, 0x7d, 0x9a, 0xcc, 0xf0, 0x73, 0x68, 0x28, 0xde, 0x48, 0xc4, 0x53, 0xe6, 0xed, 0xb5, 0x51, + 0xe7, 0xa0, 0xf7, 0x7f, 0x41, 0x26, 0x3c, 0x0f, 0x15, 0x67, 0x10, 0x4f, 0x19, 0xa9, 0x47, 0x05, + 0x0a, 0x9e, 0x40, 0x7d, 0x73, 0x8a, 0xeb, 0x50, 0x3d, 0x7b, 0xf7, 0x7e, 0xe8, 0x56, 0x70, 0x03, + 0x6c, 0xf2, 0xf1, 0x43, 0x7f, 0xe8, 0xa2, 0xe0, 0x3b, 0x82, 0xda, 0x80, 0x2e, 0xd2, 0x84, 0xe1, + 0x00, 0x6c, 0xdd, 0xcb, 0x43, 0x6d, 0x74, 0x47, 0x86, 0x49, 0xe1, 0xa7, 0x60, 0x6b, 0x1f, 0xba, + 0x7b, 0xb3, 0x77, 0x18, 0x16, 0xae, 0xc6, 0xe1, 0x67, 0x05, 0x88, 0xc9, 0xe2, 0x10, 0x40, 0x83, + 0x91, 0xbc, 0x4e, 0x99, 0x67, 0x69, 0xa5, 0x87, 0x45, 0xbd, 0x57, 0x57, 0xb1, 0x18, 0x5e, 0xa7, + 0x8c, 0x34, 0x34, 0x45, 0xc1, 0x60, 0x04, 0x2e, 0xe1, 0xf9, 0x39, 0x5d, 0x30, 0x91, 0xd2, 0x09, + 0x7b, 0x4b, 0x25, 0xc5, 0x47, 0x50, 0x33, 0xd6, 0xff, 0x39, 0x96, 0x22, 0x77, 0x4f, 0x41, 0xc1, + 0x39, 0xec, 0x97, 0xab, 0xbf, 0x84, 0xfd, 0xe5, 0xee, 0x41, 0xd1, 0xe4, 0xe1, 0xed, 0x38, 0x4b, + 0x7c, 0x52, 0x66, 0x07, 0x27, 0xd0, 0x20, 0x3c, 0x1f, 0x18, 0x0d, 0xf7, 0x52, 0x1a, 0x7c, 0x03, + 0x4c, 0xe8, 0x72, 0xce, 0xca, 0x3a, 0x1e, 0x80, 0x2d, 0x24, 0xcd, 0xa4, 0x1e, 0xba, 0x4d, 0x4c, + 0x80, 0x3b, 0xdb, 0x8a, 0x7b, 0xba, 0xa2, 0x7b, 0x2b, 0xcb, 0xf4, 0xdc, 0xfa, 0x3f, 0x82, 0x9a, + 0x76, 0x28, 0x3c, 0xab, 0xd4, 0xdb, 0xb8, 0x2f, 0x72, 0xc1, 0x23, 0xb0, 0xf5, 0x3d, 0x8c, 0xa1, + 0x3a, 0x35, 0x6e, 0x51, 0xc7, 0x21, 0x1a, 0x07, 0x09, 0x38, 0x84, 0xe7, 0x84, 0x73, 0xa9, 0x2f, + 0x29, 0x49, 0x92, 0x4b, 0x9a, 0x68, 0x92, 0x45, 0x4c, 0xa0, 0x4e, 0xe3, 0xe5, 0x94, 0x5d, 0xe9, + 0x41, 0x5b, 0xc4, 0x04, 0xb8, 0x05, 0xf5, 0x84, 0xd1, 0x59, 0x9f, 0x8a, 0x48, 0x3f, 0xb3, 0x43, + 0xb6, 0xb1, 0xba, 0x41, 0x2f, 0x97, 0x52, 0x78, 0xd5, 0xb6, 0xd5, 0x71, 0x88, 0x09, 0xd4, 0xea, + 0xdb, 0xa6, 0xcf, 0xc9, 0x66, 0xf9, 0x75, 0x58, 0x6c, 0xdd, 0x9d, 0x07, 0xdc, 0xe5, 0xe0, 0x17, + 0xe0, 0x64, 0x3b, 0x52, 0x8b, 0x47, 0xdf, 0xf9, 0x06, 0xb6, 0x29, 0x52, 0x22, 0x2a, 0xdf, 0x19, + 0xe7, 0xb2, 0xd0, 0xa8, 0xf1, 0xb3, 0xc7, 0x50, 0xdf, 0xec, 0x22, 0xfe, 0x0f, 0x2c, 0x72, 0xf1, + 0xc5, 0xad, 0x28, 0xf0, 0xe6, 0xe2, 0xcc, 0x45, 0xaf, 0x3f, 0xfd, 0x5a, 0xf9, 0xe8, 0x66, 0xe5, + 0xa3, 0x3f, 0x2b, 0x1f, 0xfd, 0x5c, 0xfb, 0x95, 0x9b, 0xb5, 0x5f, 0xf9, 0xbd, 0xf6, 0x2b, 0x5f, + 0x4f, 0xe7, 0xb1, 0x8c, 0x2e, 0xc7, 0xe1, 0x84, 0x2f, 0xba, 0x13, 0x96, 0x30, 0x21, 0x63, 0xca, + 0xb3, 0xf9, 0x16, 0x1f, 0x2f, 0xf9, 0x54, 0xfd, 0x0e, 0x76, 0x7f, 0x0a, 0xe3, 0x9a, 0xfe, 0xf0, + 0x4f, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0xae, 0x01, 0xd7, 0x34, 0x2d, 0x04, 0x00, 0x00, } func (m *Row) Marshal() (dAtA []byte, err error) { @@ -738,6 +792,43 @@ func (m *NamespaceData) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *RowShares) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RowShares) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *RowShares) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Shares) > 0 { + for iNdEx := len(m.Shares) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Shares[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwap(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *RangeNamespaceData) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -758,17 +849,33 @@ func (m *RangeNamespaceData) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.RangeNamespaceData != nil { - { - size, err := m.RangeNamespaceData.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err + if len(m.Proofs) > 0 { + for iNdEx := len(m.Proofs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Proofs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwap(dAtA, i, uint64(size)) } - i -= size - i = encodeVarintShwap(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + } + if len(m.Shares) > 0 { + for iNdEx := len(m.Shares) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Shares[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintShwap(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 } - i-- - dAtA[i] = 0x1a } if m.Start != 0 { i = encodeVarintShwap(dAtA, i, uint64(m.Start)) @@ -994,6 +1101,21 @@ func (m *NamespaceData) Size() (n int) { return n } +func (m *RowShares) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Shares) > 0 { + for _, e := range m.Shares { + l = e.Size() + n += 1 + l + sovShwap(uint64(l)) + } + } + return n +} + func (m *RangeNamespaceData) Size() (n int) { if m == nil { return 0 @@ -1003,9 +1125,17 @@ func (m *RangeNamespaceData) Size() (n int) { if m.Start != 0 { n += 1 + sovShwap(uint64(m.Start)) } - if m.RangeNamespaceData != nil { - l = m.RangeNamespaceData.Size() - n += 1 + l + sovShwap(uint64(l)) + if len(m.Shares) > 0 { + for _, e := range m.Shares { + l = e.Size() + n += 1 + l + sovShwap(uint64(l)) + } + } + if len(m.Proofs) > 0 { + for _, e := range m.Proofs { + l = e.Size() + n += 1 + l + sovShwap(uint64(l)) + } } return n } @@ -1412,7 +1542,7 @@ func (m *RowNamespaceData) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Proof == nil { - m.Proof = &Proof{} + m.Proof = &pb.Proof{} } if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -1523,6 +1653,90 @@ func (m *NamespaceData) Unmarshal(dAtA []byte) error { } return nil } +func (m *RowShares) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RowShares: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RowShares: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Shares", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Shares = append(m.Shares, &Share{}) + if err := m.Shares[len(m.Shares)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipShwap(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthShwap + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *RangeNamespaceData) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1571,9 +1785,9 @@ func (m *RangeNamespaceData) Unmarshal(dAtA []byte) error { break } } - case 3: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field RangeNamespaceData", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Shares", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -1600,10 +1814,42 @@ func (m *RangeNamespaceData) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.RangeNamespaceData == nil { - m.RangeNamespaceData = &NamespaceData{} + m.Shares = append(m.Shares, &RowShares{}) + if err := m.Shares[len(m.Shares)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Proofs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShwap + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthShwap + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthShwap + } + if postIndex > l { + return io.ErrUnexpectedEOF } - if err := m.RangeNamespaceData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Proofs = append(m.Proofs, &Proof{}) + if err := m.Proofs[len(m.Proofs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex diff --git a/share/shwap/pb/shwap.proto b/share/shwap/pb/shwap.proto index 8d40168ac9..e8178b9ba1 100644 --- a/share/shwap/pb/shwap.proto +++ b/share/shwap/pb/shwap.proto @@ -23,16 +23,21 @@ message Sample { message RowNamespaceData { repeated Share shares = 1; - Proof proof = 2; + proof.pb.Proof proof = 2; } message NamespaceData { repeated RowNamespaceData namespaceData = 1; } +message RowShares { + repeated Share shares = 1; +} + message RangeNamespaceData { int32 start = 1; - NamespaceData rangeNamespaceData = 3; + repeated RowShares shares = 2; + repeated Proof proofs = 3; } message Share { From 6cc8cd245675de5153ef756652dea9aa1d10c87a Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Fri, 7 Mar 2025 13:31:44 +0200 Subject: [PATCH 09/13] rework get namespace data --- share/shwap/range_namespace_data.go | 95 +++++++++++++++++++---------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/share/shwap/range_namespace_data.go b/share/shwap/range_namespace_data.go index 95b2c9f8ae..2e490dd0b2 100644 --- a/share/shwap/range_namespace_data.go +++ b/share/shwap/range_namespace_data.go @@ -14,8 +14,9 @@ import ( // RangeNamespaceData embeds `NamespaceData` and contains a contiguous range of shares // along with proofs for these shares. type RangeNamespaceData struct { - Start int `json:"start"` - NamespaceData `json:"namespace_data"` + Start int `json:"start"` + Shares [][]libshare.Share `json:"shares,omitempty"` + Proof []*Proof `json:"proof"` } // RangedNamespaceDataFromShares builds a range of namespaced data for the given coordinates: @@ -35,8 +36,11 @@ func RangedNamespaceDataFromShares( } odsSize := len(shares[0]) / 2 - nsData := make([]RowNamespaceData, 0, len(shares)) - rngData := RangeNamespaceData{Start: from.Row} + rngData := RangeNamespaceData{ + Start: from.Row, + Shares: make([][]libshare.Share, len(shares)), + Proof: make([]*Proof, len(shares)), + } for i, row := 0, from.Row; i < len(shares); i++ { rowShares := shares[i] // end index will be explicitly set only for the last row in range. @@ -82,17 +86,12 @@ func RangedNamespaceDataFromShares( if err != nil { return RangeNamespaceData{}, err } - dataRootProof := NewProof(row, &proof, roots) - nsData = append(nsData, - RowNamespaceData{Shares: rowShares[from.Col:exclusiveEnd], Proof: dataRootProof}, - ) - + rngData.Shares[i] = rowShares[from.Col:exclusiveEnd] + rngData.Proof[i] = NewProof(row, &proof, roots) // reset from.Col as we are moving to the next row. from.Col = 0 row++ } - - rngData.NamespaceData = nsData return rngData, nil } @@ -104,35 +103,38 @@ func (rngdata *RangeNamespaceData) Verify( to SampleCoords, dataHash []byte, ) error { + if len(rngdata.Shares) != len(rngdata.Proof) { + return fmt.Errorf( + "mismatch amount of row shares and proofs, %d:%d", + len(rngdata.Shares), len(rngdata.Proof), + ) + } if rngdata.IsEmpty() { return errors.New("empty data") } - if rngdata.NamespaceData[0].Proof.IsEmptyProof() { - return errors.New("empty proof") - } if from.Row != rngdata.Start { return fmt.Errorf("mismatched row: wanted: %d, got: %d", rngdata.Start, from.Row) } - if from.Col != rngdata.NamespaceData[0].Proof.Start() { - return fmt.Errorf("mismatched col: wanted: %d, got: %d", rngdata.NamespaceData[0].Proof.Start(), from.Col) + if from.Col != rngdata.Proof[0].Start() { + return fmt.Errorf("mismatched col: wanted: %d, got: %d", rngdata.Proof[0].Start(), from.Col) } - if to.Col != rngdata.NamespaceData[len(rngdata.NamespaceData)-1].Proof.End() { + if to.Col != rngdata.Proof[len(rngdata.Proof)-1].End() { return fmt.Errorf( "mismatched col: wanted: %d, got: %d", - rngdata.NamespaceData[len(rngdata.NamespaceData)-1].Proof.End(), to.Col, + rngdata.Proof[len(rngdata.Proof)-1].End(), to.Col, ) } - for i, nsData := range rngdata.NamespaceData { - if nsData.Proof.IsEmptyProof() { + for i, nsData := range rngdata.Shares { + if rngdata.Proof[i].IsEmptyProof() { return fmt.Errorf("nil proof for row: %d", rngdata.Start+i) } - if nsData.Proof.shareProof.IsOfAbsence() { + if rngdata.Proof[i].shareProof.IsOfAbsence() { return fmt.Errorf("absence proof for row: %d", rngdata.Start+i) } - err := nsData.Proof.VerifyInclusion(nsData.Shares, namespace, dataHash) + err := rngdata.Proof[i].VerifyInclusion(nsData, namespace, dataHash) if err != nil { return fmt.Errorf("%w for row: %d, %w", ErrFailedVerification, rngdata.Start+i, err) } @@ -140,29 +142,58 @@ func (rngdata *RangeNamespaceData) Verify( return nil } +// Flatten combines all shares from all rows within the namespace into a single slice. +func (nd *RangeNamespaceData) Flatten() []libshare.Share { + var shares []libshare.Share + for _, shrs := range nd.Shares { + shares = append(shares, shrs...) + } + return shares +} + func (rngdata *RangeNamespaceData) IsEmpty() bool { - return len(rngdata.NamespaceData) == 0 + return len(rngdata.Shares) == 0 && len(rngdata.Proof) == 0 } func (rngdata *RangeNamespaceData) ToProto() *pb.RangeNamespaceData { + pbShares := make([]*pb.RowShares, len(rngdata.Shares)) + pbProofs := make([]*pb.Proof, len(rngdata.Proof)) + for i, shr := range rngdata.Shares { + rowShares := SharesToProto(shr) + pbShares[i].Shares = rowShares + pbProofs[i] = rngdata.Proof[i].ToProto() + } return &pb.RangeNamespaceData{ - Start: int32(rngdata.Start), - RangeNamespaceData: rngdata.NamespaceData.ToProto(), + Start: int32(rngdata.Start), + Shares: pbShares, + Proofs: pbProofs, } } func (rngdata *RangeNamespaceData) CleanupData() { - for i := range rngdata.NamespaceData { - rngdata.NamespaceData[i].Shares = nil - } + rngdata.Shares = nil } func RangeNamespaceDataFromProto(nd *pb.RangeNamespaceData) (*RangeNamespaceData, error) { - data, err := NamespaceDataFromProto(nd.RangeNamespaceData) - if err != nil { - return nil, err + shares := make([][]libshare.Share, len(nd.Shares)) + + for i, shr := range nd.Shares { + shrs, err := SharesFromProto(shr.GetShares()) + if err != nil { + return nil, err + } + shares[i] = shrs + } + + proofs := make([]*Proof, len(nd.Proofs)) + for i, proof := range nd.Proofs { + p, err := ProofFromProto(proof) + if err != nil { + return nil, err + } + proofs[i] = p } - return &RangeNamespaceData{Start: int(nd.Start), NamespaceData: data}, nil + return &RangeNamespaceData{Start: int(nd.Start), Shares: shares, Proof: proofs}, nil } // RangeCoordsFromIdx accepts the start index and the length of the range and From f07fced999b73602c35731e69ab1e5cdd54291f5 Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Mon, 3 Mar 2025 18:49:18 +0200 Subject: [PATCH 10/13] chore(share/accessor): update accessor --- share/eds/accessor.go | 7 ++++ share/eds/close_once.go | 11 ++++++ share/eds/close_once_test.go | 8 ++++ share/eds/proofs_cache.go | 59 ++++++++++++++++++++++++++++- share/eds/proofs_cache_test.go | 2 +- share/eds/rsmt2d.go | 25 +++++++++++- share/eds/testing.go | 49 ++++++++++++++++++++++++ share/shwap/range_namespace_data.go | 2 +- store/cache/accessor_cache_test.go | 8 ++++ store/cache/noop.go | 8 ++++ store/file/ods.go | 28 ++++++++++++++ store/file/ods_q4.go | 8 ++++ 12 files changed, 211 insertions(+), 4 deletions(-) diff --git a/share/eds/accessor.go b/share/eds/accessor.go index eb3fea6e41..fd99a6f678 100644 --- a/share/eds/accessor.go +++ b/share/eds/accessor.go @@ -33,6 +33,13 @@ type Accessor interface { RowNamespaceData(ctx context.Context, namespace libshare.Namespace, rowIdx int) (shwap.RowNamespaceData, error) // Shares returns data (ODS) shares extracted from the Accessor. Shares(ctx context.Context) ([]libshare.Share, error) + // RangeNamespaceData returns data(ODS) shares along with their proofs from the requested range + // from the Accessor. Response might have only proofs. + RangeNamespaceData( + ctx context.Context, + ns libshare.Namespace, + from, to shwap.SampleCoords, + ) (shwap.RangeNamespaceData, error) } // AccessorStreamer is an interface that groups Accessor and Streamer interfaces. diff --git a/share/eds/close_once.go b/share/eds/close_once.go index 579b4b1e20..bb84e06b04 100644 --- a/share/eds/close_once.go +++ b/share/eds/close_once.go @@ -93,6 +93,17 @@ func (c *closeOnce) Shares(ctx context.Context) ([]libshare.Share, error) { return c.f.Shares(ctx) } +func (c *closeOnce) RangeNamespaceData( + ctx context.Context, + ns libshare.Namespace, + from, to shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + if c.closed.Load() { + return shwap.RangeNamespaceData{}, errAccessorClosed + } + return c.f.RangeNamespaceData(ctx, ns, from, to) +} + func (c *closeOnce) Reader() (io.Reader, error) { if c.closed.Load() { return nil, errAccessorClosed diff --git a/share/eds/close_once_test.go b/share/eds/close_once_test.go index e34299f805..1cd8e251ee 100644 --- a/share/eds/close_once_test.go +++ b/share/eds/close_once_test.go @@ -75,6 +75,14 @@ func (s *stubEdsAccessorCloser) RowNamespaceData( return shwap.RowNamespaceData{}, nil } +func (s *stubEdsAccessorCloser) RangeNamespaceData( + _ context.Context, + _ libshare.Namespace, + _, _ shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + return shwap.RangeNamespaceData{}, nil +} + func (s *stubEdsAccessorCloser) Shares(context.Context) ([]libshare.Share, error) { return nil, nil } diff --git a/share/eds/proofs_cache.go b/share/eds/proofs_cache.go index f73ebcdf80..18ac46d009 100644 --- a/share/eds/proofs_cache.go +++ b/share/eds/proofs_cache.go @@ -225,7 +225,6 @@ func (c *proofsCache) RowNamespaceData( if err != nil { return shwap.RowNamespaceData{}, fmt.Errorf("shares by namespace %s for row %v: %w", namespace.String(), rowIdx, err) } - return shwap.RowNamespaceData{ Shares: row, Proof: proof, @@ -255,6 +254,64 @@ func (c *proofsCache) Shares(ctx context.Context) ([]libshare.Share, error) { return shares, nil } +// RangeNamespaceData tries to find all complete rows in cache. For all incomplete rows, +// it uses the inner accessor to build the namespace data +func (c *proofsCache) RangeNamespaceData( + ctx context.Context, + ns libshare.Namespace, + from, to shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + odsSize := c.Size(ctx) / 2 + + roots, err := c.AxisRoots(ctx) + if err != nil { + return shwap.RangeNamespaceData{}, fmt.Errorf("accessing axis roots: %w", err) + } + rngdata := shwap.RangeNamespaceData{ + Start: from.Row, + Shares: make([][]libshare.Share, 0, to.Row-from.Row+1), + Proof: make([]*shwap.Proof, 0, to.Row-from.Row+1), + } + + // iterate over each row in the range [from.Row; to.Row]. + // All complete rows(from.Col = 0 && to.Col = odsSize-1) is + // requested using `RowNamespaceData` that uses cache. + // Other cases are handled using `RangeNamespaceData` as these rows are incomplete. + for row := from.Row; row <= to.Row; row++ { + startCol := from.Col + endCol := to.Col + if row != to.Row { + endCol = odsSize - 1 + } + + if startCol == 0 && endCol == odsSize-1 { + // request full row using RowNamespaceData + rowData, err := c.RowNamespaceData(ctx, ns, row) + if err != nil { + return shwap.RangeNamespaceData{}, err + } + rngdata.Shares = append(rngdata.Shares, rowData.Shares) + rngdata.Proof = append(rngdata.Proof, shwap.NewProof(row, rowData.Proof, roots)) + continue + } + + // Otherwise, fetch the partial range + data, err := c.inner.RangeNamespaceData( + ctx, ns, + shwap.SampleCoords{Row: row, Col: startCol}, + shwap.SampleCoords{Row: row, Col: endCol}, + ) + if err != nil { + return shwap.RangeNamespaceData{}, err + } + rngdata.Shares = append(rngdata.Shares, data.Shares[0]) + rngdata.Proof = append(rngdata.Proof, data.Proof[0]) + // Reset column for subsequent rows + from.Col = 0 + } + return rngdata, nil +} + func (c *proofsCache) Reader() (io.Reader, error) { odsSize := c.Size(context.TODO()) / 2 reader := NewShareReader(odsSize, c.getShare) diff --git a/share/eds/proofs_cache_test.go b/share/eds/proofs_cache_test.go index b570b15c1e..effb07fc28 100644 --- a/share/eds/proofs_cache_test.go +++ b/share/eds/proofs_cache_test.go @@ -10,7 +10,7 @@ import ( func TestCache(t *testing.T) { ODSSize := 16 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) newAccessor := func(tb testing.TB, inner *rsmt2d.ExtendedDataSquare) Accessor { diff --git a/share/eds/rsmt2d.go b/share/eds/rsmt2d.go index 1d806324b5..a5333f0ae6 100644 --- a/share/eds/rsmt2d.go +++ b/share/eds/rsmt2d.go @@ -104,7 +104,7 @@ func (eds *Rsmt2D) HalfRow(idx int, side shwap.RowSide) (shwap.Row, error) { // RowNamespaceData returns data for the given namespace and row index. func (eds *Rsmt2D) RowNamespaceData( - _ context.Context, + ctx context.Context, namespace libshare.Namespace, rowIdx int, ) (shwap.RowNamespaceData, error) { @@ -116,6 +116,29 @@ func (eds *Rsmt2D) RowNamespaceData( return shwap.RowNamespaceDataFromShares(sh, namespace, rowIdx) } +// RangeNamespaceData builds a namespace range from the given coordinates and the length of the +// range. +func (eds *Rsmt2D) RangeNamespaceData( + ctx context.Context, + ns libshare.Namespace, + from, to shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + rawShares := make([][]libshare.Share, 0, to.Row-from.Row+1) + for row := from.Row; row <= to.Row; row++ { + rawShare := eds.Row(uint(row)) + sh, err := libshare.FromBytes(rawShare) + if err != nil { + return shwap.RangeNamespaceData{}, err + } + rawShares = append(rawShares, sh) + } + roots, err := eds.AxisRoots(ctx) + if err != nil { + return shwap.RangeNamespaceData{}, err + } + return shwap.RangedNamespaceDataFromShares(rawShares, ns, roots, from, to) +} + // Shares returns data (ODS) shares extracted from the EDS. It returns new copy of the shares each // time. func (eds *Rsmt2D) Shares(_ context.Context) ([]libshare.Share, error) { diff --git a/share/eds/testing.go b/share/eds/testing.go index c8859dca5a..fb9e0dd5bd 100644 --- a/share/eds/testing.go +++ b/share/eds/testing.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-app/v3/pkg/da" libshare "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -66,6 +67,11 @@ func TestSuiteAccessor( t.Parallel() testAccessorShares(ctx, t, createAccessor, eds) }) + + t.Run(fmt.Sprintf("RangeNamespaceData:%s", name), func(t *testing.T) { + t.Parallel() + testRangeNamespaceData(ctx, t, createAccessor, size) + }) } } } @@ -283,6 +289,49 @@ func testAccessorRowNamespaceData( }) } +func testRangeNamespaceData( + ctx context.Context, + t *testing.T, + createAccessor createAccessor, + odsSize int, +) { + sharesAmount := odsSize * odsSize + namespace := libshare.RandomNamespace() + eds, _ := edstest.RandEDSWithNamespace(t, namespace, sharesAmount, odsSize) + acc := createAccessor(t, eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) + + // request all possible ranges + for startRow := 0; startRow < odsSize; startRow++ { + for endRow := startRow; endRow < odsSize; endRow++ { + multiplier := (endRow - startRow) * odsSize + for startCol := 0; startCol < odsSize; startCol++ { + for endCol := startCol; endCol < odsSize; endCol++ { + actualSharesAmount := endCol - startCol + multiplier + 1 + rngData, err := acc.RangeNamespaceData( + ctx, + namespace, + shwap.SampleCoords{Row: startRow, Col: startCol}, + shwap.SampleCoords{Row: endRow, Col: endCol}, + ) + require.NoError(t, err) + shares := rngData.Flatten() + require.Equal(t, actualSharesAmount, len(shares)) + require.NoError(t, err) + err = rngData.Verify( + namespace, + shwap.SampleCoords{Row: startRow, Col: startCol}, + shwap.SampleCoords{Row: endRow, Col: endCol}, + dah.Hash(), + ) + require.NoError(t, err) + } + } + } + } +} + func testAccessorAxisHalf( ctx context.Context, t *testing.T, diff --git a/share/shwap/range_namespace_data.go b/share/shwap/range_namespace_data.go index 2e490dd0b2..a455dddf7e 100644 --- a/share/shwap/range_namespace_data.go +++ b/share/shwap/range_namespace_data.go @@ -1,8 +1,8 @@ package shwap import ( + "errors" "fmt" - "github.com/pkg/errors" "github.com/celestiaorg/celestia-app/v3/pkg/wrapper" libshare "github.com/celestiaorg/go-square/v2/share" diff --git a/store/cache/accessor_cache_test.go b/store/cache/accessor_cache_test.go index 8b537049e1..887c6791d3 100644 --- a/store/cache/accessor_cache_test.go +++ b/store/cache/accessor_cache_test.go @@ -327,6 +327,14 @@ func (m *mockAccessor) RowNamespaceData(context.Context, libshare.Namespace, int panic("implement me") } +func (m *mockAccessor) RangeNamespaceData( + context.Context, + libshare.Namespace, + shwap.SampleCoords, shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + panic("implement me") +} + func (m *mockAccessor) Shares(context.Context) ([]libshare.Share, error) { panic("implement me") } diff --git a/store/cache/noop.go b/store/cache/noop.go index 27b33e2dd6..d211e81303 100644 --- a/store/cache/noop.go +++ b/store/cache/noop.go @@ -71,6 +71,14 @@ func (n NoopFile) RowNamespaceData(context.Context, libshare.Namespace, int) (sh return shwap.RowNamespaceData{}, nil } +func (n NoopFile) RangeNamespaceData( + _ context.Context, + _ libshare.Namespace, + _, _ shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + return shwap.RangeNamespaceData{}, nil +} + func (n NoopFile) Shares(context.Context) ([]libshare.Share, error) { return []libshare.Share{}, nil } diff --git a/store/file/ods.go b/store/file/ods.go index e9b14e8a80..493acb65bf 100644 --- a/store/file/ods.go +++ b/store/file/ods.go @@ -317,6 +317,34 @@ func (o *ODS) Reader() (io.Reader, error) { return reader, nil } +func (o *ODS) RangeNamespaceData( + ctx context.Context, + ns libshare.Namespace, + from, to shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + shares := make([][]libshare.Share, to.Row-from.Row+1) + + for row, idx := from.Row, 0; row <= to.Row; row++ { + half, err := o.readAxisHalf(rsmt2d.Row, row) + if err != nil { + return shwap.RangeNamespaceData{}, fmt.Errorf("reading axis half: %w", err) + } + + sh, err := half.Extended() + if err != nil { + return shwap.RangeNamespaceData{}, fmt.Errorf("extending the data: %w", err) + } + shares[idx] = sh + idx++ + } + + roots, err := o.AxisRoots(ctx) + if err != nil { + return shwap.RangeNamespaceData{}, err + } + return shwap.RangedNamespaceDataFromShares(shares, ns, roots, from, to) +} + func (o *ODS) axis(ctx context.Context, axisType rsmt2d.Axis, axisIdx int) ([]libshare.Share, error) { half, err := o.AxisHalf(ctx, axisType, axisIdx) if err != nil { diff --git a/store/file/ods_q4.go b/store/file/ods_q4.go index f0ca686094..89476696fd 100644 --- a/store/file/ods_q4.go +++ b/store/file/ods_q4.go @@ -189,3 +189,11 @@ func (odsq4 *ODSQ4) Close() error { } return err } + +func (odsq4 *ODSQ4) RangeNamespaceData( + ctx context.Context, + ns libshare.Namespace, + from, to shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + return odsq4.ods.RangeNamespaceData(ctx, ns, from, to) +} From d631cb9f36b42b119cee98882931e616875e6d5f Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Fri, 7 Mar 2025 13:59:26 +0200 Subject: [PATCH 11/13] feat: implement validation for RangeNamespaceData --- share/eds/validation.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/share/eds/validation.go b/share/eds/validation.go index 4f6cf0aa85..68d692e3db 100644 --- a/share/eds/validation.go +++ b/share/eds/validation.go @@ -61,3 +61,29 @@ func (f validation) RowNamespaceData( } return f.Accessor.RowNamespaceData(ctx, namespace, rowIdx) } + +func (f validation) RangeNamespaceData( + ctx context.Context, + ns libshare.Namespace, + from, to shwap.SampleCoords, +) (shwap.RangeNamespaceData, error) { + if from.Row > to.Row { + return shwap.RangeNamespaceData{}, fmt.Errorf( + "range validation: from row %d is > then to row %d", from.Row, to.Row, + ) + } + odsSharesAmount := f.Size(ctx)/2 - 1 + if from.Row > odsSharesAmount || from.Col > odsSharesAmount { + return shwap.RangeNamespaceData{}, fmt.Errorf( + "range validation: invalid start coordinates of the range:{%d;%d}. ODS shares amount %d", + from.Row, from.Col, odsSharesAmount, + ) + } + if to.Row > odsSharesAmount || to.Col > odsSharesAmount { + return shwap.RangeNamespaceData{}, fmt.Errorf( + "range validation: invalid end coordinates of the range:{%d;%d}. ODS shares amount %d", + to.Row, to.Col, odsSharesAmount, + ) + } + return f.Accessor.RangeNamespaceData(ctx, ns, from, to) +} From 77e99cee0d080d29b56ba8cedc7a4d77b8587c53 Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Fri, 7 Mar 2025 14:06:25 +0200 Subject: [PATCH 12/13] fix lint --- share/eds/rsmt2d.go | 2 +- share/shwap/range_namespace_data.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/share/eds/rsmt2d.go b/share/eds/rsmt2d.go index a5333f0ae6..3c4791fbee 100644 --- a/share/eds/rsmt2d.go +++ b/share/eds/rsmt2d.go @@ -104,7 +104,7 @@ func (eds *Rsmt2D) HalfRow(idx int, side shwap.RowSide) (shwap.Row, error) { // RowNamespaceData returns data for the given namespace and row index. func (eds *Rsmt2D) RowNamespaceData( - ctx context.Context, + _ context.Context, namespace libshare.Namespace, rowIdx int, ) (shwap.RowNamespaceData, error) { diff --git a/share/shwap/range_namespace_data.go b/share/shwap/range_namespace_data.go index a455dddf7e..0ef29a160e 100644 --- a/share/shwap/range_namespace_data.go +++ b/share/shwap/range_namespace_data.go @@ -143,9 +143,9 @@ func (rngdata *RangeNamespaceData) Verify( } // Flatten combines all shares from all rows within the namespace into a single slice. -func (nd *RangeNamespaceData) Flatten() []libshare.Share { +func (rngdata *RangeNamespaceData) Flatten() []libshare.Share { var shares []libshare.Share - for _, shrs := range nd.Shares { + for _, shrs := range rngdata.Shares { shares = append(shares, shrs...) } return shares From 4093946715f8d1cd4588a20e651d66c0ca12476a Mon Sep 17 00:00:00 2001 From: vgonkivs Date: Fri, 7 Mar 2025 14:44:20 +0200 Subject: [PATCH 13/13] fixes --- nodebuilder/tests/share_test.go | 6 ++++ share/shwap/range_namespace_data.go | 34 +++++++++++++++----- share/shwap/range_namespace_data_test.go | 40 ++++++++++++++++++++++++ 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/nodebuilder/tests/share_test.go b/nodebuilder/tests/share_test.go index 13d690a086..8d403aaaa8 100644 --- a/nodebuilder/tests/share_test.go +++ b/nodebuilder/tests/share_test.go @@ -232,6 +232,12 @@ func TestShareModule(t *testing.T) { parsedBlob, err := blob.ToNodeBlobs(blbs...) require.NoError(t, err) require.Equal(t, nodeBlob[0].Commitment, parsedBlob[0].Commitment) + + rngProofsOnly, err := client.Share.GetRange(ctx, nodeBlob[0].Namespace(), height, from, to, true) + require.NoError(t, err) + assert.Empty(t, rngProofsOnly.Shares) + err = rngProofsOnly.VerifyShares(rng.Shares, nodeBlob[0].Namespace(), from, to, dah.Hash()) + require.NoError(t, err) } }, }, diff --git a/share/shwap/range_namespace_data.go b/share/shwap/range_namespace_data.go index 0ef29a160e..b16da61668 100644 --- a/share/shwap/range_namespace_data.go +++ b/share/shwap/range_namespace_data.go @@ -25,6 +25,9 @@ type RangeNamespaceData struct { // namespace is the target namespace for the built range; // from is the coordinates of the first share of the range within the EDS. // to is the coordinates of the last inclusive share of the range within the EDS. +// TODO(@vgonkivs): proof collection can be simplified to store only +// incomplete rows(the first and the last), since we can recompute proofs +// for all complete rows func RangedNamespaceDataFromShares( shares [][]libshare.Share, namespace libshare.Namespace, @@ -95,18 +98,30 @@ func RangedNamespaceDataFromShares( return rngData, nil } -// Verify performs a basic validation of the incoming data. It ensures that the response data -// correspond to the request. +// Verify verifies the underlying shares are included in the data root. func (rngdata *RangeNamespaceData) Verify( namespace libshare.Namespace, from SampleCoords, to SampleCoords, dataHash []byte, ) error { - if len(rngdata.Shares) != len(rngdata.Proof) { + return rngdata.VerifyShares(rngdata.Shares, namespace, from, to, dataHash) +} + +// VerifyShares verifies the passed shares are included in the data root. +// `[][]libshare.Share` is a collection of the row shares from the range +// NOTE: the underlying shares will be ignored even if they are not empty. +func (rngdata *RangeNamespaceData) VerifyShares( + shares [][]libshare.Share, + namespace libshare.Namespace, + from SampleCoords, + to SampleCoords, + dataHash []byte, +) error { + if len(shares) != len(rngdata.Proof) { return fmt.Errorf( "mismatch amount of row shares and proofs, %d:%d", - len(rngdata.Shares), len(rngdata.Proof), + len(shares), len(rngdata.Proof), ) } if rngdata.IsEmpty() { @@ -126,7 +141,7 @@ func (rngdata *RangeNamespaceData) Verify( ) } - for i, nsData := range rngdata.Shares { + for i, rowShares := range shares { if rngdata.Proof[i].IsEmptyProof() { return fmt.Errorf("nil proof for row: %d", rngdata.Start+i) } @@ -134,7 +149,7 @@ func (rngdata *RangeNamespaceData) Verify( return fmt.Errorf("absence proof for row: %d", rngdata.Start+i) } - err := rngdata.Proof[i].VerifyInclusion(nsData, namespace, dataHash) + err := rngdata.Proof[i].VerifyInclusion(rowShares, namespace, dataHash) if err != nil { return fmt.Errorf("%w for row: %d, %w", ErrFailedVerification, rngdata.Start+i, err) } @@ -159,10 +174,15 @@ func (rngdata *RangeNamespaceData) ToProto() *pb.RangeNamespaceData { pbShares := make([]*pb.RowShares, len(rngdata.Shares)) pbProofs := make([]*pb.Proof, len(rngdata.Proof)) for i, shr := range rngdata.Shares { + pbShares[i] = &pb.RowShares{} rowShares := SharesToProto(shr) pbShares[i].Shares = rowShares - pbProofs[i] = rngdata.Proof[i].ToProto() } + + for i, proof := range rngdata.Proof { + pbProofs[i] = proof.ToProto() + } + return &pb.RangeNamespaceData{ Start: int32(rngdata.Start), Shares: pbShares, diff --git a/share/shwap/range_namespace_data_test.go b/share/shwap/range_namespace_data_test.go index a3b300180a..4787a7e8b3 100644 --- a/share/shwap/range_namespace_data_test.go +++ b/share/shwap/range_namespace_data_test.go @@ -2,6 +2,7 @@ package shwap_test import ( "context" + "encoding/json" "fmt" "testing" "time" @@ -71,6 +72,9 @@ func TestRangeNamespaceData(t *testing.T) { require.NoError(t, err) err = rngdata.Verify(ns, dataID.From, dataID.To, root.Hash()) require.NoError(t, err) + + err = rngdata.VerifyShares(rngdata.Shares, ns, dataID.From, dataID.To, root.Hash()) + require.NoError(t, err) }) } } @@ -96,6 +100,42 @@ func TestRangeCoordsFromIdx(t *testing.T) { } } +func TestRangeNamespaceDataMarshalUnmarshal(t *testing.T) { + const ( + odsSize = 4 + sharesAmount = odsSize * odsSize + ) + + ns := libshare.RandomNamespace() + square, root := edstest.RandEDSWithNamespace(t, ns, sharesAmount, odsSize) + eds := eds.Rsmt2D{ExtendedDataSquare: square} + + from := shwap.SampleCoords{Row: 0, Col: 0} + to := shwap.SampleCoords{Row: odsSize - 1, Col: odsSize - 1} + rngdata, err := eds.RangeNamespaceData( + context.Background(), + ns, + from, + to, + ) + require.NoError(t, err) + err = rngdata.Verify(ns, from, to, root.Hash()) + require.NoError(t, err) + + data, err := json.Marshal(rngdata) + require.NoError(t, err) + + newData := &shwap.RangeNamespaceData{} + err = json.Unmarshal(data, newData) + require.NoError(t, err) + assert.Equal(t, rngdata, *newData) + + pbData := newData.ToProto() + rangeNsData, err := shwap.RangeNamespaceDataFromProto(pbData) + require.NoError(t, err) + assert.Equal(t, rngdata, *rangeNsData) +} + func FuzzRangeCoordsFromIdx(f *testing.F) { if testing.Short() { f.Skip()