From d5eea20a92df16b5c8ee0ee6cba56bd22e741e2c Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Wed, 27 May 2026 14:36:28 +0000 Subject: [PATCH 1/3] SubtreeConsistency --- proof/proof.go | 60 ++++++++++++- proof/proof_test.go | 201 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 2 deletions(-) diff --git a/proof/proof.go b/proof/proof.go index dbf6cb5..0f05a46 100644 --- a/proof/proof.go +++ b/proof/proof.go @@ -73,8 +73,8 @@ func SubtreeInclusion(index, start, end uint64) (Nodes, error) { for n := range p.IDs { p.IDs[n].Index += start >> p.IDs[n].Level } - // For consistency, always shift p.ephem, regardless of whether it will be - // used by the proof. + // p.ephem might not be used by the resulting proof, but shift it + // unconditionally for uniformity. p.ephem.Index += start >> p.ephem.Level return p, nil @@ -110,6 +110,62 @@ func Consistency(size1, size2 uint64) (Nodes, error) { return p, nil } +// SubtreeConsistency returns the information on how to fetch and construct a +// consistency proof between a Merkle subtree covering [start, end) and log +// Merkle tree of a given size. It requires: +// - 0 <= start < end <= size +// - start to be a multiple of the smallest power of two greater than or equal to +// (end - start) +func SubtreeConsistency(start, end, size uint64) (Nodes, error) { + if err := isSubtreeValid(start, end); err != nil { + return Nodes{}, fmt.Errorf("subtree invalid: %v", err) + } + if end > size { + return Nodes{}, fmt.Errorf("subtree end %d strictly greater than tree size %d", end, size) + } + if start == 0 && end == size { + return Nodes{IDs: []compact.NodeID{}}, nil + } + + // If end == size, prove inclusion of [start, end) into the tree. + if end == size { + // Find the subtree's root, the lowest common ancestor of entries |start| and + // |end-1|. + level := uint(bits.Len64((end - 1) ^ start)) + index := (end - 1) >> level + + // Shift the tree down by |level|. + p := nodes(index, 0, index+1) + // The first node of the proof is the subtree's root. It is already known + // by the client and can be skipped. + p = p.skipFirst() + + // Shift the nodes back up. + for n := range p.IDs { + p.IDs[n].Level += level + } + // p.ephem might not be used by the resulting proof, but shift it + // unconditionally for uniformity. + p.ephem.Level += level + return p, nil + } + + // Find the root of the biggest perfect subtree of [start, end) ending at end. + level := uint(bits.TrailingZeros64(end - start)) + index := (end - 1) >> level + + // The consistency proof consists of this node (except if the subtree is full, + // in which case adding this node would be redundant because the client is + // assumed to know it from a checkpoint), and nodes of the inclusion proof + // of this node in the tree of the given size. + p := nodes(index, level, size) + // Handle the case when the subtree size is a power of 2. + if (end-start)&(end-start-1) == 0 { + return p.skipFirst(), nil + } + return p, nil +} + // nodes returns the node IDs necessary to prove that the (level, index) node // is included in the Merkle tree of the given size. func nodes(index uint64, level uint, size uint64) Nodes { diff --git a/proof/proof_test.go b/proof/proof_test.go index 41393fc..11b85be 100644 --- a/proof/proof_test.go +++ b/proof/proof_test.go @@ -412,6 +412,191 @@ func TestConsistency(t *testing.T) { } } +// TestSubtreeConsistency contains consistency proof tests. For reference, consider +// the following example: +// +// aaaaa <== Level 4 +// / \ +// ... ... +// / \ +// / \ +// / \ +// aaaa bbbb <== Level 3 +// / \ / \ +// / \ / \ +// / \ / \ +// / \ / \ +// / \ / \ +// aaa bbb ccc ddd <== Level 2 +// / \ / \ / \ / \ +// / \ / \ / \ / \ +// / \ / \ / \ / \ +// aa bb cc dd ee ff gg hh ii <== Level 1 +// / \ / \ / \ / \ / \ / \ / \ / \ / \ +// a b c d e f g h i j k l m n o p q r <== Level 0 +// | | | | | | | | | | | | | | | | | | +// d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 d10 d12 d14 d16 +// | | | | +// d11 d13 d15 d17 +// +// The consistency proof between tree size 5 and 7 consists of nodes e, f, g, +// and aaa. The node g is taken instead of its missing parent. +func TestSubtreeConsistency(t *testing.T) { + id := compact.NewNodeID + nodes := func(ids ...compact.NodeID) Nodes { + return Nodes{IDs: ids} + } + rehash := func(begin, end int, ids ...compact.NodeID) Nodes { + return Nodes{IDs: ids, begin: begin, end: end} + } + for _, tc := range []struct { + start uint64 + end uint64 + size uint64 + want Nodes + wantErr bool + }{ + // Errors. + {start: 0, end: 0, size: 0, wantErr: true}, // start = end = 0 + {start: 1, end: 1, size: 1, wantErr: true}, // start = end + {start: 2, end: 1, size: 0, wantErr: true}, // start > end + {start: 0, end: 5, size: 0, wantErr: true}, // end > size + {start: 0, end: 9, size: 8, wantErr: true}, // end > size + {start: 3, end: 5, size: 3, wantErr: true}, // start not multiple of bit_ceil(end-start) + {start: 1, end: 1<<63 + 2, size: 1<<63 + 2, wantErr: true}, // start not multiple of bit_ceil(len) with big tree + + // Small trees. + // start = 0 + {start: 0, end: 1, size: 2, want: nodes(id(0, 1))}, // b + {start: 0, end: 1, size: 4, want: nodes(id(0, 1), id(1, 1))}, // b bb + {start: 0, end: 1, size: 6, want: rehash(2, 3, id(0, 1), id(1, 1), id(1, 2))}, // b bb cc + {start: 0, end: 2, size: 3, want: rehash(0, 1, id(0, 2))}, // c + {start: 0, end: 2, size: 8, want: nodes(id(1, 1), id(2, 1))}, // bb bbb + {start: 0, end: 3, size: 7, want: rehash(3, 5, // bbb=hash(cc,g) + id(0, 2), id(0, 3), id(1, 0), id(0, 6), id(1, 2))}, // c d aa g cc + {start: 0, end: 4, size: 7, want: rehash(0, 2, // bbb=hash(cc,g) + id(0, 6), id(1, 2))}, // g cc + {start: 0, end: 5, size: 7, want: rehash(2, 3, + id(0, 4), id(0, 5), id(0, 6), id(2, 0))}, // e f g aaa + {start: 0, end: 6, size: 7, want: rehash(1, 2, + id(1, 2), id(0, 6), id(2, 0))}, // cc g aaa + {start: 0, end: 6, size: 8, want: nodes( + id(1, 2), id(1, 3), id(2, 0))}, // cc h aaa + {start: 0, end: 7, size: 8, want: nodes( + id(0, 6), id(0, 7), id(1, 2), id(2, 0))}, // g h cc aaa + // start > 0 + {start: 1, end: 2, size: 3, want: rehash(1, 2, id(0, 0), id(0, 2))}, // a c + {start: 1, end: 2, size: 5, want: rehash(2, 3, id(0, 0), id(1, 1), id(0, 4))}, // a bb e + {start: 2, end: 4, size: 5, want: rehash(1, 2, id(1, 0), id(0, 4))}, // aa e + {start: 1, end: 2, size: 7, want: rehash(2, 4, id(0, 0), id(1, 1), id(0, 6), id(1, 2))}, // a bb g cc + {start: 2, end: 4, size: 10, want: rehash(2, 3, id(1, 0), id(2, 1), id(1, 4))}, // aa bbb ee + {start: 4, end: 6, size: 10, want: rehash(2, 3, id(1, 3), id(2, 0), id(1, 4))}, // dd aaa ee + {start: 4, end: 7, size: 11, want: rehash(4, 6, // ccc=hash(ee,k) + id(0, 6), id(0, 7), id(1, 2), id(2, 0), id(0, 10), id(1, 4))}, // g h cc aaa k ee + {start: 4, end: 8, size: 11, want: rehash(1, 3, // ccc=hash(ee,k) + id(2, 0), id(0, 10), id(1, 4))}, // aaa k ee + {start: 8, end: 13, size: 15, want: rehash(2, 3, + id(0, 12), id(0, 13), id(0, 14), id(2, 2), id(3, 0))}, // m n o ccc aaaa + {start: 8, end: 14, size: 15, want: rehash(1, 2, // hh=hash(o) + id(1, 6), id(0, 14), id(2, 2), id(3, 0))}, // gg, o, ccc, aaaa + {start: 8, end: 14, size: 16, want: nodes( + id(1, 6), id(1, 7), id(2, 2), id(3, 0))}, // gg hh ccc aaaa + {start: 8, end: 15, size: 16, want: nodes( + id(0, 14), id(0, 15), id(1, 6), id(2, 2), id(3, 0))}, // o p gg ccc aaaa + // end = size + {start: 1, end: 2, size: 2, want: nodes(id(0, 0))}, // a + {start: 3, end: 4, size: 4, want: nodes(id(0, 2), id(1, 0))}, // c aa + {start: 5, end: 6, size: 6, want: nodes(id(0, 4), id(2, 0))}, // e aaa + {start: 2, end: 3, size: 3, want: nodes(id(1, 0))}, // aa + {start: 6, end: 8, size: 8, want: nodes(id(1, 2), id(2, 0))}, // cc aaa + {start: 4, end: 7, size: 7, want: nodes(id(2, 0))}, // aaa + {start: 6, end: 7, size: 7, want: nodes(id(1, 2), id(2, 0))}, // cc aaa + {start: 4, end: 8, size: 8, want: nodes(id(2, 0))}, // aaa + {start: 7, end: 8, size: 8, want: nodes(id(0, 6), id(1, 2), id(2, 0))}, // g h cc aaa + + // Same tree size. + {start: 0, end: 1, size: 1, want: Nodes{IDs: []compact.NodeID{}}}, + {start: 0, end: 2, size: 2, want: Nodes{IDs: []compact.NodeID{}}}, + {start: 0, end: 3, size: 3, want: Nodes{IDs: []compact.NodeID{}}}, + {start: 0, end: 4, size: 4, want: Nodes{IDs: []compact.NodeID{}}}, + {start: 0, end: 5, size: 5, want: Nodes{IDs: []compact.NodeID{}}}, + {start: 0, end: 7, size: 7, want: Nodes{IDs: []compact.NodeID{}}}, + {start: 0, end: 8, size: 8, want: Nodes{IDs: []compact.NodeID{}}}, + + // Smaller trees within a bigger stored tree. + // start = 0 + {start: 0, end: 2, size: 4, want: nodes(id(1, 1))}, // bb + {start: 0, end: 3, size: 5, want: rehash(3, 4, + id(0, 2), id(0, 3), id(1, 0), id(0, 4))}, // c d aa e + {start: 0, end: 3, size: 6, want: rehash(3, 4, + id(0, 2), id(0, 3), id(1, 0), id(1, 2))}, // c d aa cc + {start: 0, end: 4, size: 6, want: rehash(0, 1, id(1, 2))}, // cc + {start: 0, end: 1, size: 7, want: rehash(2, 4, // bbb=hash(cc,g) + id(0, 1), id(1, 1), id(0, 6), id(1, 2))}, // b bb g cc + // start > 0 + {start: 2, end: 4, size: 6, want: rehash(1, 2, // bbb=hash(cc) + id(1, 0), id(1, 2))}, // aa, cc + {start: 4, end: 7, size: 9, want: rehash(4, 5, // bbbb=hash(i) + id(0, 6), id(0, 7), id(1, 2), id(2, 0), id(0, 8))}, // g h cc aaa i + {start: 4, end: 7, size: 10, want: rehash(4, 5, // bbbb=hash(ee) + id(0, 6), id(0, 7), id(1, 2), id(2, 0), id(1, 4))}, // g h cc aaa ee + {start: 4, end: 8, size: 10, want: rehash(1, 2, //ccc=hash(ee) + id(2, 0), id(1, 4))}, // aa ee + {start: 2, end: 3, size: 9, want: rehash(3, 4, // bbbb=hash(i) + id(0, 3), id(1, 0), id(2, 1), id(0, 8))}, // d aa bbb i + // end = size + {start: 4, end: 6, size: 6, want: nodes(id(2, 0))}, // aaa + {start: 8, end: 9, size: 9, want: nodes(id(3, 0))}, // aaaa + {start: 8, end: 10, size: 10, want: nodes(id(3, 0))}, // aaaa + {start: 8, end: 12, size: 12, want: nodes(id(3, 0))}, // aaaa + + // Some rehashes in the middle of the returned list. + {start: 0, end: 10, size: 15, want: rehash(2, 4, + id(1, 4), id(1, 5), id(0, 14), id(1, 6), id(3, 0))}, + {start: 16, end: 26, size: 31, want: rehash(2, 4, + id(1, 12), id(1, 13), id(0, 30), id(1, 14), id(3, 2), id(4, 0))}, + {start: 0, end: 24, size: 31, want: rehash(1, 4, + id(3, 2), + id(0, 30), id(1, 14), id(2, 6), + id(4, 0), + )}, + {start: 32, end: 56, size: 63, want: rehash(1, 4, + id(3, 6), + id(0, 62), id(1, 30), id(2, 14), + id(4, 2), + id(5, 0), + )}, + {start: 0, end: 81, size: 95, want: rehash(4, 7, + id(0, 80), id(0, 81), id(1, 41), id(2, 21), + id(0, 94), id(1, 46), id(2, 22), + id(4, 4), id(6, 0), + )}, + {start: 128, end: 209, size: 223, want: rehash(4, 7, + id(0, 208), id(0, 209), id(1, 105), id(2, 53), + id(0, 222), id(1, 110), id(2, 54), + id(4, 12), id(6, 2), + id(7, 0), + )}, + } { + t.Run(fmt.Sprintf("%d:%d:%d", tc.start, tc.end, tc.size), func(t *testing.T) { + proof, err := SubtreeConsistency(tc.start, tc.end, tc.size) + if tc.wantErr { + if err == nil { + t.Fatal("accepted bad params") + } + return + } else if err != nil { + t.Fatalf("Consistency: %v", err) + } + // Ignore the ephemeral node, it is tested separately. + proof.ephem = compact.NodeID{} + if diff := cmp.Diff(tc.want, proof, cmp.AllowUnexported(Nodes{})); diff != "" { + t.Errorf("paths mismatch:\n%v", diff) + } + }) + } +} + func TestInclusionSucceedsUpToTreeSize(t *testing.T) { const maxSize = uint64(555) for ts := uint64(1); ts <= maxSize; ts++ { @@ -450,6 +635,22 @@ func TestConsistencySucceedsUpToTreeSize(t *testing.T) { } } +func TestSubtreeConsistencySucceedsUpToTreeSize(t *testing.T) { + const maxSize = uint64(100) + for s := uint64(1); s <= maxSize; s++ { + for sbe := uint64(1); sbe <= s; sbe++ { + for sbs := range sbe { + if err := isSubtreeValid(sbs, sbe); err != nil { + continue + } + if _, err := SubtreeConsistency(sbs, sbe, s); err != nil { + t.Errorf("SubtreeConsistency(sbs:%d, sbe:%d, s:%d) = %v", sbs, sbe, s, err) + } + } + } + } +} + func TestEphem(t *testing.T) { id := compact.NewNodeID for _, tc := range []struct { From c0d56c72ee3514a79447894aacf47e230224c66f Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Fri, 29 May 2026 11:25:44 +0000 Subject: [PATCH 2/3] fold use SubtreeConsistency for Consistency --- proof/proof.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/proof/proof.go b/proof/proof.go index 0f05a46..c5721f8 100644 --- a/proof/proof.go +++ b/proof/proof.go @@ -90,24 +90,7 @@ func Consistency(size1, size2 uint64) (Nodes, error) { if size1 == 0 { return Nodes{}, fmt.Errorf("consistency proof from empty tree is meaningless") } - if size1 == size2 { - return Nodes{IDs: []compact.NodeID{}}, nil - } - - // Find the root of the biggest perfect subtree that ends at size1. - level := uint(bits.TrailingZeros64(size1)) - index := (size1 - 1) >> level - // The consistency proof consists of this node (except if size1 is a power of - // two, in which case adding this node would be redundant because the client - // is assumed to know it from a checkpoint), and nodes of the inclusion proof - // of this node in the tree of size2. - p := nodes(index, level, size2) - - // Handle the case when size1 is a power of 2. - if index == 0 { - return p.skipFirst(), nil - } - return p, nil + return subtreeConsistency(0, size1, size2) } // SubtreeConsistency returns the information on how to fetch and construct a @@ -123,6 +106,10 @@ func SubtreeConsistency(start, end, size uint64) (Nodes, error) { if end > size { return Nodes{}, fmt.Errorf("subtree end %d strictly greater than tree size %d", end, size) } + return subtreeConsistency(start, end, size) +} + +func subtreeConsistency(start, end, size uint64) (Nodes, error) { if start == 0 && end == size { return Nodes{IDs: []compact.NodeID{}}, nil } From 255e881ac82ad585ea3458795597e041b70ab13e Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Tue, 23 Jun 2026 07:18:11 +0000 Subject: [PATCH 3/3] comment edits --- proof/proof.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/proof/proof.go b/proof/proof.go index c5721f8..a232bb1 100644 --- a/proof/proof.go +++ b/proof/proof.go @@ -94,8 +94,8 @@ func Consistency(size1, size2 uint64) (Nodes, error) { } // SubtreeConsistency returns the information on how to fetch and construct a -// consistency proof between a Merkle subtree covering [start, end) and log -// Merkle tree of a given size. It requires: +// consistency proof between a Merkle subtree covering [start, end) and the +// larger parent Merkle tree of a given size. It requires: // - 0 <= start < end <= size // - start to be a multiple of the smallest power of two greater than or equal to // (end - start) @@ -118,13 +118,15 @@ func subtreeConsistency(start, end, size uint64) (Nodes, error) { if end == size { // Find the subtree's root, the lowest common ancestor of entries |start| and // |end-1|. + // xor trims the common prefix between the first and last entry. The bit len + // of the result is the height of the subtree. level := uint(bits.Len64((end - 1) ^ start)) + // Then, shift the tree down by |level| to make this node a leaf. index := (end - 1) >> level - // Shift the tree down by |level|. p := nodes(index, 0, index+1) // The first node of the proof is the subtree's root. It is already known - // by the client and can be skipped. + // by the client and should be skipped. p = p.skipFirst() // Shift the nodes back up.