Align subtree definition with latest specs

cmd/proofgen/main.go

@@ -715,47 +715,92 @@ func corruptedSubtreeConsistencyProbes(dir string, size1, size2 uint64, proof []
 	return nil
 }
 
-func invalidSubtreeConsistencyProof(size1, size2 uint64, root1, root2 []byte, proof [][]byte) []subtreeConsistencyProbe {
-	cProbes := invalidConsistencyProof(size1, size2, root1, root2, proof)
-	ret := make([]subtreeConsistencyProbe, 0, len(cProbes)+1)
-	for _, p := range cProbes {
-		ret = append(ret, subtreeConsistencyProbe{
-			Start:     0,
-			End:       p.Size1,
-			Size:      p.Size2,
-			Root1:     p.Root1,
-			Root2:     p.Root2,
-			Proof:     p.Proof,
-			Desc:      p.Desc,
-			WantError: p.WantError,
-		})
-	}
-	ret = append(ret, subtreeConsistencyProbe{
-		Start:     1,
-		End:       15,
-		Size:      15,
-		Root1:     root1,
-		Root2:     root2,
-		Proof:     proof,
-		Desc:      "invalid subtree",
-		WantError: true,
-	})
+func invalidSubtreeConsistencyProof(end, size uint64, root1, root2 []byte, proof [][]byte) []subtreeConsistencyProbe {
+	ln := len(proof)
+	ret := []subtreeConsistencyProbe{
+		// Wrong end (size1).
+		{0, end - 1, size, root1, root2, proof, "size1 sub @1", true},
+		{0, end + 1, size, root1, root2, proof, "size1 plus @1", true},
+		{0, end ^ 2, size, root1, root2, proof, "size1 XOR @2", true},
+		// Wrong tree size (size2).
+		{0, end, size * 2, root1, root2, proof, "size2 mul @2", true},
+		{0, end, size / 2, root1, root2, proof, "size2 div @2", true},
+		// Wrong root.
+		{0, end, size, []byte("WrongRoot"), root2, proof, "wrong root1", true},
+		{0, end, size, root1, []byte("WrongRoot"), proof, "wrong root2", true},
+		{0, end, size, root2, root1, proof, "swapped roots", true},
+		// Empty proof.
+		{0, end, size, root1, root2, [][]byte{}, "empty proof", true},
+		// Add garbage at the end.
+		{0, end, size, root1, root2, extend(proof, []byte{}), "trailing garbage", true},
+		{0, end, size, root1, root2, extend(proof, root1), "trailing root1", true},
+		{0, end, size, root1, root2, extend(proof, root2), "trailing root2", true},
+		// Add garbage at the front.
+		{0, end, size, root1, root2, prepend(proof, []byte{}), "preceding garbage", true},
+		{0, end, size, root1, root2, prepend(proof, root1), "preceding root1", true},
+		{0, end, size, root1, root2, prepend(proof, root2), "preceding root2", true},
+		{0, end, size, root1, root2, prepend(proof, proof[0]), "preceding proof @0", true},
+	}
+
+	// Remove a node from the end.
+	if ln > 0 {
+		ret = append(ret, subtreeConsistencyProbe{0, end, size, root1, root2, proof[:ln-1], "truncated proof", true})
+	}
+
+	// Modify single bit in an element of the proof.
+	for i := range ln {
+		wrongProof := prepend(proof)                          // Copy the proof slice.
+		wrongProof[i] = append([]byte(nil), wrongProof[i]...) // But also the modified data.
+		wrongProof[i][0] ^= 16                                // Flip the bit.
+		desc := fmt.Sprintf("modified proof@%d bit @4", i)
+		ret = append(ret, subtreeConsistencyProbe{0, end, size, root1, root2, wrongProof, desc, true})
+	}
+
 	return ret
 }
 
 func staticSubtreeConsistencyProbes(dir string) error {
-	for _, p := range staticConsistencyProbes() {
-		sp := subtreeConsistencyProbe{
-			Start:     0,
-			End:       p.Size1,
-			Size:      p.Size2,
-			Root1:     p.Root1,
-			Root2:     p.Root2,
-			Proof:     p.Proof,
-			Desc:      p.Desc,
-			WantError: p.WantError,
-		}
-		if err := writeSubtreeConsistencyProbe(dir, sp); err != nil {
+	root1 := []byte("don't care 1")
+	root2 := []byte("don't care 2")
+	proof1 := [][]byte{}
+	proof2 := [][]byte{sha256EmptyTreeHash}
+
+	for _, p := range []subtreeConsistencyProbe{
+		{0, 0, 0, root1, root2, proof1, "sizes are equal (zero) but roots are not", true},
+		{0, 1, 1, root1, root2, proof1, "sizes are equal (one) but roots are not", true},
+		{0, 0, 1, root1, root2, proof1, "size1 is zero and does not equal size2", true},
+		// Sizes that are always consistent.
+		{0, 1, 1, root2, root2, proof1, "sizes are equal (one) and proof is empty", false},
+		// Empty subtree
+		{0, 0, 0, sha256EmptyTreeHash, sha256EmptyTreeHash, proof1, "subtree is empty sizes are equal (zero) subtree root valid proof is empty", false},
+		{0, 0, 0, sha256EmptyTreeHash, root1, proof1, "subtree is empty sizes are equal (zero) subtree root valid tree root random proof is empty", false},
+		{0, 0, 0, sha256EmptyTreeHash, sha256EmptyTreeHash, proof2, "subtree is empty sizes are equal (zero) roots valid but proof is not empty", true},
+		{0, 0, 0, root1, root1, proof1, "subtree is empty sizes are equal (zero) roots match but not valid", true},
+		{1, 1, 1, sha256EmptyTreeHash, sha256EmptyTreeHash, proof1, "subtree is empty sizes are equal (one) subtree root valid proof is empty", false},
+		{1, 1, 1, sha256EmptyTreeHash, root1, proof1, "subtree is empty sizes are equal (one) subtree root valid tree root random proof is empty", false},
+		{1, 1, 1, sha256EmptyTreeHash, sha256EmptyTreeHash, proof2, "subtree is empty sizes are equal (one) roots valid but proof is not empty", true},
+		{1, 1, 1, root1, root1, proof1, "subtree is empty sizes are equal (one) roots match but not valid", true},
+		{1, 1, 2, sha256EmptyTreeHash, sha256EmptyTreeHash, proof1, "subtree is empty subtree root valid proof is empty", false},
+		{1, 1, 2, sha256EmptyTreeHash, sha256EmptyTreeHash, proof1, "subtree is empty subtree root valid tree root random proof is empty", false},
+		{1, 1, 2, sha256EmptyTreeHash, sha256EmptyTreeHash, proof2, "subtree is empty roots valid but proof is not empty", true},
+		{1, 1, 2, root1, root1, proof1, "subtree is empty roots match but not valid", true},
+		// Invalid subtree boundaries (not a multiple of power of 2 >= end - start).
+		{1, 15, 15, root1, root2, proof1, "invalid subtree start 1 end 15 size 15", true},
+		{1, 3, 8, root1, root2, proof1, "invalid subtree start 1 end 3 size 8", true},
+		{2, 5, 8, root1, root2, proof1, "invalid subtree start 2 end 5 size 8", true},
+		{2, 6, 8, root1, root2, proof1, "invalid subtree start 2 end 6 size 8", true},
+		// Time travel to the past.
+		{0, 1, 0, root1, root2, proof1, "size1 is greater than size2", true},
+		{0, 2, 1, root1, root2, proof1, "size1 is greater than size2 again", true},
+		// Empty proof.
+		{0, 1, 2, root1, root2, proof1, "sizes do not match and proof is empty", true},
+		// Roots don't match.
+		{0, 1, 1, sha256EmptyTreeHash, root2, proof1, "roots do not match and sizes are one", true},
+		// Sizes match but the proof is not empty.
+		{0, 0, 0, sha256EmptyTreeHash, sha256EmptyTreeHash, proof2, "sizes match but proof is not empty and sizes are zero", true},
+		{0, 1, 1, sha256EmptyTreeHash, sha256EmptyTreeHash, proof2, "sizes match but proof is not empty and sizes are one", true},
+	} {
+		if err := writeSubtreeConsistencyProbe(dir, p); err != nil {
 			return err
 		}
 	}

proof/proof.go

@@ -113,6 +113,9 @@ func subtreeConsistency(start, end, size uint64) (Nodes, error) {
 	if start == 0 && end == size {
 		return Nodes{IDs: []compact.NodeID{}}, nil
 	}
+	if start == end {
+		return Nodes{IDs: []compact.NodeID{}}, nil
+	}
 
 	// If end == size, prove inclusion of [start, end) into the tree.
 	if end == size {
@@ -268,29 +271,25 @@ func reverse(ids []compact.NodeID) {
 	}
 }
 
-// Subtree represents a valid Merkle subtree [Start, End).
-type Subtree struct {
-	Start, End uint64
-}
-
-// FindSubtrees returns one or two subtrees that efficiently cover [start, end).
+// FindSubtrees returns three indices (start, mid, end) defining two adjacent
+// subtrees [start, mid) and [mid, end) that efficiently cover the input range.
 //
-// If the provided [start, end) range is already a valid subtree, then that subtree is returned directly.
-// Otherwise, this function continues by applying the "Selecting Two Subtrees" procedure
-// from Section 4.5.1 of draft-ietf-plants-merkle-tree-certs.
+// This function applies the "Selecting Two Subtrees" procedure from
+// Section 4.5.1 of draft-ietf-plants-merkle-tree-certs.
 //
 // Note that:
-//   - If the provided [start, end) range is already a valid subtree, then it is returned as the only entry in the slice.
-//   - The 2nd subtree, if present, is adjacent to the first, and may not be a perfect subtree.
-//   - The returned subtree(s) fully cover the [start, end) range.
-//   - There are no "extra" entries covered past end, but there may be covered entries prior to start.
-//   - The number of entries covered before start is always less than half the size of the first returned subtree.
-func FindSubtrees(start, end uint64) ([]Subtree, error) {
-	if start >= end {
-		return nil, fmt.Errorf("start %d must be strictly less than end %d", start, end)
+//   - If the input range has a size <= 1, then this function returns [start, end) and an empty subtree [end, end).
+//   - If the provided [start, end) range is already a valid subtree, then it is still split into two smaller subtrees.
+//   - The [mid, end) range, if not empty, is adjacent to the first, and may not be a perfect subtree.
+//   - The returned subtrees ranges fully cover the input range.
+//   - There are no "extra" entries covered past end, but there may be covered entries prior to start (i.e. returned start <= input start).
+//   - The number of entries covered before the input start is always less than half the size of the first returned subtree.
+func FindSubtrees(start, end uint64) (uint64, uint64, uint64, error) {
+	if start > end {
+		return 0, 0, 0, fmt.Errorf("start %d must be less than or equal to end %d", start, end)
 	}
-	if end-start == 1 || isSubtreeValid(start, end) == nil {
-		return []Subtree{{Start: start, End: end}}, nil
+	if end-start <= 1 {
+		return start, end, end, nil
 	}
 	last := end - 1
 	// Find where start and last's tree paths diverge.
@@ -301,10 +300,7 @@ func FindSubtrees(start, end uint64) ([]Subtree, error) {
 	// Maximize the left endpoint.
 	leftSplit := bits.Len64(^start & mask)
 	leftStart := start & ^((uint64(1) << leftSplit) - 1)
-	return []Subtree{
-		{Start: leftStart, End: mid},
-		{Start: mid, End: end},
-	}, nil
+	return leftStart, mid, end, nil
 }
 
 // isSubtreeValid returns whether a subtree covers a valid range.
@@ -313,12 +309,15 @@ func FindSubtrees(start, end uint64) ([]Subtree, error) {
 // - no extra node to the left of the subtree
 // - potentially extra nodes to the right of the subtree
 func isSubtreeValid(start, end uint64) error {
-	if start >= end {
-		return fmt.Errorf("start %d must be strictly less than end %d", start, end)
+	if start > end {
+		return fmt.Errorf("start %d must be less than or equal to end %d", start, end)
 	}
 	if start == 0 {
 		return nil
 	}
+	if start == end {
+		return nil
+	}
 
 	l := end - start
 

proof/proof_test.go

@@ -457,8 +457,6 @@ func TestSubtreeConsistency(t *testing.T) {
 		wantErr bool
 	}{
 		// Errors.
-		{start: 0, end: 0, size: 0, wantErr: true},                 // start = end = 0
-		{start: 1, end: 1, size: 1, wantErr: true},                 // start = end
 		{start: 2, end: 1, size: 0, wantErr: true},                 // start > end
 		{start: 0, end: 5, size: 0, wantErr: true},                 // end > size
 		{start: 0, end: 9, size: 8, wantErr: true},                 // end > size
@@ -467,6 +465,7 @@ func TestSubtreeConsistency(t *testing.T) {
 
 		// Small trees.
 		// start = 0
+		{start: 0, end: 0, size: 0, want: Nodes{IDs: []compact.NodeID{}}},             // start = end = 0
 		{start: 0, end: 1, size: 2, want: nodes(id(0, 1))},                            // b
 		{start: 0, end: 1, size: 4, want: nodes(id(0, 1), id(1, 1))},                  // b bb
 		{start: 0, end: 1, size: 6, want: rehash(2, 3, id(0, 1), id(1, 1), id(1, 2))}, // b bb cc
@@ -485,12 +484,14 @@ func TestSubtreeConsistency(t *testing.T) {
 		{start: 0, end: 7, size: 8, want: nodes(
 			id(0, 6), id(0, 7), id(1, 2), id(2, 0))}, // g h cc aaa
 		// start > 0
+		{start: 1, end: 1, size: 1, want: Nodes{IDs: []compact.NodeID{}}},                       // start = end
 		{start: 1, end: 2, size: 3, want: rehash(1, 2, id(0, 0), id(0, 2))},                     // a c
 		{start: 1, end: 2, size: 5, want: rehash(2, 3, id(0, 0), id(1, 1), id(0, 4))},           // a bb e
 		{start: 2, end: 4, size: 5, want: rehash(1, 2, id(1, 0), id(0, 4))},                     // aa e
 		{start: 1, end: 2, size: 7, want: rehash(2, 4, id(0, 0), id(1, 1), id(0, 6), id(1, 2))}, // a bb g cc
 		{start: 2, end: 4, size: 10, want: rehash(2, 3, id(1, 0), id(2, 1), id(1, 4))},          // aa bbb ee
 		{start: 4, end: 6, size: 10, want: rehash(2, 3, id(1, 3), id(2, 0), id(1, 4))},          // dd aaa ee
+		{start: 4, end: 4, size: 10, want: Nodes{IDs: []compact.NodeID{}}},                      // start = end
 		{start: 4, end: 7, size: 11, want: rehash(4, 6, // ccc=hash(ee,k)
 			id(0, 6), id(0, 7), id(1, 2), id(2, 0), id(0, 10), id(1, 4))}, // g h cc aaa k ee
 		{start: 4, end: 8, size: 11, want: rehash(1, 3, // ccc=hash(ee,k)
@@ -637,9 +638,9 @@ func TestConsistencySucceedsUpToTreeSize(t *testing.T) {
 
 func TestSubtreeConsistencySucceedsUpToTreeSize(t *testing.T) {
 	const maxSize = uint64(100)
-	for s := uint64(1); s <= maxSize; s++ {
-		for sbe := uint64(1); sbe <= s; sbe++ {
-			for sbs := range sbe {
+	for s := range maxSize + 1 {
+		for sbe := range s + 1 {
+			for sbs := range sbe + 1 {
 				if err := isSubtreeValid(sbs, sbe); err != nil {
 					continue
 				}
@@ -823,26 +824,26 @@ func inclusion(t *testing.T, index, size uint64) Nodes {
 
 func TestFindSubtrees(t *testing.T) {
 	for _, tc := range []struct {
-		start, end uint64
-		want       []Subtree
-		wantErr    bool
+		start, end                      uint64
+		wantStart, wantMid, wantEnd     uint64
+		wantErr                         bool
 	}{
-		// Already-valid subtrees are returned as-is.
 		// Single entry subtrees:
-		{start: 0, end: 1, want: []Subtree{{Start: 0, End: 1}}},
-		{start: 3, end: 4, want: []Subtree{{Start: 3, End: 4}}},
+		{start: 0, end: 1, wantStart: 0, wantMid: 1, wantEnd: 1},
+		{start: 3, end: 4, wantStart: 3, wantMid: 4, wantEnd: 4},
 		// Perfectly aligned subtrees:
-		{start: 4, end: 6, want: []Subtree{{Start: 4, End: 6}}},
-		{start: 16, end: 32, want: []Subtree{{Start: 16, End: 32}}},
+		{start: 4, end: 6, wantStart: 4, wantMid: 5, wantEnd: 6},
+		{start: 16, end: 32, wantStart: 16, wantMid: 24, wantEnd: 32},
 		// Non-perfect trees are split into two:
-		{start: 5, end: 13, want: []Subtree{{Start: 4, End: 8}, {Start: 8, End: 13}}},
-		{start: 7, end: 9, want: []Subtree{{Start: 7, End: 8}, {Start: 8, End: 9}}},
+		{start: 5, end: 13, wantStart: 4, wantMid: 8, wantEnd: 13},
+		{start: 7, end: 9, wantStart: 7, wantMid: 8, wantEnd: 9},
+		// Empty subtrees.
+		{start: 5, end: 5, wantStart: 5, wantMid: 5, wantEnd: 5},
 		// Invalid inputs:
-		{start: 5, end: 5, wantErr: true},
 		{start: 6, end: 5, wantErr: true},
 	} {
 		t.Run(fmt.Sprintf("%d:%d", tc.start, tc.end), func(t *testing.T) {
-			got, err := FindSubtrees(tc.start, tc.end)
+			gotStart, gotMid, gotEnd, err := FindSubtrees(tc.start, tc.end)
 			if tc.wantErr {
 				if err == nil {
 					t.Fatal("expected error, got nil")
@@ -852,8 +853,8 @@ func TestFindSubtrees(t *testing.T) {
 			if err != nil {
 				t.Fatalf("FindSubtrees: %v", err)
 			}
-			if diff := cmp.Diff(tc.want, got); diff != "" {
-				t.Errorf("FindSubtrees mismatch (-want +got):\n%s", diff)
+			if gotStart != tc.wantStart || gotMid != tc.wantMid || gotEnd != tc.wantEnd {
+				t.Errorf("FindSubtrees(%d, %d) = (%d, %d, %d), want (%d, %d, %d)", tc.start, tc.end, gotStart, gotMid, gotEnd, tc.wantStart, tc.wantMid, tc.wantEnd)
 			}
 		})
 	}

proof/verify.go

@@ -107,10 +107,22 @@ func VerifyConsistency(hasher merkle.LogHasher, size1, size2 uint64, proof [][]b
 // VerifySubtreeConsistency checks that the passed-in subtree consistency proof
 // is valid between the passed in subtree indices and parent tree size, with
 // respect to the corresponding subtree root node hash. It Requires:
-//   - 0 <= start < end <= size.
+//   - 0 <= start <= end <= size.
 //   - start to be a multiple of the smallest power of two greater than or equal to
 //     (end - start)
 func VerifySubtreeConsistency(hasher merkle.LogHasher, start, end, size uint64, proof [][]byte, subRoot, parentRoot []byte) error {
+	if start == end {
+		if end > size {
+			return fmt.Errorf("size (%d) < end (%d)", size, end)
+		}
+		if len(proof) > 0 {
+			return errors.New("start=end, but proof is not empty")
+		}
+		if !bytes.Equal(subRoot, hasher.EmptyRoot()) {
+			return errors.New("start=end, but subRoot is not empty root")
+		}
+		return nil
+	}
 	hash2, err := RootFromSubtreeConsistencyProof(hasher, start, end, size, proof, subRoot)
 	if err != nil {
 		return err
@@ -144,7 +156,7 @@ func RootFromConsistencyProof(hasher merkle.LogHasher, size1, size2 uint64, proo
 // a consistency proof.
 //
 // It requires:
-//   - 0 <= start < end <= size.
+//   - 0 <= start <= end <= size.
 //   - start to be a multiple of the smallest power of two greater than or equal to
 //     (end - start)
 //
@@ -156,6 +168,16 @@ func RootFromSubtreeConsistencyProof(hasher merkle.LogHasher, start, end, size u
 		return nil, fmt.Errorf("subtree invalid: %v", err)
 	case size < end:
 		return nil, fmt.Errorf("size (%d) < end (%d)", size, end)
+	case start == end && size == 0:
+		if len(proof) > 0 {
+			return nil, errors.New("start=end, but proof is not empty")
+		}
+		if !bytes.Equal(subRoot, hasher.EmptyRoot()) {
+			return nil, errors.New("start=end, but subRoot is not empty root")
+		}
+		return hasher.EmptyRoot(), nil
+	case start == end:
+		return nil, errors.New("cannot reconstruct non-empty tree root from empty subtree")
 	case start == 0 && size == end:
 		if len(proof) > 0 {
 			return nil, errors.New("start=0 and end=size, but proof is not empty")

testdata/subtreeconsistency/additional/invalid-subtree-start-1-end-15-size-15.json

@@ -0,0 +1,10 @@
+{
+  "start": 1,
+  "end": 15,
+  "size": 15,
+  "root1": "ZG9uJ3QgY2FyZSAx",
+  "root2": "ZG9uJ3QgY2FyZSAy",
+  "proof": [],
+  "desc": "invalid subtree start 1 end 15 size 15",
+  "wantErr": true
+}

testdata/subtreeconsistency/additional/invalid-subtree-start-1-end-3-size-8.json

@@ -0,0 +1,10 @@
+{
+  "start": 1,
+  "end": 3,
+  "size": 8,
+  "root1": "ZG9uJ3QgY2FyZSAx",
+  "root2": "ZG9uJ3QgY2FyZSAy",
+  "proof": [],
+  "desc": "invalid subtree start 1 end 3 size 8",
+  "wantErr": true
+}

testdata/subtreeconsistency/additional/invalid-subtree-start-2-end-5-size-8.json

@@ -0,0 +1,10 @@
+{
+  "start": 2,
+  "end": 5,
+  "size": 8,
+  "root1": "ZG9uJ3QgY2FyZSAx",
+  "root2": "ZG9uJ3QgY2FyZSAy",
+  "proof": [],
+  "desc": "invalid subtree start 2 end 5 size 8",
+  "wantErr": true
+}