Add appendix with accumulated subtree test vectors

Author: FiloSottile

demo/log.go

@@ -128,6 +128,70 @@ func (mt *MerkleTree) SubtreeInclusionProof(index, start, end int) ([]byte, erro
 	return proof, nil
 }
 
+func (mt *MerkleTree) SubtreeConsistencyProof(start, end, n int) ([]byte, error) {
+	if !IsValidSubtree(start, end) {
+		return nil, fmt.Errorf("invalid subtree: [%d, %d)", start, end)
+	}
+	if end > n {
+		return nil, fmt.Errorf("subtree [%d, %d) contains more elements than tree of size %d", start, end, n)
+	}
+	if n > mt.Size() {
+		return nil, fmt.Errorf("tree of size %d is larger than the Merkle Tree of size %d", n, mt.Size())
+	}
+	return mt.subtreeSubproof(start, end, 0, n, true)
+}
+
+// subtreeSubproof implements SUBTREE_SUBPROOF(start - lo, end - lo, D[lo:hi],
+// known) over the tree's entries, with the subtree and window described in
+// absolute indices. known reports whether the subtree hash is already known to
+// the verifier and so may be omitted from the proof.
+func (mt *MerkleTree) subtreeSubproof(start, end, lo, hi int, known bool) ([]byte, error) {
+	if start == lo && end == hi {
+		// The subtree is the whole window.
+		if known {
+			return nil, nil
+		}
+		h, err := mt.SubtreeHash(lo, hi)
+		if err != nil {
+			return nil, err
+		}
+		return h[:], nil
+	}
+	// The window has more than one element, so split it at the largest power
+	// of two smaller than its size.
+	k := 1 << (bits.Len(uint(hi-lo-1)) - 1)
+	split := lo + k
+	var proof []byte
+	var siblingStart, siblingEnd int
+	var err error
+	switch {
+	case end <= split:
+		// The subtree is entirely in the left child, so recurse into it and
+		// include the right child.
+		proof, err = mt.subtreeSubproof(start, end, lo, split, known)
+		siblingStart, siblingEnd = split, hi
+	case split <= start:
+		// The subtree is entirely in the right child, so recurse into it and
+		// include the left child.
+		proof, err = mt.subtreeSubproof(start, end, split, hi, known)
+		siblingStart, siblingEnd = lo, split
+	default:
+		// The subtree spans the split, which implies start == lo. Recurse into
+		// the right child, no longer knowing its subtree hash, and include the
+		// left child.
+		proof, err = mt.subtreeSubproof(split, end, split, hi, false)
+		siblingStart, siblingEnd = lo, split
+	}
+	if err != nil {
+		return nil, err
+	}
+	h, err := mt.SubtreeHash(siblingStart, siblingEnd)
+	if err != nil {
+		return nil, err
+	}
+	return append(proof, h[:]...), nil
+}
+
 func SubtreesForInterval(start, end int) (start1, end1, start2, end2 int, err error) {
 	if 0 > start || start >= end {
 		err = fmt.Errorf("invalid interval [%d, %d)", start, end)

demo/vectors_test.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"testing"
+)
+
+// These tests reproduce the accumulated test vectors from the "Subtree Test
+// Vectors" appendix. For trees of sizes up to 130, they fold the output of each
+// subtree algorithm over every valid input into a single rolling SHA-256, which
+// is compared against the value published in the draft.
+
+const subtreeVectorMax = 130
+
+// subtreeVectorTree builds the tree D used by the test vectors, with leaf values
+// d[0] = 0x00, d[1] = 0x01, and so on.
+func subtreeVectorTree() *MerkleTree {
+	entries := make([][]byte, subtreeVectorMax)
+	for i := range entries {
+		entries[i] = []byte{byte(i)}
+	}
+	return NewMerkleTree(entries)
+}
+
+// writeProofLine writes prefix followed by, for each hash in the concatenated
+// proof, a space and the hash's hexadecimal encoding, then a newline. An empty
+// proof contributes no hashes and so leaves no trailing space.
+func writeProofLine(w io.Writer, prefix string, proof []byte) {
+	io.WriteString(w, prefix)
+	for off := 0; off < len(proof); off += HashSize {
+		fmt.Fprintf(w, " %x", proof[off:off+HashSize])
+	}
+	io.WriteString(w, "\n")
+}
+
+func TestSubtreeHashVectors(t *testing.T) {
+	tree := subtreeVectorTree()
+	h := sha256.New()
+	for end := 1; end <= subtreeVectorMax; end++ {
+		for start := 0; start < end; start++ {
+			if !IsValidSubtree(start, end) {
+				continue
+			}
+			subtreeHash, err := tree.SubtreeHash(start, end)
+			if err != nil {
+				t.Fatalf("SubtreeHash(%d, %d): %v", start, end, err)
+			}
+			fmt.Fprintf(h, "[%d, %d) %x\n", start, end, subtreeHash[:])
+		}
+	}
+	const want = "94a95384a8c69acea9b50d035a58285b3a777cb7a724005faa5e1f1e1190007f"
+	if got := fmt.Sprintf("%x", h.Sum(nil)); got != want {
+		t.Errorf("subtree hash vector = %s, want %s", got, want)
+	}
+}
+
+func TestSubtreeInclusionProofVectors(t *testing.T) {
+	tree := subtreeVectorTree()
+	h := sha256.New()
+	for end := 1; end <= subtreeVectorMax; end++ {
+		for start := 0; start < end; start++ {
+			if !IsValidSubtree(start, end) {
+				continue
+			}
+			for index := start; index < end; index++ {
+				proof, err := tree.SubtreeInclusionProof(index, start, end)
+				if err != nil {
+					t.Fatalf("SubtreeInclusionProof(%d, %d, %d): %v", index, start, end, err)
+				}
+				writeProofLine(h, fmt.Sprintf("%d [%d, %d)", index, start, end), proof)
+			}
+		}
+	}
+	const want = "ac2a8f989e44d99e399db448050ff5f19757df53cfb716aa81015d3955d8163f"
+	if got := fmt.Sprintf("%x", h.Sum(nil)); got != want {
+		t.Errorf("subtree inclusion proof vector = %s, want %s", got, want)
+	}
+}
+
+func TestSubtreeConsistencyProofVectors(t *testing.T) {
+	tree := subtreeVectorTree()
+	h := sha256.New()
+	for n := 0; n <= subtreeVectorMax; n++ {
+		for end := 1; end <= n; end++ {
+			for start := 0; start < end; start++ {
+				if !IsValidSubtree(start, end) {
+					continue
+				}
+				proof, err := tree.SubtreeConsistencyProof(start, end, n)
+				if err != nil {
+					t.Fatalf("SubtreeConsistencyProof(%d, %d, %d): %v", start, end, n, err)
+				}
+				writeProofLine(h, fmt.Sprintf("[%d, %d) %d", start, end, n), proof)
+			}
+		}
+	}
+	const want = "c586ebbb73a5621baf2140095d87dde934e3b6503a562a1a5215b8209edd083d"
+	if got := fmt.Sprintf("%x", h.Sum(nil)); got != want {
+		t.Errorf("subtree consistency proof vector = %s, want %s", got, want)
+	}
+}
+
+func TestEfficientCoveringSubtreeVectors(t *testing.T) {
+	h := sha256.New()
+	for end := 1; end <= subtreeVectorMax; end++ {
+		for start := 0; start < end; start++ {
+			if IsValidSubtree(start, end) {
+				fmt.Fprintf(h, "[%d, %d)\n", start, end)
+				continue
+			}
+			start1, end1, start2, end2, err := SubtreesForInterval(start, end)
+			if err != nil {
+				t.Fatalf("SubtreesForInterval(%d, %d): %v", start, end, err)
+			}
+			fmt.Fprintf(h, "[%d, %d) [%d, %d)\n", start1, end1, start2, end2)
+		}
+	}
+	const want = "e0aecb912a10c57d753b6ecc64db73217f9bc4ed10fcb4e9062be3b6fbe1ebfd"
+	if got := fmt.Sprintf("%x", h.Sum(nil)); got != want {
+		t.Errorf("efficient covering subtree vector = %s, want %s", got, want)
+	}
+}

draft-ietf-plants-merkle-tree-certs.md

@@ -184,6 +184,13 @@ informative:
     author:
       org: C2SP
 
+  Accumulated:
+    title: Accumulated Test Vectors
+    target: https://words.filippo.io/accumulated/
+    date: October 9, 2024
+    author:
+    - name: Filippo Valsorda
+
 ...
 
 --- abstract
@@ -421,6 +428,8 @@ As with Merkle Trees, a subtree inclusion proof, defined in {{subtree-inclusion-
 
 Not all intervals can form subtrees. Subtrees are limited to intervals that can be efficiently proven consistent with the original tree, using subtree consistency proofs defined in {{subtree-consistency-proofs}}. However, every interval of a Merkle Tree can be efficiently covered by two subtrees. {{arbitrary-intervals}} describes how to determine these subtrees.
 
+{{subtree-test-vectors}} provides test vectors for the algorithms defined in this section.
+
 ## Definition of a Subtree
 
 Given an ordered list of `n` inputs, `D_n = {d[0], d[1], ..., d[n-1]}`, {{Section 2.1.1 of !RFC9162}} defines the Merkle Tree via the Merkle Tree Hash `MTH(D_n)`.
@@ -2251,6 +2260,119 @@ This reconstructs the hashes of the subtree and original tree, which are then co
 
 In the case when `fn` is `sn` in step 5, the condition in step 7.2.1 is always false, and `fr` is always equal to `node_hash` in step 8. In this case, steps 6 through 8 are equivalent to verifying an inclusion proof for the truncated subtree `[fn, sn + 1)` and truncated tree `tn + 1`.
 
+# Subtree Test Vectors
+
+The following are "accumulated" {{Accumulated}} test vectors for the various subtree algorithms defined in {{subtrees}}.
+
+They are hash values of the outputs of all possible inputs for each algorithm, for trees of sizes up to 130. They can be used to verify that an implementation matches the specification, without having to include a large number of individual test vectors.
+
+For all the test vectors, a tree `D_n` of size `n` is constructed with leaf values `d[0] = 0x00, d[1] = 0x01, ...`. The hash function used is SHA-256. The hash values are encoded in hexadecimal.
+
+## Subtree Hashes
+
+For each value of `end` from 1 to 130, and each value of `start` from 0 to `end - 1`, if `[start, end)` is a valid subtree, add to the rolling hash the ASCII string `[START, END) HASH` followed by a newline (U+000A), where `START` and `END` are the decimal representations of `start` and `end`, respectively, and `HASH` is the hexadecimal encoding of `MTH(D[start:end])`, according to {{subtrees}}.
+
+The final hash value is
+
+~~~
+94a95384a8c69acea9b50d035a58285b3a777cb7a724005faa5e1f1e1190007f
+~~~
+
+In Python, this can be expressed as:
+
+~~~python
+import hashlib
+h = hashlib.sha256()
+for end in range(1, 131):
+    for start in range(end):
+        if valid_subtree(start, end):
+            subtree_hash = MTH(D[start:end])
+            h.update(f'[{start}, {end}) {subtree_hash.hex()}\n'.encode())
+assert h.hexdigest() == '94a95384a8c69acea9b50d035a58285b3a777cb7a724005faa5e1f1e1190007f'
+~~~
+
+## Subtree Inclusion Proofs {#subtree-inclusion-proof-vectors}
+
+For each value of `end` from 1 to 130, and each value of `start` from 0 to `end - 1`, if `[start, end)` is a valid subtree, for each value of `index` from `start` to `end - 1`, add to the rolling hash the ASCII string `INDEX [START, END)`, then, for each hash in the inclusion proof ({{subtree-inclusion-proofs}}) for `d[index]` in the subtree `[start, end)`, a space (U+0020) followed by the hexadecimal encoding of that hash, and finally a newline (U+000A), where `INDEX` is the decimal representation of `index`, and `START` and `END` are the decimal representations of `start` and `end`, respectively.
+
+The final hash value is
+
+~~~
+ac2a8f989e44d99e399db448050ff5f19757df53cfb716aa81015d3955d8163f
+~~~
+
+In Python, this can be expressed as:
+
+~~~python
+import hashlib
+h = hashlib.sha256()
+for end in range(1, 131):
+    for start in range(end):
+        if valid_subtree(start, end):
+            for index in range(start, end):
+                inclusion_proof = get_inclusion_proof(D, start, end, index)
+                line = f'{index} [{start}, {end})'
+                for p in inclusion_proof:
+                    line += f' {p.hex()}'
+                h.update(f'{line}\n'.encode())
+assert h.hexdigest() == 'ac2a8f989e44d99e399db448050ff5f19757df53cfb716aa81015d3955d8163f'
+~~~
+
+## Subtree Consistency Proofs {#subtree-consistency-proof-vectors}
+
+For each value of `n` from 0 to 130, and each value of `end` from 1 to `n`, and each value of `start` from 0 to `end - 1`, if `[start, end)` is a valid subtree, add to the rolling hash the ASCII string `[START, END) N`, then, for each hash in the consistency proof ({{subtree-consistency-proofs}}) for the subtree `[start, end)` and tree of size `n`, a space (U+0020) followed by the hexadecimal encoding of that hash, and finally a newline (U+000A), where `START` and `END` are the decimal representations of `start` and `end`, respectively, and `N` is the decimal representation of `n`.
+
+The final hash value is
+
+~~~
+c586ebbb73a5621baf2140095d87dde934e3b6503a562a1a5215b8209edd083d
+~~~
+
+In Python, this can be expressed as:
+
+~~~python
+import hashlib
+h = hashlib.sha256()
+for n in range(131):
+    for end in range(1, n + 1):
+        for start in range(end):
+            if valid_subtree(start, end):
+                consistency_proof = get_consistency_proof(D, n, start, end)
+                line = f'[{start}, {end}) {n}'
+                for p in consistency_proof:
+                    line += f' {p.hex()}'
+                h.update(f'{line}\n'.encode())
+assert h.hexdigest() == 'c586ebbb73a5621baf2140095d87dde934e3b6503a562a1a5215b8209edd083d'
+~~~
+
+## Efficient Covering Subtrees
+
+For each value of `end` from 1 to 130, and each value of `start` from 0 to `end - 1`:
+
+* if `[start, end)` is a valid subtree, add to the rolling hash the ASCII string `[START, END)` followed by a newline (U+000A), where `START` and `END` are the decimal representations of `start` and `end`, respectively;
+* otherwise, add to the rolling hash the ASCII string `[LEFT_START, LEFT_END) [RIGHT_START, RIGHT_END)` followed by a newline (U+000A), where `LEFT_START`, `LEFT_END`, `RIGHT_START`, and `RIGHT_END` are the decimal representations of the start and end of the left and right subtrees, respectively, that efficiently cover ({{arbitrary-intervals}}) `[start, end)`.
+
+The final hash value is
+
+~~~
+e0aecb912a10c57d753b6ecc64db73217f9bc4ed10fcb4e9062be3b6fbe1ebfd
+~~~
+
+In Python, this can be expressed as:
+
+~~~python
+import hashlib
+h = hashlib.sha256()
+for end in range(1, 131):
+    for start in range(end):
+        if valid_subtree(start, end):
+            h.update(f'[{start}, {end})\n'.encode())
+        else:
+            left_start, left_end, right_start, right_end = get_covering_subtrees(start, end)
+            h.update(f'[{left_start}, {left_end}) [{right_start}, {right_end})\n'.encode())
+assert h.hexdigest() == 'e0aecb912a10c57d753b6ecc64db73217f9bc4ed10fcb4e9062be3b6fbe1ebfd'
+~~~
+
 # Acknowledgements
 {:numbered="false"}
 
@@ -2445,3 +2567,5 @@ In draft-04, there is no fast issuance mode. In draft-05, frequent, non-landmark
 - Editorial fixes
 
 - Discuss the implications of subordinate CAs in Security Considerations
+
+- Added subtree test vector appendix