Verify subtree inclusion proofs (transparency-dev#227)

* verifysubtreeinclusion implementation

* verifysubtreeinclusion test copy + implementation

* size --> start, end in tests

* duplicate tests to -subtree.json

* shift and add description

* fix large numbers

* add error tests

* fix error messages

* remove newlines

* reviewer's comments

* oops

Author: phbnf

proof/proof.go

@@ -65,8 +65,8 @@ func SubtreeInclusion(index, start, end uint64) (Nodes, error) {
 	if index < start || index >= end {
 		return Nodes{}, fmt.Errorf("index %d out of bounds for subtree [%d, %d)", index, start, end)
 	}
-	if !isSubtreeValid(start, end) {
-		return Nodes{}, fmt.Errorf("start %d not a multiple of bit_ceil(end - start) = %d", start, end-start)
+	if err := isSubtreeValid(start, end); err != nil {
+		return Nodes{}, fmt.Errorf("subtree invalid: %v", err)
 	}
 
 	// Shift the subtree to the left, such that it starts at 0.
@@ -228,15 +228,18 @@ func reverse(ids []compact.NodeID) {
 // - all the subtree nodes
 // - no extra node to the left of the subtree
 // - potentially extra nodes to the right of the subtree
-func isSubtreeValid(start, end uint64) bool {
+func isSubtreeValid(start, end uint64) error {
 	l := end - start
 	if start == 0 {
-		return true
-	} else if (l) > uint64(1)<<63 {
+		return nil
+	} else if l > uint64(1)<<63 {
 		// special-case large subtree to avoid panic
-		return false
+		return fmt.Errorf("start %d must be 0 when subtree length %d > 1<<63. ", start, l)
 	}
-	return start%bitCeil(l) == 0
+	if bc := bitCeil(l); start%bc != 0 {
+		return fmt.Errorf("start %d not a multiple of bit_ceil(end - start) = %d", start, bc)
+	}
+	return nil
 }
 
 // bitCeil returns the smallest power of 2 larger than n.

proof/proof_test.go

@@ -425,7 +425,7 @@ func TestInclusionSubtreeSucceedsUpToTreeSize(t *testing.T) {
 	const maxSize = uint64(555)
 	for sbe := uint64(1); sbe <= maxSize; sbe++ {
 		for sbs := uint64(0); sbs < sbe; sbs++ {
-			if !isSubtreeValid(sbs, sbe) {
+			if err := isSubtreeValid(sbs, sbe); err != nil {
 				continue
 			}
 			for i := sbs; i < sbe; i++ {

proof/verify.go

@@ -51,6 +51,30 @@ func VerifyInclusion(hasher merkle.LogHasher, index, size uint64, leafHash []byt
 	return verifyMatch(calcRoot, root)
 }
 
+// VerifySubtreeInclusion verifies the correctness of the subtree inclusion
+// proof for the leaf with the specified hash and index, relative to the
+// provided subtree [start, end) subtree and subtree root hash.
+// It requires:
+//   - 0 <= start <= index < end
+//   - start to be a multiple of the smallest power of two greater than or equal to
+//     (end - start)
+func VerifySubtreeInclusion(hasher merkle.LogHasher, index, start, end uint64, leafHash []byte, proof [][]byte, root []byte) error {
+	if start >= end {
+		return fmt.Errorf("start %d greater than or equal to end %d", start, end)
+	}
+	if index < start || index >= end {
+		return fmt.Errorf("index %d out of bounds for subtree [%d, %d)", index, start, end)
+	}
+	if err := isSubtreeValid(start, end); err != nil {
+		return fmt.Errorf("subtree invalid: %v", err)
+	}
+	calcRoot, err := RootFromInclusionProof(hasher, index-start, end-start, leafHash, proof)
+	if err != nil {
+		return err
+	}
+	return verifyMatch(calcRoot, root)
+}
+
 // RootFromInclusionProof calculates the expected root hash for a tree of the
 // given size, provided a leaf index and hash with the corresponding inclusion
 // proof. Requires 0 <= index < size.

proof/verify_test.go

@@ -38,6 +38,20 @@ type inclusionProbe struct {
 	WantError bool   `json:"wantErr"`
 }
 
+// subtreeInclusionProbe is a parameter set for subtree inclusion proof
+// verification.
+type subtreeInclusionProbe struct {
+	LeafIdx  uint64   `json:"leafIdx"`
+	Start    uint64   `json:"start"`
+	End      uint64   `json:"end"`
+	Root     []byte   `json:"root"`
+	LeafHash []byte   `json:"leafHash"`
+	Proof    [][]byte `json:"proof"`
+
+	Desc      string `json:"desc"`
+	WantError bool   `json:"wantErr"`
+}
+
 // consistencyProbe is a parameter set for consistency proof verification.
 type consistencyProbe struct {
 	Size1 uint64   `json:"size1"`
@@ -102,6 +116,58 @@ func TestVerifyInclusionProbes(t *testing.T) {
 	}
 }
 
+func TestVerifySubtreeInclusionProbes(t *testing.T) {
+	var probes []subtreeInclusionProbe
+
+	if err := filepath.WalkDir("../testdata/subtreeinclusion", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		if filepath.Ext(d.Name()) != ".json" {
+			return nil
+		}
+
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+
+		var probe subtreeInclusionProbe
+		if err := json.Unmarshal(data, &probe); err != nil {
+			return fmt.Errorf("failed to parse subtree inclusion probe json: %s", err)
+		}
+
+		probes = append(probes, probe)
+
+		return nil
+	}); err != nil {
+		t.Errorf("failed to read subtree inclusion probes: %s", err)
+	}
+
+	var wrong []string
+	for _, p := range probes {
+		err := VerifySubtreeInclusion(rfc6962.DefaultHasher, p.LeafIdx, p.Start, p.End, p.LeafHash, p.Proof, p.Root)
+		if p.WantError && err == nil {
+			wrong = append(wrong, fmt.Sprintf("expected error but didn't get one: %s", p.Desc))
+			continue
+		}
+
+		if !p.WantError && err != nil {
+			wrong = append(wrong, fmt.Sprintf("unexpected error: %s, %s", p.Desc, err))
+			continue
+		}
+	}
+
+	if len(wrong) > 0 {
+		t.Errorf("errors verifying subtree inclusion probes: \n%d out of %d failures \nError messages: \n%s", len(wrong), len(probes), strings.Join(wrong, "\n"))
+	}
+}
+
 func TestVerifyConsistencyProbes(t *testing.T) {
 	var probes []consistencyProbe
 

testdata/subtreeinclusion/0/empty-root-subtree.json

@@ -0,0 +1,10 @@
+{
+  "leafIdx": 1,
+  "start": 1,
+  "end": 2,
+  "root": "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
+  "leafHash": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "proof": null,
+  "desc": "empty root - subtree",
+  "wantErr": true
+}

testdata/subtreeinclusion/0/empty-root.json

@@ -0,0 +1,10 @@
+{
+  "leafIdx": 0,
+  "start": 0,
+  "end": 1,
+  "root": "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
+  "leafHash": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "proof": null,
+  "desc": "empty root",
+  "wantErr": true
+}

testdata/subtreeinclusion/0/happy-path-subtree.json

@@ -0,0 +1,10 @@
+{
+  "leafIdx": 1,
+  "start": 1,
+  "end": 2,
+  "root": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "leafHash": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "proof": null,
+  "desc": "happy path - subtree",
+  "wantErr": false
+}

testdata/subtreeinclusion/0/happy-path.json

@@ -0,0 +1,10 @@
+{
+  "leafIdx": 0,
+  "start": 0,
+  "end": 1,
+  "root": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "leafHash": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "proof": null,
+  "desc": "happy path",
+  "wantErr": false
+}

testdata/subtreeinclusion/0/leafIdx-XOR-@2-subtree.json

@@ -0,0 +1,10 @@
+{
+  "leafIdx": 3,
+  "start": 1,
+  "end": 2,
+  "root": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "leafHash": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "proof": null,
+  "desc": "leafIdx XOR @2 - subtree",
+  "wantErr": true
+}

testdata/subtreeinclusion/0/leafIdx-XOR-@2.json

@@ -0,0 +1,10 @@
+{
+  "leafIdx": 2,
+  "start": 0,
+  "end": 1,
+  "root": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "leafHash": "bjQLnP+zepicpUTmu3gKLHiQHT+zNzh2hRGjBhevoB0=",
+  "proof": null,
+  "desc": "leafIdx XOR @2",
+  "wantErr": true
+}