feat(rlp): add RawList for working with un-decoded lists ethereum#33755 (#2040)

This adds a new type wrapper that decodes as a list, but does not
actually decode the contents of the list. The type parameter exists as a
marker, and enables decoding the elements lazily. RawList can also be
used for building a list incrementally.

Co-authored-by: Felix Lange <fjl@twurst.com>

Author: gzliudan

rlp/encode.go

@@ -102,6 +102,29 @@ func EncodeToReader(val interface{}) (size int, r io.Reader, err error) {
 	return buf.size(), &encReader{buf: buf}, nil
 }
 
+// EncodeToRawList encodes val as an RLP list and returns it as a RawList.
+func EncodeToRawList[T any](val []T) (RawList[T], error) {
+	if len(val) == 0 {
+		return RawList[T]{}, nil
+	}
+
+	// Encode the value to an internal buffer.
+	buf := getEncBuffer()
+	defer encBufferPool.Put(buf)
+	if err := buf.encode(val); err != nil {
+		return RawList[T]{}, err
+	}
+
+	// Create the RawList. RawList assumes the initial list header is padded
+	// 9 bytes, so we have to determine the offset where the value should be
+	// placed.
+	contentSize := buf.lheads[0].size
+	bytes := make([]byte, contentSize+9)
+	offset := 9 - headsize(uint64(contentSize))
+	buf.copyTo(bytes[offset:])
+	return RawList[T]{enc: bytes}, nil
+}
+
 type listhead struct {
 	offset int // index of this header in string data
 	size   int // total size of encoded data (including list headers)

rlp/iterator.go

@@ -16,31 +16,35 @@
 
 package rlp
 
-type listIterator struct {
-	data []byte
-	next []byte
-	err  error
+// Iterator is an iterator over the elements of an encoded container.
+type Iterator struct {
+	data   []byte
+	next   []byte
+	offset int
+	err    error
 }
 
-// NewListIterator creates an iterator for the (list) represented by data
-func NewListIterator(data RawValue) (*listIterator, error) {
+// NewListIterator creates an iterator for the (list) represented by data.
+func NewListIterator(data RawValue) (*Iterator, error) {
 	k, t, c, err := readKind(data)
 	if err != nil {
 		return nil, err
 	}
 	if k != List {
 		return nil, ErrExpectedList
 	}
-	it := &listIterator{
-		data: data[t : t+c],
-	}
+	it := &Iterator{data: data[t : t+c], offset: int(t)}
 	return it, nil
 }
 
+func newIterator(data []byte) *Iterator {
+	return &Iterator{data: data}
+}
+
 // Next forwards the iterator one step.
 // Returns true if there is a next item or an error occurred on this step (check Err()).
 // On parse error, the iterator is marked finished and subsequent calls return false.
-func (it *listIterator) Next() bool {
+func (it *Iterator) Next() bool {
 	if len(it.data) == 0 {
 		return false
 	}
@@ -52,17 +56,34 @@ func (it *listIterator) Next() bool {
 		it.data = nil
 		return true
 	}
-	it.next = it.data[:t+c]
-	it.data = it.data[t+c:]
+	length := t + c
+	it.next = it.data[:length]
+	it.data = it.data[length:]
+	it.offset += int(length)
 	it.err = nil
 	return true
 }
 
-// Value returns the current value
-func (it *listIterator) Value() []byte {
+// Count returns the remaining number of items.
+// Note this is O(n) and the result may be incorrect if the list data is invalid.
+// The returned count is always an upper bound on the remaining items
+// that will be visited by the iterator.
+func (it *Iterator) Count() int {
+	count, _ := CountValues(it.data)
+	return count
+}
+
+// Value returns the current value.
+func (it *Iterator) Value() []byte {
 	return it.next
 }
 
-func (it *listIterator) Err() error {
+// Offset returns the offset of the current value into the list data.
+func (it *Iterator) Offset() int {
+	return it.offset - len(it.next)
+}
+
+// Err returns the error that caused Next to return false, if any.
+func (it *Iterator) Err() error {
 	return it.err
 }

rlp/iterator_test.go

@@ -38,10 +38,18 @@ func TestIterator(t *testing.T) {
 		t.Fatal("expected two elems, got zero")
 	}
 	txs := it.Value()
+	if offset := it.Offset(); offset != 3 {
+		t.Fatal("wrong offset", offset, "want 3")
+	}
+
 	// Check that uncles exist
 	if !it.Next() {
 		t.Fatal("expected two elems, got one")
 	}
+	if offset := it.Offset(); offset != 219 {
+		t.Fatal("wrong offset", offset, "want 219")
+	}
+
 	txit, err := NewListIterator(txs)
 	if err != nil {
 		t.Fatal(err)

rlp/raw.go

@@ -17,8 +17,10 @@
 package rlp
 
 import (
+	"fmt"
 	"io"
 	"reflect"
+	"slices"
 )
 
 // RawValue represents an encoded RLP value and can be used to delay
@@ -28,6 +30,121 @@ type RawValue []byte
 
 var rawValueType = reflect.TypeOf(RawValue{})
 
+// RawList represents an encoded RLP list.
+type RawList[T any] struct {
+	// The list is stored in encoded form.
+	// Note this buffer has some special properties:
+	//
+	//   - if the buffer is nil, it's the zero value, representing
+	//     an empty list.
+	//   - if the buffer is non-nil, it must have a length of at least
+	//     9 bytes, which is reserved padding for the encoded list header.
+	//     The remaining bytes, enc[9:], store the content bytes of the list.
+	//
+	// The implementation code mostly works with the Content method because it
+	// returns something valid either way.
+	enc []byte
+}
+
+// Content returns the RLP-encoded data of the list.
+// This does not include the list-header.
+// The return value is a direct reference to the internal buffer, not a copy.
+func (r *RawList[T]) Content() []byte {
+	if r.enc == nil {
+		return nil
+	} else {
+		return r.enc[9:]
+	}
+}
+
+// EncodeRLP writes the encoded list to the writer.
+func (r RawList[T]) EncodeRLP(w io.Writer) error {
+	_, err := w.Write(r.Bytes())
+	return err
+}
+
+// Bytes returns the RLP encoding of the list.
+// Note the return value aliases the internal buffer.
+func (r *RawList[T]) Bytes() []byte {
+	if r == nil || r.enc == nil {
+		return []byte{0xC0} // zero value encodes as empty list
+	}
+	n := puthead(r.enc, 0xC0, 0xF7, uint64(len(r.Content())))
+	copy(r.enc[9-n:], r.enc[:n])
+	return r.enc[9-n:]
+}
+
+// DecodeRLP decodes the list. This does not perform validation of the items!
+func (r *RawList[T]) DecodeRLP(s *Stream) error {
+	k, size, err := s.Kind()
+	if err != nil {
+		return err
+	}
+	if k != List {
+		return fmt.Errorf("%w for %T", ErrExpectedList, r)
+	}
+	enc := make([]byte, 9+size)
+	if err := s.readFull(enc[9:]); err != nil {
+		return err
+	}
+	*r = RawList[T]{enc: enc}
+	return nil
+}
+
+// Items decodes and returns all items in the list.
+func (r *RawList[T]) Items() ([]T, error) {
+	items := make([]T, r.Len())
+	it := r.ContentIterator()
+	for i := 0; it.Next(); i++ {
+		if err := DecodeBytes(it.Value(), &items[i]); err != nil {
+			return items[:i], err
+		}
+	}
+	return items, nil
+}
+
+// Len returns the number of items in the list.
+func (r *RawList[T]) Len() int {
+	len, _ := CountValues(r.Content())
+	return len
+}
+
+// Size returns the encoded size of the list.
+func (r *RawList[T]) Size() uint64 {
+	return ListSize(uint64(len(r.Content())))
+}
+
+// Empty returns true if the list contains no items.
+func (r *RawList[T]) Empty() bool {
+	return len(r.Content()) == 0
+}
+
+// ContentIterator returns an iterator over the content of the list.
+// Note the offsets returned by iterator.Offset are relative to the
+// Content bytes of the list.
+func (r *RawList[T]) ContentIterator() *Iterator {
+	return newIterator(r.Content())
+}
+
+// Append adds an item to the end of the list.
+func (r *RawList[T]) Append(item T) error {
+	if r.enc == nil {
+		r.enc = make([]byte, 9)
+	}
+
+	eb := getEncBuffer()
+	defer encBufferPool.Put(eb)
+
+	if err := eb.encode(item); err != nil {
+		return err
+	}
+	prevEnd := len(r.enc)
+	end := prevEnd + eb.size()
+	r.enc = slices.Grow(r.enc, eb.size())[:end]
+	eb.copyTo(r.enc[prevEnd:end])
+	return nil
+}
+
 // StringSize returns the encoded size of a string.
 func StringSize(s string) uint64 {
 	switch n := len(s); n {