Merge pull request #138 from proost/doc-utf8-compatibility

doc: utf8 compatibility

Author: proost

common/item_sketch_string.go

@@ -67,6 +67,11 @@ func (f ItemSketchStringSerDe) SizeOfMany(mem []byte, offsetBytes int, numItems
 	return offset - offsetBytes, nil
 }
 
+// SerializeOneToSlice writes the item to a byte slice.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the input string is encoded as valid UTF-8.
 func (f ItemSketchStringSerDe) SerializeOneToSlice(item string) []byte {
 	if len(item) == 0 {
 		return []byte{}
@@ -78,6 +83,11 @@ func (f ItemSketchStringSerDe) SerializeOneToSlice(item string) []byte {
 	return bytesOut
 }
 
+// SerializeManyToSlice writes items to a byte slice.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that those strings are encoded as valid UTF-8.
 func (f ItemSketchStringSerDe) SerializeManyToSlice(items []string) []byte {
 	if len(items) == 0 {
 		return []byte{}
@@ -98,6 +108,11 @@ func (f ItemSketchStringSerDe) SerializeManyToSlice(items []string) []byte {
 	return bytesOut
 }
 
+// DeserializeManyFromSlice reconstructs bytes from its serialized form.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func (f ItemSketchStringSerDe) DeserializeManyFromSlice(mem []byte, offsetBytes int, numItems int) ([]string, error) {
 	if numItems <= 0 {
 		return []string{}, nil

frequencies/items_sketch.go

@@ -105,6 +105,10 @@ func NewFrequencyItemsSketchWithMaxMapSize[C comparable](maxMapSize int, hasher
 // sketch and must be a power of 2.  The maximum capacity of this internal hash map is
 // 0.75 times * maxMapSize. Both the ultimate accuracy and size of this sketch are a
 // function of maxMapSize.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func NewFrequencyItemsSketchFromSlice[C comparable](slc []byte, hasher common.ItemSketchHasher[C], serde common.ItemSketchSerde[C]) (*ItemsSketch[C], error) {
 	if serde == nil {
 		return nil, errors.New("no SerDe provided")
@@ -327,6 +331,10 @@ func (i *ItemsSketch[C]) IsEmpty() bool {
 // Update this sketch with an item and a frequency count of one.
 //
 // item for which the frequency should be increased.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the input string is encoded as valid UTF-8.
 func (i *ItemsSketch[C]) Update(item C) error {
 	return i.UpdateMany(item, 1)
 }
@@ -337,6 +345,10 @@ func (i *ItemsSketch[C]) Update(item C) error {
 // and is only used by the sketch to determine uniqueness.
 // count the amount by which the frequency of the item should be increased.
 // A count of zero is a no-op, and a negative count will throw an exception.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that input strings are encoded as valid UTF-8.
 func (i *ItemsSketch[C]) UpdateMany(item C, count int64) error {
 	if internal.IsNil(item) || count == 0 {
 		return nil
@@ -374,6 +386,10 @@ func (i *ItemsSketch[C]) UpdateMany(item C, count int64) error {
 //
 // return a sketch whose estimates are within the guarantees of the largest error tolerance
 // of the two merged sketches.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that string values in both sketches are encoded as valid UTF-8.
 func (i *ItemsSketch[C]) Merge(other *ItemsSketch[C]) (*ItemsSketch[C], error) {
 	if other == nil || other.IsEmpty() {
 		return i, nil
@@ -412,7 +428,11 @@ func (i *ItemsSketch[C]) ToString() (string, error) {
 	return sb.String(), nil
 }
 
-// ToSlice returns a slice representation of this sketch
+// ToSlice returns a slice representation of this sketch.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func (i *ItemsSketch[C]) ToSlice() ([]byte, error) {
 	if i.hashMap.serde == nil {
 		return nil, errors.New("no SerDe provided")

kll/items_sketch.go

@@ -108,6 +108,10 @@ func NewKllItemsSketchWithDefault[C comparable](compareFn common.CompareFn[C], s
 }
 
 // NewKllItemsSketchFromSlice create a new ItemsSketch from the given byte slice (serialized sketch).
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func NewKllItemsSketchFromSlice[C comparable](sl []byte, compareFn common.CompareFn[C], serde common.ItemSketchSerde[C]) (*ItemsSketch[C], error) {
 	if serde == nil {
 		return nil, fmt.Errorf("no SerDe provided")
@@ -447,12 +451,20 @@ func (s *ItemsSketch[C]) GetSortedView() (*ItemsSketchSortedView[C], error) {
 }
 
 // Update this sketch with the given item.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the input string is encoded as valid UTF-8.
 func (s *ItemsSketch[C]) Update(item C) {
 	s.updateItem(item, s.compareFn)
 	s.sortedView = nil
 }
 
 // Merge the given sketch into this sketch.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that string values in both sketches are encoded as valid UTF-8.
 func (s *ItemsSketch[C]) Merge(other *ItemsSketch[C]) {
 	if other.IsEmpty() {
 		return
@@ -476,6 +488,10 @@ func (s *ItemsSketch[C]) Reset() {
 }
 
 // ToSlice returns the serialized byte array of this sketch.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func (s *ItemsSketch[C]) ToSlice() ([]byte, error) {
 	if s.serde == nil {
 		return nil, fmt.Errorf("no SerDe provided")

sampling/reservoir_items_sketch.go

@@ -95,6 +95,10 @@ func NewReservoirItemsSketch[T any](
 }
 
 // Update adds an item to the sketch using reservoir sampling algorithm.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the input string is encoded as valid UTF-8.
 func (s *ReservoirItemsSketch[T]) Update(item T) {
 	if s.n < int64(s.k) {
 		// Initial phase: store all items until reservoir is full
@@ -304,6 +308,10 @@ func resizeFactorFromHeaderByte(b byte) (ResizeFactor, error) {
 }
 
 // ToSlice serializes the sketch to a byte slice.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func (s *ReservoirItemsSketch[T]) ToSlice(serde ItemsSerDe[T]) ([]byte, error) {
 	rfBits, err := resizeFactorBitsFor(s.rf)
 	if err != nil {
@@ -365,6 +373,10 @@ func (s *ReservoirItemsSketch[T]) String() string {
 }
 
 // NewReservoirItemsSketchFromSlice deserializes a sketch from a byte slice.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func NewReservoirItemsSketchFromSlice[T any](data []byte, serde ItemsSerDe[T]) (*ReservoirItemsSketch[T], error) {
 	if len(data) < 8 {
 		return nil, errors.New("data too short")

sampling/reservoir_items_union.go

@@ -53,6 +53,10 @@ func NewReservoirItemsUnion[T any](maxK int) (*ReservoirItemsUnion[T], error) {
 }
 
 // Update adds a single item to the union.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the input string is encoded as valid UTF-8.
 func (u *ReservoirItemsUnion[T]) Update(item T) {
 	if u.gadget == nil {
 		u.gadget, _ = NewReservoirItemsSketch[T](u.maxK)
@@ -62,6 +66,10 @@ func (u *ReservoirItemsUnion[T]) Update(item T) {
 
 // UpdateSketch merges another sketch into the union.
 // This implements Java's update(ReservoirItemsSketch) with twoWayMergeInternal logic.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that string values in both sketches are encoded as valid UTF-8.
 func (u *ReservoirItemsUnion[T]) UpdateSketch(sketch *ReservoirItemsSketch[T]) error {
 	if sketch == nil || sketch.IsEmpty() {
 		return nil
@@ -91,6 +99,10 @@ func (u *ReservoirItemsUnion[T]) UpdateSketch(sketch *ReservoirItemsSketch[T]) e
 
 // UpdateFromRaw creates a sketch from raw components and merges it.
 // Useful in distributed environments. Items slice is used directly, not copied.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that input strings are encoded as valid UTF-8.
 func (u *ReservoirItemsUnion[T]) UpdateFromRaw(n int64, k int, items []T) error {
 	if len(items) == 0 {
 		return nil
@@ -234,6 +246,10 @@ const (
 )
 
 // ToSlice serializes the union to a byte slice.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func (u *ReservoirItemsUnion[T]) ToSlice(serde ItemsSerDe[T]) ([]byte, error) {
 	empty := u.gadget == nil || u.gadget.IsEmpty()
 
@@ -267,6 +283,10 @@ func (u *ReservoirItemsUnion[T]) ToSlice(serde ItemsSerDe[T]) ([]byte, error) {
 }
 
 // NewReservoirItemsUnionFromSlice deserializes a union from a byte slice.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 func NewReservoirItemsUnionFromSlice[T any](data []byte, serde ItemsSerDe[T]) (*ReservoirItemsUnion[T], error) {
 	if len(data) < 8 {
 		return nil, errors.New("data too short")

sampling/varopt_items_sketch.go

@@ -203,6 +203,10 @@ func (s *VarOptItemsSketch[T]) peekMin() (float64, error) {
 
 // Update adds an item with the given weight to the sketch.
 // Weight must be positive and finite.
+//
+// If the sketch contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the input string is encoded as valid UTF-8.
 func (s *VarOptItemsSketch[T]) Update(item T, weight float64) error {
 	return s.update(item, weight, false)
 }

tuple/arrayofstrings_sketch.go

@@ -78,6 +78,9 @@ func (s *ArrayOfStringsSummary) Update(values []string) {
 // ArrayOfStringsSummaryWriter writes an ArrayOfStringsSummary to the provided io.Writer in binary format.
 // It validates the length of the string slice and computes total bytes for serialization.
 // Returns an error if the input exceeds the maximum allowed slice length or if any write operation fails.
+//
+// If the caller cares about cross-language compatibility,
+// it is the caller's responsibility to ensure that those strings are encoded as valid UTF-8.
 func ArrayOfStringsSummaryWriter(w io.Writer, summary *ArrayOfStringsSummary) error {
 	return writeStrings(w, summary.values)
 }
@@ -121,6 +124,9 @@ func computeStringsTotalBytes(values []string) uint32 {
 // ArrayOfStringsSummaryReader reads an ArrayOfStringsSummary from the provided io.Reader in binary format.
 // It validates the length of the string slice and reads the total bytes for deserialization.
 // Returns an error if the input exceeds the maximum allowed slice length or if any read operation fails.
+//
+// If the caller cares about cross-language compatibility,
+// it is the caller's responsibility to ensure that the serialized string data is encoded as valid UTF-8.
 func ArrayOfStringsSummaryReader(r io.Reader) (*ArrayOfStringsSummary, error) {
 	values, err := readStrings(r)
 	if err != nil {
@@ -201,13 +207,19 @@ func ArrayOfStringsInlineSummaryUpdateFunc(s ArrayOfStringsInlineSummary, values
 // ArrayOfStringsInlineSummaryWriter writes an ArrayOfStringsInlineSummary to the provided io.Writer in binary format.
 // It validates the length of the string slice and computes total bytes for serialization.
 // Returns an error if the input exceeds the maximum allowed slice length or if any write operation fails.
+//
+// If the caller cares about cross-language compatibility,
+// it is the caller's responsibility to ensure that those strings are encoded as valid UTF-8.
 func ArrayOfStringsInlineSummaryWriter(w io.Writer, summary ArrayOfStringsInlineSummary) error {
 	return writeStrings(w, summary.values)
 }
 
 // ArrayOfStringsInlineSummaryReader reads an ArrayOfStringsInlineSummary from the provided io.Reader in binary format.
 // It validates the length of the string slice and reads the total bytes for deserialization.
 // Returns an error if the input exceeds the maximum allowed slice length or if any read operation fails.
+//
+// If the caller cares about cross-language compatibility,
+// it is the caller's responsibility to ensure that the serialized string data is encoded as valid UTF-8.
 func ArrayOfStringsInlineSummaryReader(r io.Reader) (ArrayOfStringsInlineSummary, error) {
 	values, err := readStrings(r)
 	if err != nil {

tuple/decoder.go

@@ -29,6 +29,10 @@ import (
 
 // SummaryReader reads and returns a summary from the reader.
 // Implementations should read the format written by a corresponding SummaryWriter.
+//
+// If the summary contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that the serialized string data is encoded as valid UTF-8.
 type SummaryReader[S Summary] func(r io.Reader) (S, error)
 
 // Decoder decodes a compact sketch from the given reader.

tuple/encoder.go

@@ -26,6 +26,10 @@ import (
 
 // SummaryWriter writes a summary to the writer.
 // Implementations should write the summary in a format that can be read by a corresponding SummaryReader.
+//
+// If the summary contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that those strings are encoded as valid UTF-8.
 type SummaryWriter[S Summary] func(w io.Writer, s S) error
 
 // Encoder encodes a compact tuple sketch to bytes.

tuple/intersection.go

@@ -103,6 +103,10 @@ func NewIntersectionWithSummaryMergeFunc[S Summary](
 }
 
 // Update updates the intersection with a given sketch.
+//
+// If the summary contains string values and the caller cares about
+// cross-language compatibility, it is the caller's responsibility to ensure
+// that both sketches use string values encoded as valid UTF-8.
 func (i *Intersection[S]) Update(sketch Sketch[S]) error {
 	if i.hashtable.isEmpty {
 		return nil