Skip to content

KeyValue.Update/Create returns "unknown" 10164 error code in case of bucket replicas > 1 #2097

Description

@vdorokhin

Observed behavior

When a KV bucket has Replicas > 1, concurrent Create/Update operations against a single key may fail with error code 10164 (JSStreamWrongLastSequenceConstantErr from the server) instead of 10071.
Because the Go client only recognizes code 10071, these errors cannot be handled by errors.Is(err, jetstream.ErrKeyExists) or by known error code validation (which only supports the constant jetstream.JSErrCodeStreamWrongLastSequence / 10071).

A similar issue was reported in the Python client: nats.py#782. It would be good to properly support error code 10164 in the Go client as well.
This may be related to nats-server#7281, but I'm not certain, so I opened this issue in the client repository.

Expected behavior

CAS operation related error codes (10164, 10071) — indicate wrong last sequence on the server side. The client should be able to handle 10164 alongside 10071 and can retry the CAS operation gracefully

Server and client version

Server: nats:2.14.2-alpine
Client: github.com/nats-io/nats.go v1.52.0

Host environment

MacOS
Docker

Steps to reproduce

Use docker-compose.yml file to spin up nats cluster and run following script:

package main

import (
	"context"
	"crypto/rand"
	"encoding/hex"
	"errors"
	"fmt"
	"log/slog"
	"os/signal"
	"sync"
	"syscall"
	"time"

	"github.com/nats-io/nats.go"
	"github.com/nats-io/nats.go/jetstream"
)

const (
	KVReplicas = 2 // Set >1 to reproduce unexpected errors with concurrent operations
	LoopCount  = 10
)

var (
	urls = []string{
		"nats://localhost:4222",
		"nats://localhost:4223",
		"nats://localhost:4224",
	}
	kvCfg = jetstream.KeyValueConfig{
		Bucket:         "test-nats-kv-bucket",
		MaxBytes:       10000000, // 10MB
		LimitMarkerTTL: time.Minute,
		Replicas:       KVReplicas,
		Description:    "KeyValue for test",
	}
)

func main() {
	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
	defer cancel()

	jss := must(createJSs(urls))

	kvs := make([]jetstream.KeyValue, len(urls))
	for i, js := range jss {
		kvs[i] = must(js.CreateOrUpdateKeyValue(ctx, kvCfg))
	}

	wg := runCreate(ctx, kvs, LoopCount)
	wg.Wait()

	wg = runUpdate(ctx, kvs, LoopCount)
	wg.Wait()
}

func runCreate(ctx context.Context, kvs []jetstream.KeyValue, loopCount int) *sync.WaitGroup {
	type create struct {
		key   string
		value []byte
	}

	createCh := make(chan create, len(kvs))
	defer close(createCh)

	createFn := func(ctx context.Context, kv jetstream.KeyValue) {
		for {
			select {
			case <-ctx.Done():
				return
			case c, ok := <-createCh:
				if !ok {
					return
				}

				if _, err := kv.Create(ctx, c.key, c.value); err != nil {
					if errors.Is(err, jetstream.ErrKeyExists) {
						slog.Info("create status code", slog.Any("error", err))
					} else {
						slog.Error("create status code", slog.Any("error", err))
					}
				}
			}
		}
	}

	wg := &sync.WaitGroup{}

	for _, kv := range kvs {
		wg.Go(func() {
			createFn(ctx, kv)
		})
	}

	kvsCount := len(kvs)

	for range loopCount {
		select {
		case <-ctx.Done():
			slog.Info("create: context done")
			return wg
		case <-time.After(250 * time.Millisecond):
		}

		newValue := create{
			key:   randomString(6),
			value: mustRandomBytes(6),
		}

		for range kvsCount {
			createCh <- newValue
		}
	}

	return wg
}

func runUpdate(ctx context.Context, kvs []jetstream.KeyValue, loopCount int) *sync.WaitGroup {
	expectedKey := randomString(6)

	must(kvs[0].Create(ctx, expectedKey, []byte("start value")))

	type update struct {
		key   string
		value []byte
	}

	updateCh := make(chan update, len(kvs))
	defer close(updateCh)

	updateFn := func(ctx context.Context, kv jetstream.KeyValue) {
		for {
			select {
			case <-ctx.Done():
				return
			case u, ok := <-updateCh:
				if !ok {
					return
				}

				valueEntry := must(kv.Get(ctx, u.key))

				if _, err := kv.Update(ctx, u.key, u.value, valueEntry.Revision()); err != nil {
					apiError, casted := errors.AsType[*jetstream.APIError](err)
					if !casted {
						slog.Error("unexpected error", slog.Any("error", err))
						continue
					}

					errorCode := apiError.ErrorCode

					if errorCode == jetstream.JSErrCodeStreamWrongLastSequence {
						slog.Info("update status code", slog.Int("code", int(errorCode)))
					} else {
						slog.Error("update status code", slog.Int("code", int(errorCode)))
					}
				}
			}
		}
	}

	wg := &sync.WaitGroup{}

	for _, kv := range kvs {
		wg.Go(func() {
			updateFn(ctx, kv)
		})
	}

	kvsCount := len(kvs)

	for range loopCount {
		select {
		case <-ctx.Done():
			slog.Info("update: context done")
			return wg
		case <-time.After(250 * time.Millisecond):
		}

		newValue := update{
			key:   expectedKey,
			value: mustRandomBytes(6),
		}

		for range kvsCount {
			updateCh <- newValue
		}
	}

	return wg
}

func createJSs(urls []string) ([]jetstream.JetStream, error) {
	jss := make([]jetstream.JetStream, len(urls))
	for i, url := range urls {
		conn, err := newNatsConnect(url)
		if err != nil {
			return nil, fmt.Errorf("new nats connection for %q: %w", url, err)
		}

		stream, err := jetstream.New(conn)
		if err != nil {
			return nil, fmt.Errorf("new jetstream for %q: %w", url, err)
		}

		jss[i] = stream
	}

	return jss, nil
}

func newNatsConnect(url string) (*nats.Conn, error) {
	opts := []nats.Option{
		nats.MaxReconnects(5),
		nats.ReconnectWait(3 * time.Second),
		nats.Timeout(5 * time.Second),
	}

	return nats.Connect(url, opts...)
}

func randomString(n int) string {
	return hex.EncodeToString(mustRandomBytes(n))
}

func mustRandomBytes(n int) []byte {
	b := make([]byte, n)

	must(rand.Read(b))

	return b
}

func must[T any](val T, err any) T {
	if err != nil {
		panic(err)
	}

	return val
}

Metadata

Metadata

Assignees

Labels

defectSuspected defect such as a bug or regression

Type

No type

Fields

No fields configured for issues without a type.

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions