[pull] master from mudler:master

backend/cpp/ds4/CMakeLists.txt

@@ -84,9 +84,10 @@ elseif(DS4_GPU STREQUAL "cpu")
     set(DS4_OBJS "${DS4_DIR}/ds4_cpu.o")
 endif()
 
-# Upstream splits distributed inference, tensor-parallel transport, the SSD
-# expert cache, and layer placement into GPU-agnostic translation units. Link
-# them regardless of DS4_GPU.
+# Upstream splits image preprocessing, distributed inference, tensor-parallel
+# transport, the SSD expert cache, and layer placement into GPU-agnostic
+# translation units. Link them regardless of DS4_GPU.
+list(APPEND DS4_OBJS "${DS4_DIR}/ds4_image.o")
 list(APPEND DS4_OBJS "${DS4_DIR}/ds4_distributed.o")
 list(APPEND DS4_OBJS "${DS4_DIR}/ds4_tp.o")
 list(APPEND DS4_OBJS "${DS4_DIR}/ds4_ssd.o")

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577
+# Upstream pin lives below as DS4_VERSION?=9ab705347c1775e7599ede7eb81a6255ec7dccb5
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577
+DS4_VERSION?=9ab705347c1775e7599ede7eb81a6255ec7dccb5
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -79,21 +79,22 @@ else
 endif
 endif
 
-# Upstream splits distributed inference, tensor-parallel transport, the SSD
-# expert cache, and layer placement into GPU-agnostic translation units. They
-# are shared by every GPU mode, so append them unconditionally below.
+# Upstream splits image preprocessing, distributed inference, tensor-parallel
+# transport, the SSD expert cache, and layer placement into GPU-agnostic
+# translation units. They are shared by every GPU mode, so append them
+# unconditionally below.
 ifeq ($(BUILD_TYPE),cublas)
     CMAKE_ARGS += -DDS4_GPU=cuda
-    DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \
+    DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \
         cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \
         cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o
 else ifeq ($(UNAME_S),Darwin)
     CMAKE_ARGS += -DDS4_GPU=metal
-    DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+    DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else
     # CPU reference path (Linux only - macOS CPU path is broken by VM bug per ds4 README).
     CMAKE_ARGS += -DDS4_GPU=cpu
-    DS4_OBJ_TARGET := ds4_cpu.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+    DS4_OBJ_TARGET := ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 endif
 
 ifneq ($(NATIVE),true)
@@ -120,9 +121,9 @@ ds4/ds4.o: ds4
 ifeq ($(BUILD_TYPE),cublas)
 	+$(MAKE) -C ds4 $(DS4_ARCH_MAKEVARS) $(DS4_OBJ_TARGET)
 else ifeq ($(UNAME_S),Darwin)
-	+$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+	+$(MAKE) -C ds4 ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 else
-	+$(MAKE) -C ds4 ds4_cpu.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
+	+$(MAKE) -C ds4 ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o
 endif
 
 grpc-server: ds4/ds4.o

backend/go/stablediffusion-ggml/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=6b3edaaf32cc19e5bb2d819c788bd557eddc8eba
+STABLEDIFFUSION_GGML_VERSION?=d8fb10c02977c8ca999f3fb4e02df9ecf10f7ba6
 
 CMAKE_ARGS+=-DGGML_MAX_NAME=128
 

backend/go/stablediffusion-ggml/cpp/gosd.cpp

@@ -401,7 +401,6 @@ int load_model(const char *model, char *model_path, char* options[], int threads
     const char *params_backend_arg = "";
     const char *rpc_servers_arg = "";
     const char *max_vram_arg = "";
-    bool stream_layers = false;
 
     int n_threads = threads;
     enum sd_type_t wtype = SD_TYPE_COUNT;
@@ -510,7 +509,10 @@ int load_model(const char *model, char *model_path, char* options[], int threads
         if (!strcmp(optname, "params_backend")) params_backend_arg = strdup(optval);
         if (!strcmp(optname, "rpc_servers")) rpc_servers_arg = strdup(optval);
         if (!strcmp(optname, "max_vram")) max_vram_arg = strdup(optval);
-        if (!strcmp(optname, "stream_layers")) stream_layers = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0);
+        if (!strcmp(optname, "stream_layers")) {
+            // Retained as a no-op for existing configurations. Upstream now
+            // selects segmented weight streaming automatically.
+        }
 
         // vae_decode_only is still accepted for backwards compatibility with
         // existing gallery configs, but upstream dropped the option (the model
@@ -650,11 +652,9 @@ int load_model(const char *model, char *model_path, char* options[], int threads
             ctx_params.rpc_servers = env_rpc_servers;
         }
     }
-    // max_vram: GiB budget or per-backend spec for graph-cut segmented param
-    // offload ("0" = disabled, "-1" = auto). stream_layers only has effect when
-    // max_vram is set.
+    // max_vram is an optional GiB budget or per-backend spec for automatic
+    // graph-cut execution. A zero value uses the live free-VRAM budget.
     if (strlen(max_vram_arg) > 0) ctx_params.max_vram = max_vram_arg;
-    ctx_params.stream_layers = stream_layers;
     ctx_params.diffusion_flash_attn = diffusion_flash_attn;
     ctx_params.tae_preview_only = tae_preview_only;
     ctx_params.diffusion_conv_direct = diffusion_conv_direct;
@@ -1438,4 +1438,3 @@ int unload() {
     free_sd_ctx(sd_c);
     return 0;
 }
-

core/http/endpoints/localai/face_register.go

@@ -1,6 +1,7 @@
 package localai
 
 import (
+	"errors"
 	"net/http"
 
 	"github.com/labstack/echo/v4"
@@ -33,22 +34,31 @@ func FaceRegisterEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a
 			return echo.NewHTTPError(http.StatusBadRequest, "name is required")
 		}
 
-		img, err := decodeImageInput(input.Img)
-		if err != nil {
-			return err
+		if (input.Img == "") == (len(input.Embedding) == 0) {
+			return echo.NewHTTPError(http.StatusBadRequest, "provide exactly one of img or embedding")
 		}
-
-		xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name)
-		embedding, err := backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg)
-		if err != nil {
-			return mapBackendError(err)
+		embedding := input.Embedding
+		if len(embedding) == 0 {
+			img, err := decodeImageInput(input.Img)
+			if err != nil {
+				return err
+			}
+			xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name)
+			embedding, err = backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg)
+			if err != nil {
+				return mapBackendError(err)
+			}
 		}
 
 		stored, err := registry.Register(c.Request().Context(), embedding, facerecognition.Metadata{
-			Name:   input.Name,
-			Labels: input.Labels,
+			Name:         input.Name,
+			RegisteredAt: input.RegisteredAt,
+			Labels:       input.Labels,
 		})
 		if err != nil {
+			if errors.Is(err, facerecognition.ErrInvalidEmbedding) || errors.Is(err, facerecognition.ErrDimensionMismatch) {
+				return echo.NewHTTPError(http.StatusBadRequest, err.Error())
+			}
 			return err
 		}
 		return c.JSON(http.StatusOK, schema.FaceRegisterResponse{

core/http/endpoints/localai/face_register_test.go

@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: MIT
+
+package localai_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"time"
+
+	"github.com/labstack/echo/v4"
+	"github.com/mudler/LocalAI/core/config"
+	. "github.com/mudler/LocalAI/core/http/endpoints/localai"
+	"github.com/mudler/LocalAI/core/http/middleware"
+	"github.com/mudler/LocalAI/core/schema"
+	"github.com/mudler/LocalAI/core/services/facerecognition"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+type registrationRecorder struct {
+	facerecognition.Registry
+	vector []float32
+	meta   facerecognition.Metadata
+	err    error
+}
+
+func (r *registrationRecorder) Register(_ context.Context, v []float32, m facerecognition.Metadata) (facerecognition.Metadata, error) {
+	r.vector = v
+	r.meta = m
+	m.ID = "saved-id"
+	return m, r.err
+}
+
+var _ = Describe("Face registration replay", func() {
+	var reg *registrationRecorder
+	call := func(in schema.FaceRegisterRequest) (*httptest.ResponseRecorder, error) {
+		e := echo.New()
+		rec := httptest.NewRecorder()
+		c := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/face/register", nil), rec)
+		c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &in)
+		c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{})
+		// No model loader: replay must not call the embedding backend.
+		err := FaceRegisterEndpoint(nil, nil, nil, reg)(c)
+		return rec, err
+	}
+	BeforeEach(func() { reg = &registrationRecorder{} })
+	It("accepts the saved vector and timestamp without running inference", func() {
+		at := time.Now().UTC()
+		in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{1, 0}, RegisteredAt: at, Labels: map[string]string{"client_id": "alice"}}
+		in.Model = "faces"
+		rec, err := call(in)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(rec.Code).To(Equal(http.StatusOK))
+		Expect(reg.vector).To(Equal(in.Embedding))
+		Expect(reg.meta.RegisteredAt).To(Equal(at))
+		Expect(reg.meta.Labels).To(Equal(in.Labels))
+		Expect(rec.Body.String()).To(ContainSubstring("saved-id"))
+	})
+	It("rejects ambiguous and missing inputs before inference", func() {
+		for _, in := range []schema.FaceRegisterRequest{
+			{Name: "Alice"},
+			{Name: "Alice", Img: "image", Embedding: []float32{1, 0}},
+		} {
+			in.Model = "faces"
+			_, err := call(in)
+			Expect(err).To(HaveOccurred())
+			Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest))
+			Expect(reg.vector).To(BeNil())
+		}
+	})
+	It("reports invalid vectors as a client error", func() {
+		reg.err = facerecognition.ErrInvalidEmbedding
+		in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{0, 0}}
+		in.Model = "faces"
+		_, err := call(in)
+		Expect(err).To(HaveOccurred())
+		Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest))
+	})
+})

core/schema/localai.go

@@ -353,10 +353,12 @@ type FaceEmbedResponse struct {
 // FaceRegisterRequest enrolls a face into the 1:N recognition store.
 type FaceRegisterRequest struct {
 	BasicModelRequest
-	Img    string            `json:"img"`
-	Name   string            `json:"name"`
-	Labels map[string]string `json:"labels,omitempty"`
-	Store  string            `json:"store,omitempty"` // vector store model; empty = local-store default
+	RegisteredAt time.Time         `json:"registered_at,omitempty"` // original enrollment time when replaying a saved embedding
+	Embedding    []float32         `json:"embedding,omitempty"`
+	Img          string            `json:"img"`
+	Name         string            `json:"name"`
+	Labels       map[string]string `json:"labels,omitempty"`
+	Store        string            `json:"store,omitempty"` // vector store model; empty = local-store default
 }
 
 type FaceRegisterResponse struct {

core/services/facerecognition/registry.go

@@ -56,5 +56,6 @@ type Match struct {
 var (
 	ErrNotFound          = errors.New("facerecognition: id not found")
 	ErrEmptyEmbedding    = errors.New("facerecognition: embedding is empty")
+	ErrInvalidEmbedding  = errors.New("facerecognition: embedding must be finite and nonzero")
 	ErrDimensionMismatch = errors.New("facerecognition: embedding dimension mismatch")
 )

core/services/facerecognition/replay_test.go

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: MIT
+
+package facerecognition
+
+import (
+	"context"
+	"encoding/json"
+	"math"
+	"sync"
+	"testing"
+	"time"
+
+	grpc "github.com/mudler/LocalAI/pkg/grpc"
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	ggrpc "google.golang.org/grpc"
+)
+
+func TestEnrollmentReplay(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Enrollment replay") }
+
+type replayStore struct {
+	grpc.Backend
+	mu      sync.Mutex
+	entries map[string][]byte
+}
+
+func (s *replayStore) StoresSet(_ context.Context, in *pb.StoresSetOptions, _ ...ggrpc.CallOption) (*pb.Result, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for i, k := range in.Keys {
+		b, _ := json.Marshal(k.Floats)
+		s.entries[string(b)] = append([]byte(nil), in.Values[i].Bytes...)
+	}
+	return &pb.Result{Success: true}, nil
+}
+
+var _ = Describe("Enrollment replay", func() {
+	It("keeps the identity across registry instances and a cleared store", func(ctx SpecContext) {
+		storage := &replayStore{entries: map[string][]byte{}}
+		newRegistry := func() Registry {
+			return NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0)
+		}
+		vector := []float32{1, 0, 0, 0}
+		meta := Metadata{Name: "Alice", RegisteredAt: time.Now().UTC()}
+		first, err := newRegistry().Register(ctx, vector, meta)
+		Expect(err).NotTo(HaveOccurred())
+		again, err := newRegistry().Register(ctx, vector, meta)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(again).To(Equal(first))
+		Expect(storage.entries).To(HaveLen(1))
+		storage.entries = map[string][]byte{}
+		restored, err := newRegistry().Register(ctx, vector, meta)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(restored).To(Equal(first))
+		Expect(storage.entries).To(HaveLen(1))
+	})
+	It("rejects zero and non-finite embeddings before writing", func(ctx SpecContext) {
+		for _, v := range [][]float32{{0, 0}, {float32(math.NaN()), 1}, {float32(math.Inf(1)), 1}} {
+			storage := &replayStore{entries: map[string][]byte{}}
+			reg := NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0)
+			_, err := reg.Register(ctx, v, Metadata{Name: "Alice"})
+			Expect(err).To(HaveOccurred())
+			Expect(storage.entries).To(BeEmpty())
+		}
+	})
+})

core/services/facerecognition/store_registry.go

@@ -2,8 +2,10 @@ package facerecognition
 
 import (
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"math"
 	"sort"
 	"sync"
 	"time"
@@ -57,13 +59,32 @@ func (r *storeRegistry) Register(ctx context.Context, embedding []float32, meta
 	if r.dim != 0 && len(embedding) != r.dim {
 		return Metadata{}, fmt.Errorf("%w: expected %d, got %d", ErrDimensionMismatch, r.dim, len(embedding))
 	}
+	var norm float64
+	key := make([]byte, 4*len(embedding))
+	for i, value := range embedding {
+		if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) {
+			return Metadata{}, ErrInvalidEmbedding
+		}
+		norm += float64(value) * float64(value)
+		// The store treats negative and positive zero as the same key.
+		if value == 0 {
+			value = 0
+		}
+		binary.LittleEndian.PutUint32(key[i*4:], math.Float32bits(value))
+	}
+	if norm == 0 {
+		return Metadata{}, ErrInvalidEmbedding
+	}
 
 	backend, err := r.resolve(ctx, r.storeName)
 	if err != nil {
 		return Metadata{}, fmt.Errorf("facerecognition: resolve store: %w", err)
 	}
 
-	meta.ID = uuid.NewString()
+	// The vector store upserts by the exact embedding. Derive the ID from the
+	// same key so replaying a saved vector preserves identity across replicas
+	// and after the in-memory store restarts.
+	meta.ID = uuid.NewSHA1(uuid.NewSHA1(uuid.NameSpaceOID, []byte(r.storeName)), key).String()
 	if meta.RegisteredAt.IsZero() {
 		meta.RegisteredAt = time.Now().UTC()
 	}