fix(distributed): resolve paths for virtual models (mudler#11911)

Virtual model names have no primary file to anchor the worker path.
Companion assets still stage successfully, but relative options retain
an incorrect model directory and fail to load.

Derive the worker root from successfully staged option assets when the
primary path is absent. Cover Buffalo packs, files, directories,
overrides, and failed transfers. Document the frontend upgrade.

Assisted-by: Codex:gpt-6 golangci-lint

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

Author: localai-org-maint-bot

core/services/nodes/router.go

@@ -1599,8 +1599,15 @@ func (r *SmartRouter) stageModelFiles(ctx context.Context, node *BackendNode, op
 
 	// Stage file paths referenced in generic Options (key:value pairs where values
 	// are file paths). Options stay as relative paths — backends resolve them via ModelPath.
-	r.stageGenericOptions(ctx, node, opts.Options, frontendModelsDir, localModelDir, keyMapper.Key)
-	r.stageGenericOptions(ctx, node, opts.Overrides, frontendModelsDir, localModelDir, keyMapper.Key)
+	for _, options := range [][]string{opts.Options, opts.Overrides} {
+		remoteRoot := r.stageGenericOptions(ctx, node, options, frontendModelsDir, localModelDir, keyMapper.Key)
+		if opts.ModelFile == "" && remoteRoot != "" {
+			// Virtual models have no primary file from which to derive the
+			// worker root. Their relative options must resolve against the
+			// companion assets we actually staged, not the frontend's root.
+			opts.ModelPath = remoteRoot
+		}
+	}
 
 	return opts, nil
 }
@@ -1831,7 +1838,9 @@ func (r *SmartRouter) stageCompanionFiles(ctx context.Context, node *BackendNode
 // that resolve to existing files relative to the frontend models directory or
 // the model's own directory. Option values are NOT rewritten — backends resolve
 // them via ModelPath. keyFn generates the namespaced storage key for each file.
-func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode, options []string, frontendModelsDir, modelDir string, keyFn func(string) string) {
+// Returns the staged models root, or empty when no asset was staged.
+func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode, options []string, frontendModelsDir, modelDir string, keyFn func(string) string) string {
+	remoteRoot := ""
 	for _, opt := range options {
 		optKey, val, ok := strings.Cut(opt, ":")
 		if !ok || val == "" {
@@ -1856,18 +1865,23 @@ func (r *SmartRouter) stageGenericOptions(ctx context.Context, node *BackendNode
 		// worker; a single file is staged directly. Values are never rewritten —
 		// backends resolve relative paths via ModelPath.
 		if err == nil && info.IsDir() {
-			r.stageOptionDir(ctx, node, absPath, keyFn)
+			if remoteDir := r.stageOptionDir(ctx, node, absPath, keyFn); remoteDir != "" {
+				remoteRoot = DeriveRemoteModelPath(remoteDir, relativeToModelsDir(frontendModelsDir, absPath, filepath.Base(absPath)))
+			}
 			xlog.Debug("Staged option directory", "option", optKey, "localPath", absPath)
 			continue
 		}
 
 		key := keyFn(absPath)
-		if _, err := r.fileStager.EnsureRemote(ctx, node.ID, absPath, key); err != nil {
+		remotePath, err := r.fileStager.EnsureRemote(ctx, node.ID, absPath, key)
+		if err != nil {
 			xlog.Warn("Failed to stage option file, skipping", "option", opt, "path", absPath, "error", err)
 			continue
 		}
+		remoteRoot = DeriveRemoteModelPath(remotePath, relativeToModelsDir(frontendModelsDir, absPath, filepath.Base(absPath)))
 		xlog.Debug("Staged option file", "option", optKey, "localPath", absPath)
 	}
+	return remoteRoot
 }
 
 // resolveOptionPath finds an existing local path for an option value: an
@@ -1895,8 +1909,10 @@ func resolveOptionPath(val, frontendModelsDir, modelDir string) (string, bool) {
 // stageOptionDir stages every regular file under an option-declared directory
 // (e.g. sherpa-onnx's espeak-ng-data) using the structure-preserving key, so the
 // tree is recreated beside the model on the worker. Per-file errors are logged
-// and skipped; the option value itself is not rewritten.
-func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir string, keyFn func(string) string) {
+// and skipped; the option value itself is not rewritten. Returns the remote
+// directory derived from a successfully staged file, or empty when none succeeds.
+func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir string, keyFn func(string) string) string {
+	remoteDir := ""
 	_ = filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error {
 		if walkErr != nil || d.IsDir() {
 			return nil
@@ -1911,11 +1927,17 @@ func (r *SmartRouter) stageOptionDir(ctx context.Context, node *BackendNode, dir
 		if isHashSidecar(path) {
 			return nil
 		}
-		if _, err := r.fileStager.EnsureRemote(ctx, node.ID, path, keyFn(path)); err != nil {
+		remotePath, err := r.fileStager.EnsureRemote(ctx, node.ID, path, keyFn(path))
+		if err != nil {
 			xlog.Warn("Failed to stage option directory file, skipping", "path", path, "error", err)
+			return nil
+		}
+		if rel, err := filepath.Rel(dir, path); err == nil {
+			remoteDir = DeriveRemoteModelPath(remotePath, rel)
 		}
 		return nil
 	})
+	return remoteDir
 }
 
 // probeHealth checks whether a backend process on the given node/addr is alive

core/services/nodes/router_virtual_model_test.go

@@ -0,0 +1,83 @@
+// SPDX-License-Identifier: MIT
+
+package nodes
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+type failedCompanionStager struct{ FileStager }
+
+func (failedCompanionStager) EnsureRemote(context.Context, string, string, string) (string, error) {
+	return "", errors.New("worker unavailable")
+}
+
+var _ = Describe("staging virtual model companions", func() {
+	DescribeTable("anchors relative assets on the worker",
+		func(options, overrides []string, files []string) {
+			modelsDir := GinkgoT().TempDir()
+			for _, name := range files {
+				path := filepath.Join(modelsDir, name)
+				Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
+				Expect(os.WriteFile(path, []byte("weights"), 0600)).To(Succeed())
+			}
+			stager := &fakeFileStager{}
+			router := &SmartRouter{fileStager: stager, stagingTracker: NewStagingTracker()}
+			input := &pb.ModelOptions{Model: "insightface-buffalo-m", ModelFile: filepath.Join(modelsDir, "insightface-buffalo-m"), ModelPath: modelsDir, Options: options, Overrides: overrides}
+			staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "insightface-buffalo-m")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(staged.ModelPath).To(Equal("/remote/models/insightface-buffalo-m"))
+			Expect(staged.Options).To(Equal(options))
+			Expect(staged.Overrides).To(Equal(overrides))
+			Expect(input.ModelPath).To(Equal(modelsDir))
+			Expect(input.ModelFile).To(Equal(filepath.Join(modelsDir, "insightface-buffalo-m")))
+			Expect(stager.ensureCalls).To(HaveLen(len(files)))
+			for _, call := range stager.ensureCalls {
+				rel, err := filepath.Rel(modelsDir, call.localPath)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(filepath.Join(staged.ModelPath, rel)).To(Equal("/remote/" + call.key))
+			}
+		},
+		Entry("Buffalo pack and MiniFASNet files", []string{"model_pack:buffalo_m", "antispoof_v2_onnx:MiniFASNetV2.onnx", "antispoof_v1se_onnx:MiniFASNetV1SE.onnx"}, nil, []string{"buffalo_m/det_2.5g.onnx", "buffalo_m/w600k_r50.onnx", "MiniFASNetV2.onnx", "MiniFASNetV1SE.onnx"}),
+		Entry("only a nested companion directory", []string{"model_pack:packs/buffalo_m"}, nil, []string{"packs/buffalo_m/det_2.5g.onnx"}),
+		Entry("only a companion file", []string{"antispoof_v2_onnx:MiniFASNetV2.onnx"}, nil, []string{"MiniFASNetV2.onnx"}),
+		Entry("only override assets", nil, []string{"antispoof_v2_onnx:MiniFASNetV2.onnx"}, []string{"MiniFASNetV2.onnx"}),
+	)
+	It("keeps the original path when no assets are staged", func() {
+		modelsDir := GinkgoT().TempDir()
+		router := &SmartRouter{fileStager: &fakeFileStager{}, stagingTracker: NewStagingTracker()}
+		input := &pb.ModelOptions{Model: "virtual", ModelFile: filepath.Join(modelsDir, "virtual"), ModelPath: modelsDir, Options: []string{"engine:insightface"}}
+		staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "virtual")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(staged.ModelPath).To(Equal(modelsDir))
+	})
+	DescribeTable("keeps the original root when companion staging fails",
+		func(directory bool) {
+			modelsDir := GinkgoT().TempDir()
+			relative := "MiniFASNetV2.onnx"
+			if directory {
+				relative = "buffalo_m/det_2.5g.onnx"
+			}
+			local := filepath.Join(modelsDir, relative)
+			Expect(os.MkdirAll(filepath.Dir(local), 0750)).To(Succeed())
+			Expect(os.WriteFile(local, []byte("weights"), 0600)).To(Succeed())
+			value := relative
+			if directory {
+				value = "buffalo_m"
+			}
+			router := &SmartRouter{fileStager: failedCompanionStager{}, stagingTracker: NewStagingTracker()}
+			input := &pb.ModelOptions{Model: "virtual", ModelFile: filepath.Join(modelsDir, "virtual"), ModelPath: modelsDir, Options: []string{"asset:" + value}}
+			staged, err := router.stageModelFiles(context.Background(), &BackendNode{ID: "worker"}, input, "virtual")
+			Expect(err).NotTo(HaveOccurred())
+			Expect(staged.ModelPath).To(Equal(modelsDir))
+		}, Entry("file", false), Entry("directory", true),
+	)
+
+})

docs/content/features/distributed-mode.md

@@ -1190,6 +1190,10 @@ Notes:
 - Verify `--heartbeat-interval` is not set too high
 - Offline nodes automatically restore to healthy when they re-register (no re-approval needed)
 
+**InsightFace reports a missing MiniFASNet file after staging:**
+- Gallery models such as `insightface-buffalo-m` use a virtual primary name and load their files through options. The frontend derives the worker's model directory from successfully staged companion files or directories, so relative options resolve inside the model's staging directory.
+- If logs show matching hashes for the staged files but InsightFace still reports a bare filename such as `MiniFASNetV2.onnx` as missing, upgrade the frontend to include this path-resolution fix. Re-uploading the same files does not correct the directory passed to the backend.
+
 **Backend not installing:**
 - Check the worker logs for `backend.install` events