Faces: Carry the detector provenance when upgrading a marker

Author: lastzero

internal/entity/face.go

@@ -19,6 +19,8 @@ import (
 )
 
 var faceMutex = sync.Mutex{}
+
+// UpdateFaces reports whether a matching pass changed a cluster, so callers know to refresh.
 var UpdateFaces = atomic.Bool{}
 
 // Face represents the face of a Subject.

internal/entity/file.go

@@ -745,11 +745,9 @@ func (m *File) SetInstanceID(id string) {
 	}
 }
 
-// RedactForSession removes identifying per-file metadata a shared-only session must not see when it
-// accesses a file through sharing: the XMP InstanceID (a content-provenance identifier) is cleared and
-// markers are omitted. Sessions with full library or admin access (and nil sessions) are unchanged.
-// This is the per-file counterpart of Photo.RedactForSession, so single-file reads (GetFile) and the
-// picture read (GetPhoto) strip the same fields.
+// RedactForSession removes identifying per-file metadata a shared-only session must not see: the
+// XMP InstanceID is cleared and markers are omitted. Full-library, admin and nil sessions are
+// unchanged. Counterpart of Photo.RedactForSession, so GetFile and GetPhoto strip the same fields.
 func (m *File) RedactForSession(sess *Session) *File {
 	if m == nil || sess == nil {
 		return m
@@ -896,16 +894,28 @@ func (m *File) AddFace(f face.Face, subjUid string) {
 		if existing.Embeddings().Empty() {
 			landmarks := f.RelativeLandmarksJSON()
 
+			// Unmeasured stays -1 rather than keeping a value recorded for another sampling.
+			thumbSize := -1
+
+			if f.ThumbSize > 0 {
+				thumbSize = f.ThumbSize
+			}
+
 			// For an already-saved marker, persist first and mutate in-memory
 			// only on success: a failed write must not leave an unpersisted
 			// embedding (Markers.Save does not re-write existing markers), so the
 			// marker stays embedding-less and is retried on the next pass.
 			if existing.MarkerUID != "" {
+				// thumb_size and score travel with the detector that produced them, or both gates
+				// read the half this upgrade did not replace: the size bar would fall back to an
+				// XMP-declared box extent, and the score bar is looked up by detect_model.
 				values := Values{
 					"embeddings_json": f.Embeddings.JSON(),
 					"embed_model":     f.EmbedModel,
 					"detect_model":    f.DetectModel,
 					"landmarks_json":  landmarks,
+					"thumb_size":      thumbSize,
+					"score":           f.Score,
 				}
 
 				if err := existing.Updates(values); err != nil {
@@ -916,6 +926,8 @@ func (m *File) AddFace(f face.Face, subjUid string) {
 
 			existing.SetEmbeddings(f.Embeddings, f.EmbedModel, f.DetectModel)
 			existing.LandmarksJSON = landmarks
+			existing.ThumbSize = thumbSize
+			existing.Score = f.Score
 		}
 
 		return

internal/entity/file_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/photoprism/photoprism/internal/ai/face"
 	"github.com/photoprism/photoprism/internal/config/customize"
+	"github.com/photoprism/photoprism/internal/thumb/crop"
 	"github.com/photoprism/photoprism/pkg/clean"
 	"github.com/photoprism/photoprism/pkg/fs"
 	"github.com/photoprism/photoprism/pkg/http/header"
@@ -689,6 +690,47 @@ func TestFile_AddFaces(t *testing.T) {
 	})
 }
 
+// TestFile_AddFaceUpgradesProvenance pins that upgrading an embedding-less marker carries the
+// detector's own numbers with the vector. Left behind, the size bar falls back to an XMP box extent
+// and the score bar is looked up by the newly written detect_model.
+func TestFile_AddFaceUpgradesProvenance(t *testing.T) {
+	file := &File{
+		FileUID:    "fs6sg6bw45bnlqdw",
+		FileHash:   "0e3d3e2e5b2f4b1a9a3d7c1e5f6a8b9c0d1e2f30",
+		FileWidth:  3000,
+		FileHeight: 2000,
+	}
+
+	// A sidecar region: a generous box, no embedding, and a score from no detector.
+	area := crop.Area{Name: "face", X: 0.4, Y: 0.4, W: 0.2, H: 0.2}
+	existing := NewMarker(*file, area, "", SrcXmp, MarkerFace, MarkerSize(area, *file), 10)
+	require.NotNil(t, existing)
+	require.NoError(t, existing.Create())
+	t.Cleanup(func() { UnscopedDb().Delete(existing) })
+
+	markers := Markers{*existing}
+	file.markers = &markers
+
+	f := face.Face{
+		Rows:       2000,
+		Cols:       3000,
+		Score:      88,
+		Area:       face.NewArea("face", 1000, 1500, 300),
+		Embeddings: face.Embeddings{face.RandomEmbedding()},
+		EmbedModel: face.EmbeddingModelName(),
+		ThumbSize:  97,
+	}
+
+	file.AddFace(f, "")
+
+	stored := &Marker{}
+	require.NoError(t, UnscopedDb().First(stored, "marker_uid = ?", existing.MarkerUID).Error)
+
+	assert.Equal(t, 97, stored.ThumbSize, "the extent the vector was sampled at must be recorded")
+	assert.Equal(t, 88, stored.Score, "the score must come from the detector that produced the vector")
+	assert.Equal(t, 97, (*file.Markers())[0].ThumbSize, "and the in-memory marker must match the row")
+}
+
 func TestFile_ValidFaceCount(t *testing.T) {
 	t.Run("FileFixturesExampleBridge", func(t *testing.T) {
 		file := FileFixturesExampleBridge

internal/entity/photo.go

@@ -28,6 +28,7 @@ import (
 )
 
 const (
+	// PhotoUID is the prefix that identifies a generated photo UID.
 	PhotoUID = byte('p')
 
 	// UUIDBytes is the byte budget for the photos.uuid column (VARBINARY(255)). A
@@ -36,9 +37,13 @@ const (
 	UUIDBytes = 255
 )
 
-var IndexUpdateInterval = 3 * time.Hour           // 3 Hours
-var MetadataUpdateInterval = 24 * 3 * time.Hour   // 3 Days
-var MetadataEstimateInterval = 24 * 7 * time.Hour // 7 Days
+// How long a photo may go without being re-indexed, having its metadata refreshed, or having an
+// estimate recomputed.
+var (
+	IndexUpdateInterval      = 3 * time.Hour
+	MetadataUpdateInterval   = 24 * 3 * time.Hour
+	MetadataEstimateInterval = 24 * 7 * time.Hour
+)
 
 var photoMutex = sync.Mutex{}
 var labelKeywordsSkipSrc = []string{SrcTitle, SrcCaption, SrcSubject, SrcKeyword}
@@ -316,11 +321,7 @@ func (m *Photo) Create() error {
 		return err
 	}
 
-	if err := m.SaveDetails(); err != nil {
-		return err
-	}
-
-	return nil
+	return m.SaveDetails()
 }
 
 // Save writes Photo changes, creates missing rows, and re-resolves the primary file relationship.
@@ -628,11 +629,7 @@ func (m *Photo) UpdateLabels() error {
 		return err
 	}
 
-	if err := m.UpdateKeywordLabels(); err != nil {
-		return err
-	}
-
-	return nil
+	return m.UpdateKeywordLabels()
 }
 
 // SubjectNames returns all known subject names.

internal/entity/photo_quality.go

@@ -7,6 +7,7 @@ import (
 	"github.com/photoprism/photoprism/pkg/txt"
 )
 
+// NonPhotographicKeywords names the keywords that mark a picture as not a photograph.
 var NonPhotographicKeywords = map[string]bool{
 	"screenshot":  true,
 	"screenshots": true,

internal/entity/query/covers.go

@@ -498,6 +498,9 @@ func UpdateSubjectCovers(public bool) (err error) {
 	// One statement for both dialects, so a library cannot get a different cover per backend.
 	// The uid guard keeps an empty subject uid from correlating against every unassigned marker,
 	// and a person left without an eligible marker gets no cover rather than a null one.
+	//
+	// Ranked on size rather than thumb_size: a cover wants the face that fills most of the frame,
+	// which a fixed-size detection thumbnail measures, not the one with the most available detail.
 	res := Db().Exec(`UPDATE subjects SET thumb = COALESCE((
 		SELECT m.thumb FROM markers m
 			JOIN files f ON f.file_uid = m.file_uid AND f.deleted_at IS NULL
@@ -526,13 +529,11 @@ func UpdateSubjectCovers(public bool) (err error) {
 	return err
 }
 
-// UpdateCoversAsync runs UpdateCovers in a goroutine and logs the
-// returned error, if any, as a warning. The launched goroutine is
-// registered with the shared entity package WaitGroup so config.CloseDb
-// can drain in-flight work via entity.WaitForAsyncJobs before tearing
-// down the database connection. A deferred recover guards against any
-// future shutdown race producing a process-killing panic instead of a
-// clean log line.
+// UpdateCoversAsync runs UpdateCovers in a goroutine and logs a returned error as a warning.
+//
+// Registered with the shared WaitGroup so config.CloseDb can drain in-flight work through
+// entity.WaitForAsyncJobs before closing the connection, and a deferred recover keeps a shutdown
+// race from killing the process.
 func UpdateCoversAsync() {
 	entity.AsyncJobAdd()
 	go func() {

internal/entity/search/faces.go

@@ -41,6 +41,10 @@ func representativeMarkerJoin(facesTable, unknown string) (string, []any) {
 		conds = append(conds, "m2.subj_uid <> ''")
 	}
 
+	// Filtered on the sampled extent but ranked on the detection size, which is deliberate: the two
+	// answer different questions. thumb_size is available detail, which decides whether a vector can
+	// be trusted; size is the face's extent in a fixed-size thumbnail, so it tracks how much of the
+	// frame the face fills, which is what picks a picture to represent a person.
 	return fmt.Sprintf(`JOIN markers m ON m.marker_uid = (
 		SELECT m2.marker_uid FROM markers m2
 		WHERE %s

internal/photoprism/index_faces.go

@@ -22,26 +22,19 @@ func DetectFaces(jpeg *MediaFile, expected int) (face.Faces, error) {
 
 	start := time.Now()
 
-	engineName := face.ActiveEngineName()
-
-	var thumbSize thumb.Name
-
-	if engineName == face.EngineONNX || Config().ThumbSizePrecached() < 1280 {
-		thumbSize = thumb.Fit720
-	} else {
-		thumbSize = thumb.Fit1280
-	}
-
-	thumbName, err := jpeg.Thumbnail(Config().ThumbCachePath(), thumbSize)
+	// Always Fit720, which entity.ClusterSizeCond depends on: markers.size is recorded in the
+	// pixels of whatever image detection ran on, and it is a lower bound on the extent an
+	// embedding was sampled from only while that image is the narrowest rendition.
+	thumbName, err := jpeg.Thumbnail(Config().ThumbCachePath(), thumb.Fit720)
 
 	if err != nil {
 		log.Debugf("%s (detect faces)", err)
 		return face.Faces{}, err
 	}
 
 	if thumbName == "" {
-		log.Debugf("vision: thumb %s not found in %s (detect faces)", thumbSize, clean.Log(jpeg.BaseName()))
-		return face.Faces{}, fmt.Errorf("thumbnail %s not found", thumbSize)
+		log.Debugf("vision: thumb %s not found in %s (detect faces)", thumb.Fit720, clean.Log(jpeg.BaseName()))
+		return face.Faces{}, fmt.Errorf("thumbnail %s not found", thumb.Fit720)
 	}
 
 	faces, err := vision.DetectFaces(thumbName, Config().FaceSize(), Config().FaceSizeRetry(), true, expected)