Request exif and people metadata for sidecars

The sidecar is the server's asset response verbatim, and exifInfo/people
are optional fields on that response: Immich omits exifInfo entirely and
leaves people empty unless the search asks for them. We never asked, so
every sidecar we have ever written is missing capture date, camera make
and model, GPS coordinates, rating and faces.

Set withExif and withPeople on the metadata search.

Assets already on disk are skipped before the sidecar write, so this fix
alone would only ever reach newly downloaded files and existing archives
would keep their thin sidecars forever. Add --refresh-sidecars, which
rewrites the sidecar of an asset already present without re-downloading
the original.

Album membership, also reported in #9, is not addressed here: Immich's
asset response has no album field in any form, so no flag can supply it.
Split out to #11.

Refs #9

Author: jochbru

README.md

@@ -19,6 +19,11 @@ immich-archiver --url https://photos.example.com --api-key your-user-api-key --d
 On a second run, assets already present on disk (verified by filename + a matching asset ID in
 the sidecar) are skipped, so re-running is cheap.
 
+Because skipped assets never reach the write path, their sidecars are left untouched — an
+archive keeps whatever sidecars it was originally built with. To rewrite them against the
+current server metadata without re-downloading a single original, run with
+`--refresh-sidecars`.
+
 ### Layout
 
 By default, assets land in `<dir>/{year}/{year}-{month}/`, e.g.:
@@ -47,6 +52,7 @@ Live Photos are downloaded as a still + a paired motion video sharing the same b
 | `--concurrency` | `4` | parallel downloads |
 | `--retries` | `3` | retry attempts on network/server errors |
 | `--dry-run` | `false` | preview without writing |
+| `--refresh-sidecars` | `false` | rewrite sidecars of assets already on disk (originals are not re-downloaded) |
 | `--verbose` / `-v` | `false` | log one line per asset instead of a progress summary |
 
 ## Development

cmd/progress.go

@@ -15,15 +15,17 @@ type progress struct {
 	out     io.Writer
 	verbose bool
 	dryRun  bool
+	refresh bool // report the refreshed count; suppressed when nothing can refresh
 	mu      sync.Mutex
 
 	downloaded int64
 	skipped    int64
+	refreshed  int64
 	failed     int64
 }
 
-func newProgress(out io.Writer, verbose, dryRun bool) *progress {
-	return &progress{out: out, verbose: verbose, dryRun: dryRun}
+func newProgress(out io.Writer, verbose, dryRun, refresh bool) *progress {
+	return &progress{out: out, verbose: verbose, dryRun: dryRun, refresh: refresh}
 }
 
 func (p *progress) report(e archive.Event) {
@@ -32,6 +34,8 @@ func (p *progress) report(e archive.Event) {
 		atomic.AddInt64(&p.downloaded, 1)
 	case archive.ActionSkipped:
 		atomic.AddInt64(&p.skipped, 1)
+	case archive.ActionRefreshed:
+		atomic.AddInt64(&p.refreshed, 1)
 	case archive.ActionFailed:
 		atomic.AddInt64(&p.failed, 1)
 	}
@@ -49,14 +53,30 @@ func (p *progress) report(e archive.Event) {
 			_, _ = fmt.Fprintf(p.out, "%s %s\n", verb, e.Filename)
 		case archive.ActionSkipped:
 			_, _ = fmt.Fprintf(p.out, "skipped (already present) %s\n", e.Filename)
+		case archive.ActionRefreshed:
+			refreshVerb := "refreshed sidecar"
+			if p.dryRun {
+				refreshVerb = "would refresh sidecar"
+			}
+			_, _ = fmt.Fprintf(p.out, "%s %s\n", refreshVerb, e.Filename)
 		case archive.ActionFailed:
 			_, _ = fmt.Fprintf(p.out, "FAILED %s: %v\n", e.Filename, e.Err)
 		}
 		return
 	}
 
-	_, _ = fmt.Fprintf(p.out, "\rdownloaded %d, skipped %d, failed %d",
-		atomic.LoadInt64(&p.downloaded), atomic.LoadInt64(&p.skipped), atomic.LoadInt64(&p.failed))
+	_, _ = fmt.Fprintf(p.out, "\rdownloaded %d, skipped %d%s, failed %d",
+		atomic.LoadInt64(&p.downloaded), atomic.LoadInt64(&p.skipped),
+		p.refreshedSegment(atomic.LoadInt64(&p.refreshed)), atomic.LoadInt64(&p.failed))
+}
+
+// refreshedSegment renders ", refreshed N" only when sidecar refreshing is
+// enabled, so the normal status line keeps its existing shape.
+func (p *progress) refreshedSegment(n int64) string {
+	if !p.refresh {
+		return ""
+	}
+	return fmt.Sprintf(", refreshed %d", n)
 }
 
 func (p *progress) finish(stats archive.Stats) {
@@ -67,5 +87,6 @@ func (p *progress) finish(stats archive.Stats) {
 	if p.dryRun {
 		verb = "Would download"
 	}
-	_, _ = fmt.Fprintf(p.out, "%s: %d, skipped: %d, failed: %d\n", verb, stats.Downloaded, stats.Skipped, stats.Failed)
+	_, _ = fmt.Fprintf(p.out, "%s: %d, skipped: %d%s, failed: %d\n",
+		verb, stats.Downloaded, stats.Skipped, p.refreshedSegment(int64(stats.Refreshed)), stats.Failed)
 }

cmd/root.go

@@ -26,6 +26,7 @@ type flags struct {
 	concurrency        int
 	retries            int
 	dryRun             bool
+	refreshSidecars    bool
 	verbose            bool
 }
 
@@ -61,6 +62,7 @@ func newRootCmd() *cobra.Command {
 	cmd.Flags().IntVar(&f.concurrency, "concurrency", 4, "number of assets to download in parallel")
 	cmd.Flags().IntVar(&f.retries, "retries", 3, "number of retry attempts for failed downloads/requests")
 	cmd.Flags().BoolVar(&f.dryRun, "dry-run", false, "list what would be downloaded without writing anything")
+	cmd.Flags().BoolVar(&f.refreshSidecars, "refresh-sidecars", false, "rewrite the .json sidecar of assets already on disk (originals are not re-downloaded)")
 	cmd.Flags().BoolVarP(&f.verbose, "verbose", "v", false, "log a line per asset instead of showing a progress bar")
 
 	return cmd
@@ -103,7 +105,7 @@ func runSync(cmd *cobra.Command, f *flags) error {
 		sharedDir = f.dir + string(os.PathSeparator) + "shared-with-me"
 	}
 
-	p := newProgress(cmd.OutOrStdout(), f.verbose, f.dryRun)
+	p := newProgress(cmd.OutOrStdout(), f.verbose, f.dryRun, f.refreshSidecars)
 	s := &archive.Syncer{
 		Source: client,
 		Options: archive.Options{
@@ -116,6 +118,7 @@ func runSync(cmd *cobra.Command, f *flags) error {
 			Retries:            f.retries,
 			RetryDelay:         2 * time.Second,
 			DryRun:             f.dryRun,
+			RefreshSidecars:    f.refreshSidecars,
 		},
 		Reporter: p.report,
 	}

internal/archive/sync.go

@@ -33,6 +33,13 @@ type Options struct {
 	Retries            int
 	RetryDelay         time.Duration
 	DryRun             bool
+
+	// RefreshSidecars rewrites the sidecar of an asset already on disk
+	// instead of skipping it outright. Without this, an archive built by an
+	// older version keeps its sidecars forever: assets present on disk never
+	// reach the write path, so a metadata fix only ever reaches newly
+	// downloaded files. Original files are still never re-downloaded.
+	RefreshSidecars bool
 }
 
 const unknownDateDir = "unknown-date"
@@ -45,6 +52,7 @@ const (
 	ActionDownloaded Action = "downloaded"
 	ActionSkipped    Action = "skipped"     // already present on disk
 	ActionWouldFetch Action = "would-fetch" // dry-run
+	ActionRefreshed  Action = "refreshed"   // file kept, sidecar rewritten
 	ActionFailed     Action = "failed"
 )
 
@@ -65,6 +73,7 @@ type Reporter func(Event)
 type Stats struct {
 	Downloaded int
 	Skipped    int
+	Refreshed  int
 	Failed     int
 }
 
@@ -120,6 +129,8 @@ func (s *Syncer) Run(ctx context.Context) (Stats, error) {
 			stats.Downloaded++
 		case ActionSkipped:
 			stats.Skipped++
+		case ActionRefreshed:
+			stats.Refreshed++
 		case ActionFailed:
 			stats.Failed++
 		}
@@ -147,7 +158,12 @@ func (s *Syncer) Run(ctx context.Context) (Stats, error) {
 		}()
 	}
 
-	err := s.Source.SearchAssets(ctx, immich.SearchMetadataQuery{WithDeleted: false, IsArchived: nil}, func(a *immich.Asset) error {
+	err := s.Source.SearchAssets(ctx, immich.SearchMetadataQuery{
+		WithDeleted: false,
+		IsArchived:  nil,
+		WithExif:    true,
+		WithPeople:  true,
+	}, func(a *immich.Asset) error {
 		if a.IsTrashed {
 			return nil
 		}
@@ -238,7 +254,19 @@ func (s *Syncer) downloadOne(ctx context.Context, a *immich.Asset, dir, desiredN
 		return
 	}
 	if exists {
-		report(Event{AssetID: a.ID, Filename: filename, Action: ActionSkipped})
+		if !s.Options.RefreshSidecars {
+			report(Event{AssetID: a.ID, Filename: filename, Action: ActionSkipped})
+			return
+		}
+		if s.Options.DryRun {
+			report(Event{AssetID: a.ID, Filename: filename, Action: ActionRefreshed})
+			return
+		}
+		if err := WriteSidecar(dir, filename, a.RawJSON); err != nil {
+			report(Event{AssetID: a.ID, Filename: filename, Action: ActionFailed, Err: err})
+			return
+		}
+		report(Event{AssetID: a.ID, Filename: filename, Action: ActionRefreshed})
 		return
 	}
 	if s.Options.DryRun {

internal/archive/sync_test.go

@@ -21,6 +21,8 @@ type fakeSource struct {
 	files     map[string]string // asset id -> file content
 	albums    []immich.Album
 	albumAsts map[string][]*immich.Asset
+
+	lastQuery immich.SearchMetadataQuery
 }
 
 func newFakeSource() *fakeSource {
@@ -59,6 +61,7 @@ func (f *fakeSource) add(a *immich.Asset, content string) {
 }
 
 func (f *fakeSource) SearchAssets(ctx context.Context, query immich.SearchMetadataQuery, fn func(*immich.Asset) error) error {
+	f.lastQuery = query
 	for _, a := range f.assets {
 		if err := fn(a); err != nil {
 			return err
@@ -138,6 +141,117 @@ func TestSyncerDownloadsAndWritesSidecar(t *testing.T) {
 	}
 }
 
+// Immich omits exifInfo and leaves people empty unless the search asks for
+// them, and the sidecar is that response verbatim — so forgetting these flags
+// silently strips capture date, camera, GPS, rating and faces from every
+// sidecar written.
+func TestSyncerRequestsExifAndPeople(t *testing.T) {
+	dir := t.TempDir()
+	src := newFakeSource()
+	src.add(mustAsset(t, "a1", "IMG_0001.jpg", "2005-06-15T10:00:00.000Z", nil), "photo-bytes")
+
+	s := &Syncer{Source: src, Options: baseOptions(dir)}
+	if _, err := s.Run(context.Background()); err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if !src.lastQuery.WithExif {
+		t.Error("search query did not set withExif")
+	}
+	if !src.lastQuery.WithPeople {
+		t.Error("search query did not set withPeople")
+	}
+}
+
+func TestSyncerRefreshSidecarsRewritesExistingSidecar(t *testing.T) {
+	dir := t.TempDir()
+	src := newFakeSource()
+	src.add(mustAsset(t, "a1", "IMG_0001.jpg", "2005-06-15T10:00:00.000Z", map[string]any{
+		"exifInfo": map[string]any{"make": "Canon"},
+	}), "photo-bytes")
+
+	// Simulate an archive written by an older version: original in place,
+	// sidecar missing the metadata.
+	assetDir := filepath.Join(dir, "2005", "2005-06")
+	if err := os.MkdirAll(assetDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	target := filepath.Join(assetDir, "IMG_0001.jpg")
+	if err := os.WriteFile(target, []byte("photo-bytes"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(target+".json", []byte(`{"id":"a1"}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	opts := baseOptions(dir)
+	opts.RefreshSidecars = true
+	s := &Syncer{Source: src, Options: opts}
+	stats, err := s.Run(context.Background())
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if stats.Refreshed != 1 || stats.Downloaded != 0 || stats.Skipped != 0 {
+		t.Fatalf("stats = %+v", stats)
+	}
+
+	sidecar, err := os.ReadFile(target + ".json")
+	if err != nil {
+		t.Fatalf("reading sidecar: %v", err)
+	}
+	var decoded map[string]any
+	if err := json.Unmarshal(sidecar, &decoded); err != nil {
+		t.Fatalf("sidecar not valid JSON: %s", sidecar)
+	}
+	if decoded["exifInfo"] == nil {
+		t.Fatalf("sidecar was not refreshed with exifInfo: %s", sidecar)
+	}
+
+	// The original must not be re-downloaded or otherwise disturbed.
+	data, err := os.ReadFile(target)
+	if err != nil || string(data) != "photo-bytes" {
+		t.Fatalf("original file changed: %q %v", data, err)
+	}
+}
+
+func TestSyncerRefreshSidecarsDryRunWritesNothing(t *testing.T) {
+	dir := t.TempDir()
+	src := newFakeSource()
+	src.add(mustAsset(t, "a1", "IMG_0001.jpg", "2005-06-15T10:00:00.000Z", map[string]any{
+		"exifInfo": map[string]any{"make": "Canon"},
+	}), "photo-bytes")
+
+	assetDir := filepath.Join(dir, "2005", "2005-06")
+	if err := os.MkdirAll(assetDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	target := filepath.Join(assetDir, "IMG_0001.jpg")
+	if err := os.WriteFile(target, []byte("photo-bytes"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(target+".json", []byte(`{"id":"a1"}`), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
+	opts := baseOptions(dir)
+	opts.RefreshSidecars = true
+	opts.DryRun = true
+	s := &Syncer{Source: src, Options: opts}
+	stats, err := s.Run(context.Background())
+	if err != nil {
+		t.Fatalf("Run: %v", err)
+	}
+	if stats.Refreshed != 1 {
+		t.Fatalf("stats = %+v", stats)
+	}
+	sidecar, err := os.ReadFile(target + ".json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(sidecar) != `{"id":"a1"}` {
+		t.Fatalf("dry run rewrote sidecar: %s", sidecar)
+	}
+}
+
 func TestSyncerSkipsAlreadyDownloaded(t *testing.T) {
 	dir := t.TempDir()
 	src := newFakeSource()

internal/immich/types.go

@@ -39,10 +39,17 @@ func (a *Asset) UnmarshalJSON(data []byte) error {
 }
 
 // SearchMetadataQuery is the request body for POST /search/metadata.
+//
+// WithExif and WithPeople matter more than they look: exifInfo and people are
+// optional fields on Immich's asset response, omitted (or left as an empty
+// array) unless explicitly requested. Since the sidecar is the server response
+// verbatim, leaving them unset silently strips capture date, camera make and
+// model, GPS coordinates, rating and faces from every sidecar we write.
 type SearchMetadataQuery struct {
 	Page        int    `json:"page"`
 	Size        int    `json:"size,omitempty"`
 	WithExif    bool   `json:"withExif,omitempty"`
+	WithPeople  bool   `json:"withPeople,omitempty"`
 	WithDeleted bool   `json:"withDeleted,omitempty"`
 	IsArchived  *bool  `json:"isArchived,omitempty"`
 	PersonalOwn bool   `json:"-"` // filtered client-side, Immich search has no such flag