fix(share): gzip media snapshots

Author: vincentkoc

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Changes
 
+- Media-enabled `discrawl publish` now migrates shared attachment media to gzip-compressed files while still importing older raw-media snapshots.
+
 ### Fixes
 
 ## 0.8.0 - 2026-05-15

README.md

@@ -530,7 +530,7 @@ Once `share.remote` is configured, read commands auto-fetch and import when the
 
 Hybrid mode is supported too: keep normal Discord credentials configured and set `share.remote`. `discrawl sync --update=auto` and `discrawl messages --sync` import the Git snapshot first, usually as a changed-shard delta, then use live Discord for latest-message deltas. Use `sync --all-channels` or `sync --full` when you intentionally want a broader live repair/backfill pass.
 
-Git snapshots publish non-DM archive tables and cached non-DM attachment media by default. DMs, desktop wiretap rows, DM media, and local secrets are never exported. Use `publish --no-media` to omit cached media files.
+Git snapshots publish non-DM archive tables and cached non-DM attachment media by default. Cached media is written as gzip-compressed files under `media/` and restored to raw local cache files on import. Older snapshots that contain raw media files still import, and the next media-enabled `publish` rewrites the media tree into gzip form. DMs, desktop wiretap rows, DM media, and local secrets are never exported. Use `publish --no-media` to omit cached media files.
 Subscribers can use `subscribe --no-media` or `update --no-media` to import only SQLite rows and skip restoring cached files.
 
 Media backup is a two-step publisher workflow: first fetch bytes with `discrawl sync --with-media` or `discrawl attachments fetch`, then publish with `discrawl publish --push`. Scheduled publishers that should include media can set `sync.attachment_media = true` and leave `share.media = true`, which is the default. `publish` never downloads missing Discord files by itself; it exports only media already present in the local cache.

docs/README.md

@@ -7,7 +7,7 @@ Mirror Discord guilds into local SQLite. Search server history without depending
 - discovers every guild a bot can access and syncs channels, threads, members, and message history into SQLite
 - maintains FTS5 indexes for fast literal search; optional embeddings for semantic and hybrid recall
 - imports classifiable Discord Desktop cache messages with `wiretap`, including proven DMs under `@me`
-- downloads attachment media into the local cache and includes cached non-DM media in Git snapshots
+- downloads attachment media into the local cache and includes cached non-DM media as gzip-compressed Git snapshot files
 - tails the Gateway for live updates with periodic repair sweeps
 - publishes the archive as sharded NDJSON snapshots in a private Git repo so readers can search offline with no Discord credentials
 - exposes read-only SQL, channel/member directories, mention queries, digests, and trend analytics

docs/commands/attachments.md

@@ -38,8 +38,8 @@ discrawl --json attachments --missing --all
 - media bytes are stored under `cache_dir/media`, not in SQLite
 - SQLite stores attachment metadata, content hash, cached media path, fetch status, and errors
 - Discord CDN URLs can expire or be removed; those fetches are recorded as failed with their HTTP status, commonly `404`
-- `attachments fetch` only populates the local cache; run `publish --push` afterward to copy cached non-DM media into the Git snapshot repo
-- `publish` backs up cached non-DM media files by default; use `publish --no-media` to omit them
+- `attachments fetch` only populates the local cache; run `publish --push` afterward to copy cached non-DM media into the Git snapshot repo as gzip-compressed files
+- `publish` backs up cached non-DM media files by default and migrates older raw snapshot media to gzip form; use `publish --no-media` to omit them
 - `@me` DM media is local-only and is not published to Git snapshots
 
 ## See also

docs/commands/publish.md

@@ -60,7 +60,7 @@ README files without Discrawl report markers are left alone.
 ## What is published
 
 - non-DM archive tables (DM `@me` rows are always excluded)
-- cached non-DM attachment media files under `media/` unless `--no-media` is used
+- cached non-DM attachment media files under `media/` as gzip-compressed files unless `--no-media` is used
 - when filters are enabled: only matching guilds, channels, messages, events,
   attachments, mentions, channel-scoped sync-state rows, member rows referenced
   by matching messages, and matching embedding rows
@@ -83,6 +83,10 @@ fetch` before publishing when the Git snapshot should include newly discovered
 media. Scheduled publishers can set `sync.attachment_media = true` and leave
 `share.media = true`, the default.
 
+Media snapshots are gzip-only on publish. Legacy snapshots that stored raw
+`media/...` files remain importable for backward compatibility, and the next
+media publish rewrites the media tree as `media/...gz`.
+
 ## See also
 
 - [Git snapshots guide](../guides/git-snapshots.html)

docs/commands/sync.md

@@ -68,7 +68,7 @@ discrawl sync --with-media
 
 - `--latest-only` is the default for untargeted `sync`. Use `--all-channels` to opt out without doing a full historical crawl.
 - `--with-media` records expired or removed Discord CDN URLs as failed fetches with the HTTP status, commonly `404`.
-- `--with-media` updates the local cache only; run `publish --push` afterward to include cached non-DM media in the Git backup.
+- `--with-media` updates the local cache only; run `publish --push` afterward to include cached non-DM media in the Git backup as gzip-compressed files.
 - `--since` does not mark older history as complete, so a later `sync --full` without `--since` can continue the backfill.
 - Long runs emit periodic progress logs to stderr.
 - Heartbeat logs (`message sync waiting`) name the oldest active channel and per-channel page activity if in-flight channels stop completing for a while.

docs/configuration.md

@@ -122,7 +122,7 @@ Set `discord.token_source = "keyring"` if you want to require keyring lookup and
 - changing `[search.embeddings]` provider/model/input version retargets pending jobs and resets prior attempts; existing vectors for another identity remain in SQLite but are not used for semantic search
 - changing `db_path` does not migrate existing data; copy the file yourself if you want to keep history
 - `sync.attachment_media = true` makes `sync` behave like `sync --with-media`; media bytes are cached under `cache_dir/media`, and CDN `404`/other fetch failures are recorded on attachment rows
-- `share.media = false` makes publish/update/auto-update omit or skip restoring cached media; `subscribe --no-media` writes this for Git-only readers. With the default `share.media = true`, publish/update include cached non-DM media, but publish does not fetch missing Discord files by itself.
+- `share.media = false` makes publish/update/auto-update omit or skip restoring cached media; `subscribe --no-media` writes this for Git-only readers. With the default `share.media = true`, publish/update include cached non-DM media as gzip-compressed snapshot files, but publish does not fetch missing Discord files by itself.
 - `[share.filter]` narrows only `publish` output; sync can still keep a richer local archive
 - `share.filter.public_only` exports only channels visible to the guild
   `@everyone` role after category/channel permission overwrites; private

docs/guides/git-snapshots.md

@@ -80,8 +80,8 @@ discrawl sync --full              # historical backfill
 ## What is published
 
 - non-DM archive tables (DM `@me` rows are always excluded)
-- cached non-DM attachment media by default; use `publish --no-media` to omit
-  files that are already in `cache_dir/media`
+- cached non-DM attachment media as gzip-compressed files by default; use
+  `publish --no-media` to omit files that are already in `cache_dir/media`
 - with publish filters: only matching channel-scoped rows, matching embedding
   rows, and member rows referenced by matching messages
 - with publish filters: no share manifest state and no guild-level member
@@ -101,10 +101,14 @@ discrawl publish --push
 
 `sync --with-media` and `attachments fetch` download Discord attachment bytes
 into `cache_dir/media`. `publish --push` then exports cached non-DM media into
-the Git snapshot repo. `publish` does not fetch missing Discord files itself,
-so scheduled Git backups that should include media must fetch media before
-publishing. Set `sync.attachment_media = true` for scheduled sync jobs and leave
-`share.media = true` to include cached media in publish/update flows.
+the Git snapshot repo as gzip-compressed `media/...gz` files. Imports restore
+those files back into the raw local cache layout. Older snapshots that contain
+raw `media/...` files still import; the next media publish clears the legacy
+media tree and rewrites it in gzip form. `publish` does not fetch missing
+Discord files itself, so scheduled Git backups that should include media must
+fetch media before publishing. Set `sync.attachment_media = true` for scheduled
+sync jobs and leave `share.media = true` to include cached media in
+publish/update flows.
 
 Discord CDN URLs can expire or be removed. Those fetches are stored as failed
 with their HTTP status, commonly `404`; this does not block publishing files

docs/security.md

@@ -42,7 +42,7 @@ Attachment binaries are not stored in SQLite. Only attachment metadata, optional
 
 Set `sync.attachment_text = false` if you want to keep attachment metadata and filenames but disable attachment body fetches for text indexing.
 
-Git snapshots include cached non-DM media files by default. Use `publish --no-media` to omit them. `publish` exports only files already in the local cache; it does not fetch missing Discord media. DM media under `@me` stays local-only.
+Git snapshots include cached non-DM media files by default. Use `publish --no-media` to omit them. `publish` exports only files already in the local cache; it does not fetch missing Discord media. Published media is gzip-compressed under `media/`, while import still accepts older raw-media snapshots for backward compatibility. DM media under `@me` stays local-only.
 
 ## What is sent over the wire
 

internal/share/share.go

@@ -883,8 +883,8 @@ func exportMedia(ctx context.Context, db *sql.DB, opts Options, filter *snapshot
 	if strings.TrimSpace(opts.CacheDir) == "" {
 		return nil, nil
 	}
-	if err := os.RemoveAll(filepath.Join(opts.RepoPath, "media")); err != nil {
-		return nil, fmt.Errorf("reset media dir: %w", err)
+	if err := resetCompressedMediaExport(opts.RepoPath); err != nil {
+		return nil, err
 	}
 	rows, err := db.QueryContext(ctx, `
 		select attachment_id, message_id, guild_id, channel_id, coalesce(media_path, ''), coalesce(content_sha256, '')
@@ -934,22 +934,30 @@ func exportMedia(ctx context.Context, db *sql.DB, opts Options, filter *snapshot
 		}
 		manifest.Attachments++
 		seen[mediaPath] = struct{}{}
-		rel := filepath.ToSlash(filepath.Join("media", mediaPath))
-		target, err := media.RepoPath(opts.RepoPath, mediaPath)
+		rel := compressedMediaManifestPath(mediaPath)
+		target, err := compressedMediaRepoPath(opts.RepoPath, mediaPath)
 		if err != nil {
 			return nil, err
 		}
-		if err := copyFile(target, source); err != nil {
-			return nil, fmt.Errorf("copy media %s: %w", mediaPath, err)
+		if err := copyGzipFile(target, source); err != nil {
+			return nil, fmt.Errorf("compress media %s: %w", mediaPath, err)
 		}
 		hash, err := fileSHA256(target)
 		if err != nil {
 			return nil, err
 		}
-		if expectedHash != "" && hash != expectedHash {
-			return nil, fmt.Errorf("media hash mismatch for %s: got %s want %s", mediaPath, hash, expectedHash)
+		sourceHash, err := fileSHA256(source)
+		if err != nil {
+			return nil, err
+		}
+		if expectedHash != "" && sourceHash != expectedHash {
+			return nil, fmt.Errorf("media hash mismatch for %s: got %s want %s", mediaPath, sourceHash, expectedHash)
 		}
-		manifest.Files = append(manifest.Files, snapshot.FileManifest{Path: rel, Size: info.Size(), SHA256: hash})
+		compressedInfo, err := os.Stat(target)
+		if err != nil {
+			return nil, fmt.Errorf("stat compressed media %s: %w", rel, err)
+		}
+		manifest.Files = append(manifest.Files, snapshot.FileManifest{Path: rel, Size: compressedInfo.Size(), SHA256: hash})
 		manifest.Bytes += info.Size()
 	}
 	if err := rows.Err(); err != nil {
@@ -961,6 +969,15 @@ func exportMedia(ctx context.Context, db *sql.DB, opts Options, filter *snapshot
 	return manifest, nil
 }
 
+func resetCompressedMediaExport(repoPath string) error {
+	// A publish rewrites media from the local cache. Clearing the tree here is
+	// the forward migration from legacy raw media files to gzip-only snapshots.
+	if err := os.RemoveAll(filepath.Join(repoPath, "media")); err != nil {
+		return fmt.Errorf("reset media dir: %w", err)
+	}
+	return nil
+}
+
 func validateMediaRoots(opts Options) error {
 	if strings.TrimSpace(opts.CacheDir) == "" {
 		return nil
@@ -1030,11 +1047,11 @@ func importMedia(ctx context.Context, opts Options, manifest *MediaManifest) (in
 		if err := ctx.Err(); err != nil {
 			return copied, err
 		}
-		mediaPath, ok := strings.CutPrefix(filepath.ToSlash(file.Path), "media/")
+		mediaPath, compressed, ok := mediaPathFromManifest(file.Path)
 		if !ok || strings.TrimSpace(mediaPath) == "" {
 			return copied, fmt.Errorf("invalid media manifest path %q", file.Path)
 		}
-		source, err := media.RepoPath(opts.RepoPath, mediaPath)
+		source, err := mediaSourcePath(opts.RepoPath, mediaPath, compressed)
 		if err != nil {
 			return copied, err
 		}
@@ -1059,17 +1076,55 @@ func importMedia(ctx context.Context, opts Options, manifest *MediaManifest) (in
 		if err != nil {
 			return copied, err
 		}
-		if sameFileHash(target, hash) {
+		targetHash := hash
+		if compressed {
+			targetHash, err = gzipFileSHA256(source)
+			if err != nil {
+				return copied, fmt.Errorf("hash compressed media %s: %w", file.Path, err)
+			}
+		}
+		if sameFileHash(target, targetHash) {
 			continue
 		}
-		if err := copyFile(target, source); err != nil {
+		if compressed {
+			err = restoreGzipFile(target, source)
+		} else {
+			err = copyFile(target, source)
+		}
+		if err != nil {
 			return copied, fmt.Errorf("restore media %s: %w", file.Path, err)
 		}
 		copied++
 	}
 	return copied, nil
 }
 
+func compressedMediaManifestPath(mediaPath string) string {
+	return filepath.ToSlash(filepath.Join("media", mediaPath+".gz"))
+}
+
+func mediaPathFromManifest(path string) (string, bool, bool) {
+	mediaPath, ok := strings.CutPrefix(filepath.ToSlash(path), "media/")
+	if !ok {
+		return "", false, false
+	}
+	if strings.HasSuffix(mediaPath, ".gz") {
+		return strings.TrimSuffix(mediaPath, ".gz"), true, true
+	}
+	return mediaPath, false, true
+}
+
+func compressedMediaRepoPath(repoPath, mediaPath string) (string, error) {
+	return media.RepoPath(repoPath, mediaPath+".gz")
+}
+
+func mediaSourcePath(repoPath, mediaPath string, compressed bool) (string, error) {
+	if compressed {
+		return compressedMediaRepoPath(repoPath, mediaPath)
+	}
+	return media.RepoPath(repoPath, mediaPath)
+}
+
 func regularMediaFile(root, path, label string) (os.FileInfo, error) {
 	root = filepath.Clean(root)
 	path = filepath.Clean(path)
@@ -1145,6 +1200,83 @@ func copyFile(target, source string) error {
 	return nil
 }
 
+func copyGzipFile(target, source string) error {
+	if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+		return err
+	}
+	src, err := os.Open(source) // #nosec G304 -- source is constrained by media path helpers.
+	if err != nil {
+		return err
+	}
+	defer func() { _ = src.Close() }()
+	tmp, err := os.CreateTemp(filepath.Dir(target), ".copy-*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	gz, err := gzip.NewWriterLevel(tmp, gzip.BestCompression)
+	if err != nil {
+		_ = tmp.Close()
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	if _, err := io.Copy(gz, src); err != nil {
+		_ = gz.Close()
+		_ = tmp.Close()
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	if err := gz.Close(); err != nil {
+		_ = tmp.Close()
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	if err := os.Rename(tmpPath, target); err != nil {
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	return nil
+}
+
+func restoreGzipFile(target, source string) error {
+	if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+		return err
+	}
+	src, err := os.Open(source) // #nosec G304 -- source is constrained by media path helpers.
+	if err != nil {
+		return err
+	}
+	defer func() { _ = src.Close() }()
+	gz, err := gzip.NewReader(src)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = gz.Close() }()
+	tmp, err := os.CreateTemp(filepath.Dir(target), ".copy-*")
+	if err != nil {
+		return err
+	}
+	tmpPath := tmp.Name()
+	if _, err := io.Copy(tmp, gz); err != nil {
+		_ = tmp.Close()
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	if err := os.Rename(tmpPath, target); err != nil {
+		_ = os.Remove(tmpPath)
+		return err
+	}
+	return nil
+}
+
 func fileSHA256(path string) (string, error) {
 	file, err := os.Open(path) // #nosec G304 -- callers pass confined repo/cache paths.
 	if err != nil {
@@ -1158,6 +1290,24 @@ func fileSHA256(path string) (string, error) {
 	return hex.EncodeToString(hasher.Sum(nil)), nil
 }
 
+func gzipFileSHA256(path string) (string, error) {
+	file, err := os.Open(path) // #nosec G304 -- callers pass confined repo/cache paths.
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = file.Close() }()
+	gz, err := gzip.NewReader(file)
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = gz.Close() }()
+	hasher := sha256.New()
+	if _, err := io.Copy(hasher, gz); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
 func sameFileHash(path, hash string) bool {
 	current, err := fileSHA256(path)
 	return err == nil && current == hash