add message-key deduplication mode

rusq · rusq · commit d0df57641530 · 2026-06-26T20:03:44.000+10:00
diff --git a/cmd/slackdump/internal/diag/dedupe.go b/cmd/slackdump/internal/diag/dedupe.go
@@ -11,6 +11,7 @@ import (
 	"github.com/rusq/slackdump/v4/cmd/slackdump/internal/cfg"
 	dedupecmd "github.com/rusq/slackdump/v4/cmd/slackdump/internal/diag/dedupe"
 	"github.com/rusq/slackdump/v4/cmd/slackdump/internal/golang/base"
+	"github.com/rusq/slackdump/v4/internal/chunk/backend/dbase/repository"
 	"github.com/rusq/slackdump/v4/source"
 )
 
@@ -22,9 +23,10 @@ var cmdDedupe = &base.Command{
 	Long: `
 Dedupe removes identical duplicate messages, users, channels, channel users,
 and files created by resume look-back overlap. The latest copy of each
-identical payload is kept. By default it only reports what would be removed.
-Pass the archive directory, not the slackdump.sqlite file. Use -execute to
-perform deduplication.
+identical payload is kept. Use -mode message-key to collapse message rows by
+channel and timestamp even when Slack-regenerated fields differ. By default it
+only reports what would be removed. Pass the archive directory, not the
+slackdump.sqlite file. Use -execute to perform deduplication.
 
 The same cleanup can be run automatically after a successful resume with:
 
@@ -34,11 +36,14 @@ The same cleanup can be run automatically after a successful resume with:
 
 var dedupeFlags struct {
 	execute bool
+	mode    repository.MessageDedupeMode
 }
 
 func init() {
 	cmdDedupe.Run = runDedupe
+	dedupeFlags.mode = repository.MessageDedupeExact
 	cmdDedupe.Flag.BoolVar(&dedupeFlags.execute, "execute", false, "actually remove duplicate entities")
+	cmdDedupe.Flag.Var(&dedupeFlags.mode, "mode", "message dedupe mode: exact or message-key")
 }
 
 func ensureDb(ctx context.Context, dir string) (*sqlx.DB, error) {
@@ -82,6 +87,7 @@ func runDedupe(ctx context.Context, cmd *base.Command, args []string) error {
 		Execute:  dedupeFlags.execute,
 		Report:   os.Stdout,
 		Database: dir,
+		Mode:     dedupeFlags.mode,
 	})
 	return err
 }
diff --git a/cmd/slackdump/internal/diag/dedupe/dedupe.go b/cmd/slackdump/internal/diag/dedupe/dedupe.go
@@ -15,17 +15,20 @@ type Options struct {
 	Execute  bool
 	Report   io.Writer
 	Database string
+	Mode     repository.MessageDedupeMode
 }
 
 type Result struct {
 	Counts  repository.DedupeCounts
 	Removed repository.DedupeResult
 }
 
-var newRepo = repository.NewDedupeRepository
+var newRepo = func(mode repository.MessageDedupeMode) repository.DedupeRepository {
+	return repository.NewDedupeRepository(repository.WithMessageDedupeMode(mode))
+}
 
 func Run(ctx context.Context, db *sqlx.DB, opts Options) (Result, error) {
-	repo := newRepo()
+	repo := newRepo(opts.Mode)
 
 	counts, err := repo.Preview(ctx, db)
 	if err != nil {
@@ -34,6 +37,7 @@ func Run(ctx context.Context, db *sqlx.DB, opts Options) (Result, error) {
 
 	slog.DebugContext(ctx, "dedupe preview",
 		"database", opts.Database,
+		"mode", opts.Mode.String(),
 		"duplicate_messages", counts.Messages,
 		"duplicate_users", counts.Users,
 		"duplicate_channels", counts.Channels,
@@ -68,6 +72,7 @@ func Run(ctx context.Context, db *sqlx.DB, opts Options) (Result, error) {
 
 	slog.InfoContext(ctx, "dedupe execute",
 		"database", opts.Database,
+		"mode", opts.Mode.String(),
 		"removed_messages", removed.MessagesRemoved,
 		"removed_users", removed.UsersRemoved,
 		"removed_channels", removed.ChannelsRemoved,
diff --git a/cmd/slackdump/internal/diag/dedupe/dedupe_test.go b/cmd/slackdump/internal/diag/dedupe/dedupe_test.go
@@ -32,7 +32,7 @@ func TestRun(t *testing.T) {
 
 	t.Run("preview only with report", func(t *testing.T) {
 		var called bool
-		newRepo = func() repository.DedupeRepository {
+		newRepo = func(repository.MessageDedupeMode) repository.DedupeRepository {
 			return stubRepo{
 				previewFn: func(context.Context, *sqlx.DB) (repository.DedupeCounts, error) {
 					return repository.DedupeCounts{Messages: 2, Chunks: 1}, nil
@@ -53,7 +53,7 @@ func TestRun(t *testing.T) {
 	})
 
 	t.Run("execute with report", func(t *testing.T) {
-		newRepo = func() repository.DedupeRepository {
+		newRepo = func(repository.MessageDedupeMode) repository.DedupeRepository {
 			return stubRepo{
 				previewFn: func(context.Context, *sqlx.DB) (repository.DedupeCounts, error) {
 					return repository.DedupeCounts{Messages: 1}, nil
@@ -71,7 +71,7 @@ func TestRun(t *testing.T) {
 	})
 
 	t.Run("log only mode does not write report", func(t *testing.T) {
-		newRepo = func() repository.DedupeRepository {
+		newRepo = func(repository.MessageDedupeMode) repository.DedupeRepository {
 			return stubRepo{
 				previewFn: func(context.Context, *sqlx.DB) (repository.DedupeCounts, error) {
 					return repository.DedupeCounts{}, nil
@@ -87,7 +87,7 @@ func TestRun(t *testing.T) {
 	})
 
 	t.Run("execute returns dedupe error", func(t *testing.T) {
-		newRepo = func() repository.DedupeRepository {
+		newRepo = func(repository.MessageDedupeMode) repository.DedupeRepository {
 			return stubRepo{
 				previewFn: func(context.Context, *sqlx.DB) (repository.DedupeCounts, error) {
 					return repository.DedupeCounts{}, nil
@@ -101,4 +101,22 @@ func TestRun(t *testing.T) {
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "deduplicate entities")
 	})
+
+	t.Run("passes mode to repository", func(t *testing.T) {
+		var got repository.MessageDedupeMode
+		newRepo = func(mode repository.MessageDedupeMode) repository.DedupeRepository {
+			got = mode
+			return stubRepo{
+				previewFn: func(context.Context, *sqlx.DB) (repository.DedupeCounts, error) {
+					return repository.DedupeCounts{}, nil
+				},
+				dedupeFn: func(context.Context, *sqlx.DB) (repository.DedupeResult, error) {
+					return repository.DedupeResult{}, nil
+				},
+			}
+		}
+		_, err := Run(t.Context(), nil, Options{Mode: repository.MessageDedupeKey})
+		require.NoError(t, err)
+		assert.Equal(t, repository.MessageDedupeKey, got)
+	})
 }
diff --git a/doc/usage-dedupe.md b/doc/usage-dedupe.md
@@ -1,7 +1,9 @@
 # Database Dedupe
 
 The `dedupe` tool removes identical duplicate messages, users, channels,
-channel users, and files created by resume look-back overlap.
+channel users, and files created by resume look-back overlap. It can also
+collapse messages by channel and timestamp for merged exports where Slack
+regenerated volatile message fields between exports.
 
 ## Why use dedupe?
 
@@ -18,6 +20,9 @@ slackdump tools dedupe /path/to/archive
 
 # Actually perform dedupe
 slackdump tools dedupe -execute /path/to/archive
+
+# Collapse message rows by channel and timestamp
+slackdump tools dedupe -mode message-key -execute /path/to/archive
 ```
 
 Pass the archive directory that contains `slackdump.sqlite`. You do not need to
@@ -28,6 +33,14 @@ point the command at the database file itself.
 | Flag | Description |
 |------|-------------|
 | `-execute` | Required flag to actually remove duplicate entities |
+| `-mode` | Message dedupe mode: `exact` keeps current byte-for-byte behavior; `message-key` keeps the latest row per channel and timestamp |
+
+`message-key` is useful after merging multiple Slack exports of the same
+workspace when duplicate messages differ only in volatile Slack-generated JSON
+fields, such as `blocks[].block_id`. It is intentionally opt-in because it can
+discard older edited or reaction variants for the same message timestamp. It
+also treats channel-history and thread-message copies of the same channel
+timestamp as one logical message and keeps the latest database row.
 
 ## Example
 
diff --git a/internal/chunk/backend/dbase/repository/cleanup_test.go b/internal/chunk/backend/dbase/repository/cleanup_test.go
@@ -135,11 +135,16 @@ func insertChunkForTest(t *testing.T, db *sqlx.DB, chunkID, sessionID int64, typ
 }
 
 func insertMessageWithChunkForTest(t *testing.T, db *sqlx.DB, msgID, chunkID int64, data []byte) {
+	t.Helper()
+	insertMessageInChannelWithChunkForTest(t, db, msgID, chunkID, "C001", data)
+}
+
+func insertMessageInChannelWithChunkForTest(t *testing.T, db *sqlx.DB, msgID, chunkID int64, channelID string, data []byte) {
 	t.Helper()
 	_, err := db.ExecContext(context.Background(), `
 		INSERT INTO MESSAGE (ID, CHUNK_ID, CHANNEL_ID, TS, IDX, DATA)
-		VALUES (?, ?, 'C001', '1000000000.000001', 0, ?)`,
-		msgID, chunkID, data)
+		VALUES (?, ?, ?, '1000000000.000001', 0, ?)`,
+		msgID, chunkID, channelID, data)
 	require.NoError(t, err)
 }
 
diff --git a/internal/chunk/backend/dbase/repository/dedupe.go b/internal/chunk/backend/dbase/repository/dedupe.go
@@ -28,12 +28,47 @@ type DedupeResult struct {
 	ChunksRemoved       int64
 }
 
+type MessageDedupeMode string
+
+const (
+	MessageDedupeExact MessageDedupeMode = "exact"
+	MessageDedupeKey   MessageDedupeMode = "message-key"
+)
+
+func (m *MessageDedupeMode) Set(v string) error {
+	mode := MessageDedupeMode(strings.ToLower(v))
+	switch mode {
+	case MessageDedupeExact, MessageDedupeKey:
+		*m = mode
+		return nil
+	default:
+		return fmt.Errorf("unknown message dedupe mode: %s", v)
+	}
+}
+
+func (m MessageDedupeMode) String() string {
+	if m == "" {
+		return string(MessageDedupeExact)
+	}
+	return string(m)
+}
+
 type DedupeRepository interface {
 	Preview(ctx context.Context, db *sqlx.DB) (DedupeCounts, error)
 	Deduplicate(ctx context.Context, db *sqlx.DB) (DedupeResult, error)
 }
 
-type dedupeRepository struct{}
+type DedupeOption func(*dedupeRepository)
+
+func WithMessageDedupeMode(mode MessageDedupeMode) DedupeOption {
+	return func(r *dedupeRepository) {
+		r.messageMode = mode
+	}
+}
+
+type dedupeRepository struct {
+	messageMode MessageDedupeMode
+}
 
 type dedupeMode int
 
@@ -54,7 +89,7 @@ var dedupeEntities = []dedupeEntity{
 	{
 		name:       "messages",
 		table:      "MESSAGE",
-		keyColumns: []string{"ID"},
+		keyColumns: []string{"CHANNEL_ID", "ID"},
 		chunkTypes: []chunk.ChunkType{chunk.CMessages, chunk.CThreadMessages},
 		mode:       dedupeByData,
 	},
@@ -88,13 +123,28 @@ var dedupeEntities = []dedupeEntity{
 	},
 }
 
-func NewDedupeRepository() DedupeRepository {
-	return dedupeRepository{}
+func NewDedupeRepository(opts ...DedupeOption) DedupeRepository {
+	r := dedupeRepository{messageMode: MessageDedupeExact}
+	for _, opt := range opts {
+		opt(&r)
+	}
+	return r
+}
+
+func (r dedupeRepository) entities() []dedupeEntity {
+	entities := make([]dedupeEntity, len(dedupeEntities))
+	copy(entities, dedupeEntities)
+	for i := range entities {
+		if entities[i].name == "messages" && r.messageMode == MessageDedupeKey {
+			entities[i].mode = dedupeByKey
+		}
+	}
+	return entities
 }
 
 func (r dedupeRepository) Preview(ctx context.Context, db *sqlx.DB) (DedupeCounts, error) {
 	var counts DedupeCounts
-	for _, entity := range dedupeEntities {
+	for _, entity := range r.entities() {
 		n, err := r.countDuplicates(ctx, db, entity)
 		if err != nil {
 			return DedupeCounts{}, fmt.Errorf("count duplicate %s: %w", entity.name, err)
@@ -118,7 +168,7 @@ func (r dedupeRepository) Deduplicate(ctx context.Context, db *sqlx.DB) (DedupeR
 	defer tx.Rollback()
 
 	var result DedupeResult
-	for _, entity := range dedupeEntities {
+	for _, entity := range r.entities() {
 		chunkIDs, err := r.prunableChunkIDs(ctx, tx, entity)
 		if err != nil {
 			return DedupeResult{}, fmt.Errorf("query prunable %s chunks: %w", entity.name, err)
diff --git a/internal/chunk/backend/dbase/repository/dedupe_test.go b/internal/chunk/backend/dbase/repository/dedupe_test.go
