Skip to content

Commit 6ea543b

Browse files
steipetesteipete
committed
test: cover wiretap cache checkpoint helpers
1 parent 78fcca8 commit 6ea543b

5 files changed

Lines changed: 215 additions & 50 deletions

File tree

‎internal/discorddesktop/import.go‎

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"fmt"
1010
"io"
1111
"io/fs"
12+
"maps"
1213
"os"
1314
"path/filepath"
1415
"regexp"
@@ -31,8 +32,10 @@ const (
3132
checkpointEveryFiles = 256
3233
)
3334

34-
var channelRouteRE = regexp.MustCompile(`/channels/(@me|[0-9]{12,24})/([0-9]{12,24})`)
35-
var apiMessagesRouteRE = regexp.MustCompile(`/api/v[0-9]+/channels/[0-9]{12,24}/messages`)
35+
var (
36+
channelRouteRE = regexp.MustCompile(`/channels/(@me|[0-9]{12,24})/([0-9]{12,24})`)
37+
apiMessagesRouteRE = regexp.MustCompile(`/api/v[0-9]+/channels/[0-9]{12,24}/messages`)
38+
)
3639

3740
type Options struct {
3841
Path string
@@ -616,9 +619,7 @@ func finalizeSnapshot(snap snapshot, channelLookup map[string]store.ChannelRecor
616619
}
617620

618621
func mergeUnresolved(dst, src unresolvedMessages) {
619-
for messageID, channelID := range src {
620-
dst[messageID] = channelID
621-
}
622+
maps.Copy(dst, src)
622623
}
623624

624625
func recordUnresolved(unresolved unresolvedMessages, totals scanTotals, stats *Stats) {
@@ -701,32 +702,22 @@ func newSnapshot() snapshot {
701702

702703
func newSnapshotWithContext(base snapshot) snapshot {
703704
snap := newSnapshot()
704-
for channelID, guildID := range base.routes {
705-
snap.routes[channelID] = guildID
706-
}
707-
for userID, label := range base.userLabels {
708-
snap.userLabels[userID] = label
709-
}
705+
maps.Copy(snap.routes, base.routes)
706+
maps.Copy(snap.userLabels, base.userLabels)
710707
return snap
711708
}
712709

713710
func mergeSnapshotContext(base snapshot, next snapshot) {
714711
for channelID, guildID := range next.routes {
715712
collectChannelRoute(base, channelID, guildID)
716713
}
717-
for userID, label := range next.userLabels {
718-
base.userLabels[userID] = label
719-
}
720-
for channelID, channel := range next.channels {
721-
base.channels[channelID] = channel
722-
}
714+
maps.Copy(base.userLabels, next.userLabels)
715+
maps.Copy(base.channels, next.channels)
723716
}
724717

725718
func copyChannelLookup(in map[string]store.ChannelRecord) map[string]store.ChannelRecord {
726719
out := make(map[string]store.ChannelRecord, len(in))
727-
for id, channel := range in {
728-
out[id] = channel
729-
}
720+
maps.Copy(out, in)
730721
return out
731722
}
732723

‎internal/discorddesktop/import_helpers_test.go‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
package discorddesktop
2+
3+
import (
4+
"os"
5+
"path/filepath"
6+
"testing"
7+
8+
"github.com/stretchr/testify/require"
9+
10+
"github.com/steipete/discrawl/internal/store"
11+
)
12+
13+
func TestFileFingerprintStatusHelpers(t *testing.T) {
14+
base := fileFingerprint{Size: 123, ModUnixNS: 456}
15+
require.True(t, sameFileFingerprint(base, fileFingerprint{Size: 123, ModUnixNS: 456, Status: fileStatusSkipped}))
16+
require.False(t, sameFileFingerprint(base, fileFingerprint{Size: 124, ModUnixNS: 456}))
17+
require.False(t, sameFileFingerprint(base, fileFingerprint{Size: 123, ModUnixNS: 457}))
18+
19+
require.True(t, isImportedFingerprint(base))
20+
require.True(t, isImportedFingerprint(importedFingerprint(base)))
21+
require.False(t, isImportedFingerprint(skippedFingerprint(base)))
22+
require.Equal(t, fileStatusImported, importedFingerprint(base).Status)
23+
require.Equal(t, fileStatusSkipped, skippedFingerprint(base).Status)
24+
require.Equal(t, wiretapFileIndexScope, fileIndexScope(Options{}))
25+
require.Equal(t, wiretapFileIndexScope, fileIndexScope(Options{FullCache: true}))
26+
}
27+
28+
func TestSnapshotCopyHelpers(t *testing.T) {
29+
base := newSnapshot()
30+
base.routes["111111111111111121"] = "999999999999999996"
31+
base.userLabels["222222222222222232"] = userLabel{Name: "Alice"}
32+
base.channels["111111111111111121"] = store.ChannelRecord{ID: "111111111111111121", GuildID: "999999999999999996", Name: "general"}
33+
34+
snap := newSnapshotWithContext(base)
35+
require.Equal(t, base.routes, snap.routes)
36+
require.Equal(t, base.userLabels, snap.userLabels)
37+
require.Empty(t, snap.channels)
38+
39+
next := newSnapshot()
40+
next.routes["111111111111111122"] = "999999999999999996"
41+
next.userLabels["222222222222222233"] = userLabel{Name: "Bob"}
42+
next.channels["111111111111111122"] = store.ChannelRecord{ID: "111111111111111122", GuildID: "999999999999999996", Name: "random"}
43+
mergeSnapshotContext(base, next)
44+
45+
require.Equal(t, "999999999999999996", base.routes["111111111111111122"])
46+
require.Equal(t, "Bob", base.userLabels["222222222222222233"].Name)
47+
require.Equal(t, "random", base.channels["111111111111111122"].Name)
48+
49+
lookup := copyChannelLookup(base.channels)
50+
lookup["111111111111111122"] = store.ChannelRecord{ID: "changed"}
51+
require.Equal(t, "random", base.channels["111111111111111122"].Name)
52+
}
53+
54+
func TestSnapshotWithoutMessageEvents(t *testing.T) {
55+
snap := newSnapshot()
56+
snap.messages["333333333333333346"] = store.MessageMutation{
57+
Record: store.MessageRecord{ID: "333333333333333346"},
58+
Options: store.WriteOptions{
59+
AppendEvent: true,
60+
EnqueueEmbedding: true,
61+
},
62+
}
63+
stripped := snapshotWithoutMessageEvents(snap)
64+
require.False(t, stripped.messages["333333333333333346"].Options.AppendEvent)
65+
require.True(t, stripped.messages["333333333333333346"].Options.EnqueueEmbedding)
66+
require.True(t, snap.messages["333333333333333346"].Options.AppendEvent)
67+
}
68+
69+
func TestRouteFilteredCacheHelpers(t *testing.T) {
70+
require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Cache/Cache_Data/entry", "Cache/Cache_Data/entry"))
71+
require.Equal(t, fileSourceCacheData, sourceForPath("/tmp/discord", "/tmp/discord/Service Worker/CacheStorage/cache/entry", "Service Worker/CacheStorage/cache/entry"))
72+
require.Equal(t, fileSourceContext, sourceForPath("/tmp/discord", "/tmp/discord/Local Storage/leveldb/000001.log", "Local Storage/leveldb/000001.log"))
73+
}
74+
75+
func TestCacheFileHasRouteHint(t *testing.T) {
76+
dir := t.TempDir()
77+
require.NoError(t, os.WriteFile(filepath.Join(dir, "route"), []byte("https://discord.com/api/v9/channels/111111111111111121/messages?limit=50"), 0o600))
78+
require.NoError(t, os.WriteFile(filepath.Join(dir, "plain"), []byte("no discord route here"), 0o600))
79+
80+
root, err := os.OpenRoot(dir)
81+
require.NoError(t, err)
82+
defer func() { _ = root.Close() }()
83+
84+
ok, err := cacheFileHasRouteHint(root, "route")
85+
require.NoError(t, err)
86+
require.True(t, ok)
87+
ok, err = cacheFileHasRouteHint(root, "plain")
88+
require.NoError(t, err)
89+
require.False(t, ok)
90+
_, err = cacheFileHasRouteHint(root, "missing")
91+
require.Error(t, err)
92+
}

‎internal/discorddesktop/import_pipeline_test.go‎

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"os"
77
"path/filepath"
8+
"strconv"
89
"testing"
910

1011
"github.com/stretchr/testify/require"
@@ -71,9 +72,9 @@ func TestImportCheckpointsCacheBatches(t *testing.T) {
7172
for i := range checkpointEveryFiles + 1 {
7273
channelID := "111111111111111121"
7374
messageID := 333333333333333346 + i
74-
body := []byte(fmt.Sprintf(`https://discord.com/channels/999999999999999996/%s
75+
body := bytesf(`https://discord.com/channels/999999999999999996/%s
7576
{"id":"%d","channel_id":"%s","content":"checkpoint cache %d","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}}
76-
`, channelID, messageID, channelID, i))
77+
`, channelID, messageID, channelID, i)
7778
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", i)), body, 0o600))
7879
}
7980

@@ -101,18 +102,18 @@ func TestImportUsesLaterCacheMetadataBeforeCheckpointingEarlierBatch(t *testing.
101102

102103
channelID := "111111111111111121"
103104
guildID := "999999999999999996"
104-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), []byte(fmt.Sprintf(`https://discord.com/api/v9/channels/%s/messages?limit=50
105+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50
105106
{"id":"333333333333333346","channel_id":"%s","content":"needs later channel metadata","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}}
106-
`, channelID, channelID)), 0o600))
107+
`, channelID, channelID), 0o600))
107108
for i := 1; i < checkpointEveryFiles; i++ {
108-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", i)), []byte(fmt.Sprintf(
109+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", i)), bytesf(
109110
"https://discord.com/api/v9/channels/%s/messages?limit=50\n",
110111
channelID,
111-
)), 0o600))
112+
), 0o600))
112113
}
113-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", checkpointEveryFiles)), []byte(fmt.Sprintf(`https://discord.com/api/v9/channels/%s/messages?limit=50
114+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", checkpointEveryFiles)), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50
114115
{"id":"%s","guild_id":"%s","type":0,"name":"later-metadata"}
115-
`, channelID, channelID, guildID)), 0o600))
116+
`, channelID, channelID, guildID), 0o600))
116117

117118
st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db"))
118119
require.NoError(t, err)
@@ -147,20 +148,20 @@ func TestImportCheckpointsPartiallyResolvedRetryBatch(t *testing.T) {
147148
resolvedChannelID := "111111111111111121"
148149
unresolvedChannelID := "111111111111111122"
149150
guildID := "999999999999999996"
150-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), []byte(fmt.Sprintf(`https://discord.com/api/v9/channels/%s/messages?limit=50
151+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), bytesf(`https://discord.com/api/v10/channels/%s/messages?limit=50
151152
https://discord.com/api/v9/channels/%s/messages?limit=50
152153
{"id":"333333333333333346","channel_id":"%s","content":"partially resolved retry message","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}}
153154
{"id":"333333333333333347","channel_id":"%s","content":"still unresolved retry message","timestamp":"2026-04-23T18:20:44Z","author":{"id":"222222222222222232","username":"alice"}}
154-
`, resolvedChannelID, unresolvedChannelID, resolvedChannelID, unresolvedChannelID)), 0o600))
155+
`, resolvedChannelID, unresolvedChannelID, resolvedChannelID, unresolvedChannelID), 0o600))
155156
for i := 1; i < checkpointEveryFiles; i++ {
156-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", i)), []byte(fmt.Sprintf(
157+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", i)), bytesf(
157158
"https://discord.com/api/v9/channels/%s/messages?limit=50\n",
158159
resolvedChannelID,
159-
)), 0o600))
160+
), 0o600))
160161
}
161-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", checkpointEveryFiles)), []byte(fmt.Sprintf(`https://discord.com/api/v9/channels/%s/messages?limit=50
162+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, fmt.Sprintf("entry_%03d", checkpointEveryFiles)), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50
162163
{"id":"%s","guild_id":"%s","type":0,"name":"partially-resolved"}
163-
`, resolvedChannelID, resolvedChannelID, guildID)), 0o600))
164+
`, resolvedChannelID, resolvedChannelID, guildID), 0o600))
164165

165166
st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db"))
166167
require.NoError(t, err)
@@ -196,9 +197,9 @@ func TestImportCheckpointsUnresolvableRouteBearingCacheMisses(t *testing.T) {
196197
require.NoError(t, os.MkdirAll(cachePath, 0o755))
197198

198199
channelID := "111111111111111121"
199-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), []byte(fmt.Sprintf(`https://discord.com/api/v9/channels/%s/messages?limit=50
200+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), bytesf(`https://discord.com/api/v9/channels/%s/messages?limit=50
200201
{"id":"333333333333333346","channel_id":"%s","content":"permanent unresolved cache miss","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}}
201-
`, channelID, channelID)), 0o600))
202+
`, channelID, channelID), 0o600))
202203

203204
st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db"))
204205
require.NoError(t, err)
@@ -229,11 +230,11 @@ func TestImportDoesNotAppendEventsForSkippedMixedBatch(t *testing.T) {
229230
guildID := "999999999999999996"
230231
resolvedChannelID := "111111111111111121"
231232
unresolvedChannelID := "111111111111111122"
232-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), []byte(fmt.Sprintf(`https://discord.com/channels/%s/%s
233+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), bytesf(`https://discord.com/channels/%s/%s
233234
https://discord.com/api/v9/channels/%s/messages?limit=50
234235
{"id":"333333333333333346","channel_id":"%s","content":"mixed resolved message","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}}
235236
{"id":"333333333333333347","channel_id":"%s","content":"mixed unresolved message","timestamp":"2026-04-23T18:20:44Z","author":{"id":"222222222222222232","username":"alice"}}
236-
`, guildID, resolvedChannelID, unresolvedChannelID, resolvedChannelID, unresolvedChannelID)), 0o600))
237+
`, guildID, resolvedChannelID, unresolvedChannelID, resolvedChannelID, unresolvedChannelID), 0o600))
237238

238239
st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db"))
239240
require.NoError(t, err)
@@ -267,10 +268,10 @@ func TestImportDoesNotDuplicateEventsWhenSwitchingFullCacheModes(t *testing.T) {
267268

268269
channelID := "111111111111111121"
269270
guildID := "999999999999999996"
270-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), []byte(fmt.Sprintf(`https://discord.com/channels/%s/%s
271+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_000"), bytesf(`https://discord.com/channels/%s/%s
271272
{"id":"%s","guild_id":"%s","type":0,"name":"mode-switch"}
272273
{"id":"333333333333333346","channel_id":"%s","content":"mode switch event once","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}}
273-
`, guildID, channelID, channelID, guildID, channelID)), 0o600))
274+
`, guildID, channelID, channelID, guildID, channelID), 0o600))
274275

275276
t.Run("full then default", func(t *testing.T) {
276277
st, err := store.Open(ctx, filepath.Join(dir, "full-first.db"))
@@ -319,14 +320,14 @@ func TestImportFastCachePreservesKnownChannelMetadataAcrossBatches(t *testing.T)
319320

320321
channelID := "111111111111111121"
321322
guildID := "999999999999999996"
322-
require.NoError(t, os.WriteFile(filepath.Join(leveldbPath, "000001.log"), []byte(fmt.Sprintf(
323+
require.NoError(t, os.WriteFile(filepath.Join(leveldbPath, "000001.log"), bytesf(
323324
`{"id":"%s","guild_id":"%s","type":11,"name":"known-thread","thread_metadata":{"archived":false}}`,
324325
channelID,
325326
guildID,
326-
)), 0o600))
327-
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), []byte(fmt.Sprintf(`https://discord.com/channels/%s/%s
327+
), 0o600))
328+
require.NoError(t, os.WriteFile(filepath.Join(cachePath, "entry_0"), bytesf(`https://discord.com/channels/%s/%s
328329
{"id":"333333333333333346","channel_id":"%s","content":"thread metadata cache","timestamp":"2026-04-23T18:20:43Z","author":{"id":"222222222222222232","username":"alice"}}
329-
`, guildID, channelID, channelID)), 0o600))
330+
`, guildID, channelID, channelID), 0o600))
330331

331332
st, err := store.Open(ctx, filepath.Join(dir, "discrawl.db"))
332333
require.NoError(t, err)
@@ -374,9 +375,13 @@ func TestImportFastCacheRouteFiltersServiceWorkerCacheStorage(t *testing.T) {
374375

375376
func requireMessageCount(t *testing.T, ctx context.Context, st *store.Store, table string, expected int) {
376377
t.Helper()
377-
_, rows, err := st.ReadOnlyQuery(ctx, fmt.Sprintf("select count(*) from %s", table))
378+
_, rows, err := st.ReadOnlyQuery(ctx, "select count(*) from "+table)
378379
require.NoError(t, err)
379380
require.Len(t, rows, 1)
380381
require.Len(t, rows[0], 1)
381-
require.Equal(t, fmt.Sprint(expected), rows[0][0])
382+
require.Equal(t, strconv.Itoa(expected), rows[0][0])
383+
}
384+
385+
func bytesf(format string, args ...any) []byte {
386+
return fmt.Appendf(nil, format, args...)
382387
}

‎internal/discorddesktop/import_run.go‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,7 @@ func (r *importRun) scanContext(candidates []fileCandidate) error {
4949

5050
func (r *importRun) scanCacheBatches(candidates []fileCandidate) error {
5151
for start := 0; start < len(candidates); start += checkpointEveryFiles {
52-
end := start + checkpointEveryFiles
53-
if end > len(candidates) {
54-
end = len(candidates)
55-
}
52+
end := min(start+checkpointEveryFiles, len(candidates))
5653
batchCandidates := candidates[start:end]
5754
batch := newSnapshotWithContext(r.base)
5855
if err := scanCandidates(r.ctx, r.rootFS, r.opts, batchCandidates, batch, r.channelLookup, r.stats); err != nil {

0 commit comments

Comments
 (0)