feat: add discord attachment and mention indexing

Author: steipete

README.md

@@ -9,6 +9,8 @@ It is a bot-token crawler. No user-token hacks. Data stays local.
 - discovers every guild the configured bot can access
 - syncs channels, threads, members, and message history into SQLite
 - maintains FTS5 search indexes for fast local text search
+- extracts small text-like attachments into the local search index
+- records structured user and role mentions for direct querying
 - tails Gateway events for live updates, with periodic repair syncs
 - exposes read-only SQL for ad hoc analysis
 - keeps schema multi-guild ready while preserving a simple single-guild default UX
@@ -163,7 +165,7 @@ bin/discrawl search --include-empty "GitHub"
 bin/discrawl --json search "websocket closed"
 ```
 
-By default, `search` skips empty-body rows such as attachment-only or system-style messages. Use `--include-empty` to opt back in.
+By default, `search` skips rows with no searchable content. Attachment text, attachment filenames, embeds, and replies still count as content. Use `--include-empty` to opt back in.
 
 ### `messages`
 
@@ -182,9 +184,26 @@ Notes:
 - `--channel` accepts a channel id, exact name, `#name`, or partial name match
 - `--days` is shorthand for "since now minus N days"
 - `--all` removes the safety limit; default is `200`
-- empty-body rows are skipped by default; `--include-empty` opts back in
+- rows with no displayable/searchable content are skipped by default; `--include-empty` opts back in
 - at least one filter is required
 
+### `mentions`
+
+Lists structured user and role mentions.
+
+```bash
+bin/discrawl mentions --channel maintainers --days 7
+bin/discrawl mentions --target steipete --type user --limit 50
+bin/discrawl mentions --target 1456406468898197625
+bin/discrawl --json mentions --type role --days 1
+```
+
+Notes:
+
+- `--target` accepts an id, exact name, or partial name match
+- `--type` can be `user` or `role`
+- same guild/time filters as `messages`
+
 ### `sql`
 
 Runs read-only SQL against the local database.

internal/cli/cli.go

@@ -124,6 +124,8 @@ func (r *runtime) dispatch(rest []string) error {
 		return r.withServices(false, func() error { return r.runSearch(rest[1:]) })
 	case "messages":
 		return r.withServices(false, func() error { return r.runMessages(rest[1:]) })
+	case "mentions":
+		return r.withServices(false, func() error { return r.runMentions(rest[1:]) })
 	case "sql":
 		return r.withServices(false, func() error { return r.runSQL(rest[1:]) })
 	case "members":

internal/cli/cli_test.go

@@ -75,14 +75,42 @@ func TestStatusSearchSQLAndListings(t *testing.T) {
 		NormalizedContent: "",
 		RawJSON:           `{"author":{"username":"Peter"}}`,
 	}))
+	require.NoError(t, s.UpsertMessages(ctx, []store.MessageMutation{{
+		Record: store.MessageRecord{
+			ID:                "m3",
+			GuildID:           "g1",
+			ChannelID:         "c1",
+			ChannelName:       "general",
+			AuthorID:          "u1",
+			AuthorName:        "Peter",
+			MessageType:       0,
+			CreatedAt:         time.Now().UTC().Add(2 * time.Second).Format(time.RFC3339Nano),
+			Content:           "",
+			NormalizedContent: "trace.txt stack trace line one",
+			HasAttachments:    true,
+			RawJSON:           `{"author":{"username":"Peter"}}`,
+		},
+		Mentions: []store.MentionEventRecord{{
+			MessageID:  "m3",
+			GuildID:    "g1",
+			ChannelID:  "c1",
+			AuthorID:   "u1",
+			TargetType: "user",
+			TargetID:   "u2",
+			TargetName: "Shadow",
+			EventAt:    time.Now().UTC().Add(2 * time.Second).Format(time.RFC3339Nano),
+		}},
+	}}))
 	require.NoError(t, s.Close())
 
 	tests := [][]string{
 		{"--config", cfgPath, "status"},
 		{"--config", cfgPath, "search", "panic"},
+		{"--config", cfgPath, "search", "stack"},
 		{"--config", cfgPath, "search", "--include-empty", "Peter"},
 		{"--config", cfgPath, "messages", "--channel", "general", "--days", "7", "--all"},
 		{"--config", cfgPath, "messages", "--channel", "general", "--days", "7", "--all", "--include-empty"},
+		{"--config", cfgPath, "mentions", "--target", "Shadow", "--limit", "10"},
 		{"--config", cfgPath, "sql", "select count(*) as total from messages"},
 		{"--config", cfgPath, "members", "list"},
 		{"--config", cfgPath, "channels", "list"},
@@ -273,6 +301,7 @@ func TestRuntimeHelpersAndSubcommands(t *testing.T) {
 		rt.now = func() time.Time { return time.Date(2026, 3, 8, 12, 0, 0, 0, time.UTC) }
 		require.NoError(t, rt.runMessages([]string{"--channel", "#general", "--days", "7", "--all"}))
 		require.NoError(t, rt.runMessages([]string{"--channel", "#general", "--days", "7", "--all", "--include-empty"}))
+		require.NoError(t, rt.runMentions([]string{"--channel", "#general", "--target", "u2"}))
 		require.NoError(t, rt.runSearch([]string{"--include-empty", "Peter"}))
 		require.NoError(t, rt.runChannels([]string{"show", "c1"}))
 		require.NoError(t, rt.runChannels([]string{"list"}))
@@ -281,6 +310,19 @@ func TestRuntimeHelpersAndSubcommands(t *testing.T) {
 	}))
 }
 
+func TestRunMentionsValidation(t *testing.T) {
+	t.Parallel()
+
+	rt := &runtime{stderr: &bytes.Buffer{}}
+	rt.now = func() time.Time { return time.Date(2026, 3, 8, 12, 0, 0, 0, time.UTC) }
+
+	require.Equal(t, 2, ExitCode(rt.runMentions([]string{"--days", "-1", "--target", "u1"})))
+	require.Equal(t, 2, ExitCode(rt.runMentions([]string{"--days", "1", "--since", "2026-03-01T00:00:00Z", "--target", "u1"})))
+	require.Equal(t, 2, ExitCode(rt.runMentions([]string{"--since", "bad", "--target", "u1"})))
+	require.Equal(t, 2, ExitCode(rt.runMentions([]string{"--type", "nope", "--target", "u1"})))
+	require.Equal(t, 2, ExitCode(rt.runMentions([]string{})))
+}
+
 func TestPrintJSONAndPlain(t *testing.T) {
 	t.Parallel()
 
@@ -296,6 +338,10 @@ func TestPrintJSONAndPlain(t *testing.T) {
 	require.NoError(t, rt.print([]store.SearchResult{{GuildID: "g1", ChannelName: "general", AuthorName: "Peter", Content: "hello"}}))
 	require.Contains(t, rt.stdout.(*bytes.Buffer).String(), "hello")
 
+	rt = &runtime{stdout: &bytes.Buffer{}}
+	require.NoError(t, rt.print([]store.MentionRow{{GuildID: "g1", ChannelName: "general", AuthorName: "Peter", TargetType: "user", TargetName: "Shadow", Content: "hello"}}))
+	require.Contains(t, rt.stdout.(*bytes.Buffer).String(), "Shadow")
+
 	rt = &runtime{stdout: &bytes.Buffer{}, plain: true}
 	require.NoError(t, rt.print([]store.MemberRow{{GuildID: "g1", UserID: "u1", Username: "peter"}}))
 	require.Contains(t, rt.stdout.(*bytes.Buffer).String(), "peter")
@@ -308,6 +354,10 @@ func TestPrintJSONAndPlain(t *testing.T) {
 	require.NoError(t, rt.print([]store.MessageRow{{GuildID: "g1", ChannelID: "c1", AuthorID: "u1", MessageID: "m1", Content: "hello", CreatedAt: time.Unix(1, 0).UTC()}}))
 	require.Contains(t, rt.stdout.(*bytes.Buffer).String(), "m1")
 
+	rt = &runtime{stdout: &bytes.Buffer{}, plain: true}
+	require.NoError(t, rt.print([]store.MentionRow{{GuildID: "g1", ChannelID: "c1", AuthorID: "u1", TargetType: "user", TargetID: "u2", Content: "hello", CreatedAt: time.Unix(1, 0).UTC()}}))
+	require.Contains(t, rt.stdout.(*bytes.Buffer).String(), "u2")
+
 	rt = &runtime{stdout: &bytes.Buffer{}}
 	require.NoError(t, rt.print(struct{ OK bool }{OK: true}))
 	require.Contains(t, rt.stdout.(*bytes.Buffer).String(), "\"OK\": true")

internal/cli/mentions.go

@@ -0,0 +1,93 @@
+package cli
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/steipete/discrawl/internal/store"
+)
+
+func (r *runtime) runMentions(args []string) error {
+	fs := flag.NewFlagSet("mentions", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	channel := fs.String("channel", "", "")
+	author := fs.String("author", "", "")
+	target := fs.String("target", "", "")
+	targetType := fs.String("type", "", "")
+	days := fs.Int("days", 0, "")
+	since := fs.String("since", "", "")
+	before := fs.String("before", "", "")
+	limit := fs.Int("limit", defaultMessageLimit, "")
+	guildsFlag := fs.String("guilds", "", "")
+	guildFlag := fs.String("guild", "", "")
+	if err := fs.Parse(args); err != nil {
+		return usageErr(err)
+	}
+	if fs.NArg() != 0 {
+		return usageErr(fmt.Errorf("mentions takes flags only"))
+	}
+	if *days < 0 {
+		return usageErr(fmt.Errorf("--days must be >= 0"))
+	}
+	if *days > 0 && strings.TrimSpace(*since) != "" {
+		return usageErr(fmt.Errorf("use either --days or --since"))
+	}
+	if *limit < 0 {
+		return usageErr(fmt.Errorf("--limit must be >= 0"))
+	}
+	if targetTypeValue := strings.TrimSpace(*targetType); targetTypeValue != "" && targetTypeValue != "user" && targetTypeValue != "role" {
+		return usageErr(fmt.Errorf("--type must be user or role"))
+	}
+
+	var sinceTime time.Time
+	var beforeTime time.Time
+	var err error
+	if *days > 0 {
+		now := time.Now().UTC()
+		if r.now != nil {
+			now = r.now().UTC()
+		}
+		sinceTime = now.Add(-time.Duration(*days) * 24 * time.Hour)
+	}
+	if strings.TrimSpace(*since) != "" {
+		sinceTime, err = time.Parse(time.RFC3339, *since)
+		if err != nil {
+			return usageErr(fmt.Errorf("invalid --since: %w", err))
+		}
+	}
+	if strings.TrimSpace(*before) != "" {
+		beforeTime, err = time.Parse(time.RFC3339, *before)
+		if err != nil {
+			return usageErr(fmt.Errorf("invalid --before: %w", err))
+		}
+	}
+
+	guildIDs := r.resolveSearchGuilds(*guildFlag, *guildsFlag)
+	if strings.TrimSpace(*channel) == "" &&
+		strings.TrimSpace(*author) == "" &&
+		strings.TrimSpace(*target) == "" &&
+		strings.TrimSpace(*targetType) == "" &&
+		sinceTime.IsZero() &&
+		beforeTime.IsZero() &&
+		len(guildIDs) == 0 {
+		return usageErr(fmt.Errorf("mentions needs at least one filter"))
+	}
+
+	rows, err := r.store.ListMentions(r.ctx, store.MentionListOptions{
+		GuildIDs:   guildIDs,
+		Channel:    *channel,
+		Author:     *author,
+		Target:     *target,
+		TargetType: *targetType,
+		Since:      sinceTime,
+		Before:     beforeTime,
+		Limit:      *limit,
+	})
+	if err != nil {
+		return err
+	}
+	return r.print(rows)
+}

internal/cli/output.go

@@ -54,6 +54,11 @@ func printPlain(w io.Writer, value any) error {
 			_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\n", formatTime(row.CreatedAt), row.GuildID, row.ChannelID, row.AuthorID, row.MessageID, row.Content)
 		}
 		return nil
+	case []store.MentionRow:
+		for _, row := range v {
+			_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", formatTime(row.CreatedAt), row.GuildID, row.ChannelID, row.AuthorID, row.TargetType, row.TargetID, row.Content)
+		}
+		return nil
 	default:
 		return fmt.Errorf("no plain printer")
 	}
@@ -71,6 +76,7 @@ Commands:
   tail
   search
   messages
+  mentions
   sql
   members
   channels
@@ -112,6 +118,13 @@ func printHuman(w io.Writer, value any) error {
 			}
 		}
 		return nil
+	case []store.MentionRow:
+		for _, row := range v {
+			if _, err := fmt.Fprintf(w, "[%s/%s] %s -> %s:%s %s\n%s\n\n", row.GuildID, row.ChannelName, row.AuthorName, row.TargetType, firstNonEmpty(row.TargetName, row.TargetID), formatTime(row.CreatedAt), row.Content); err != nil {
+				return err
+			}
+		}
+		return nil
 	case []store.MemberRow:
 		tw := tabwriter.NewWriter(w, 2, 4, 2, ' ', 0)
 		_, _ = fmt.Fprintln(tw, "GUILD\tUSER\tNAME\tDISPLAY")

internal/store/mentions.go

@@ -0,0 +1,105 @@
+package store
+
+import (
+	"context"
+	"strings"
+)
+
+func (s *Store) ListMentions(ctx context.Context, opts MentionListOptions) ([]MentionRow, error) {
+	if opts.Limit <= 0 {
+		opts.Limit = 200
+	}
+	args := []any{}
+	clauses := []string{"1=1"}
+	if len(opts.GuildIDs) > 0 {
+		clauses = append(clauses, "me.guild_id in ("+placeholders(len(opts.GuildIDs))+")")
+		for _, guildID := range opts.GuildIDs {
+			args = append(args, guildID)
+		}
+	}
+	if channel := normalizeChannelFilter(opts.Channel); channel != "" {
+		clauses = append(clauses, "(me.channel_id = ? or c.name = ? or c.name like ?)")
+		args = append(args, channel, channel, "%"+channel+"%")
+	}
+	if author := strings.TrimSpace(opts.Author); author != "" {
+		clauses = append(clauses, `(me.author_id = ? or coalesce(mem.username, '') = ? or coalesce(mem.display_name, '') = ? or coalesce(mem.username, '') like ? or coalesce(mem.display_name, '') like ?)`)
+		args = append(args, author, author, author, "%"+author+"%", "%"+author+"%")
+	}
+	if target := strings.TrimSpace(opts.Target); target != "" {
+		clauses = append(clauses, `(me.target_id = ? or me.target_name = ? or me.target_name like ?)`)
+		args = append(args, target, target, "%"+target+"%")
+	}
+	if targetType := strings.TrimSpace(opts.TargetType); targetType != "" {
+		clauses = append(clauses, "me.target_type = ?")
+		args = append(args, targetType)
+	}
+	if !opts.Since.IsZero() {
+		clauses = append(clauses, "me.event_at >= ?")
+		args = append(args, opts.Since.UTC().Format(timeLayout))
+	}
+	if !opts.Before.IsZero() {
+		clauses = append(clauses, "me.event_at < ?")
+		args = append(args, opts.Before.UTC().Format(timeLayout))
+	}
+	args = append(args, opts.Limit)
+	rows, err := s.db.QueryContext(ctx, `
+		select
+			me.message_id,
+			me.guild_id,
+			me.channel_id,
+			coalesce(c.name, ''),
+			coalesce(me.author_id, ''),
+			coalesce(
+				nullif(mem.display_name, ''),
+				nullif(mem.nick, ''),
+				nullif(mem.global_name, ''),
+				nullif(mem.username, ''),
+				nullif(json_extract(m.raw_json, '$.author.global_name'), ''),
+				nullif(json_extract(m.raw_json, '$.author.username'), ''),
+				''
+			),
+			me.target_type,
+			me.target_id,
+			me.target_name,
+			case
+				when trim(coalesce(m.content, '')) <> '' then m.content
+				else m.normalized_content
+			end,
+			me.event_at
+		from mention_events me
+		left join messages m on m.id = me.message_id
+		left join channels c on c.id = me.channel_id
+		left join members mem on mem.guild_id = me.guild_id and mem.user_id = me.author_id
+		where `+strings.Join(clauses, " and ")+`
+		order by me.event_at desc, me.event_id desc
+		limit ?
+	`, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = rows.Close() }()
+
+	var out []MentionRow
+	for rows.Next() {
+		var row MentionRow
+		var created string
+		if err := rows.Scan(
+			&row.MessageID,
+			&row.GuildID,
+			&row.ChannelID,
+			&row.ChannelName,
+			&row.AuthorID,
+			&row.AuthorName,
+			&row.TargetType,
+			&row.TargetID,
+			&row.TargetName,
+			&row.Content,
+			&created,
+		); err != nil {
+			return nil, err
+		}
+		row.CreatedAt = parseTime(created)
+		out = append(out, row)
+	}
+	return out, rows.Err()
+}