fix: keep large message listings within SQLite parameter limits (#238)

Author: steipete

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Fix large message listings and TUI reply/mention hydration failing with SQLite's parameter limit, preserving all requested rows and display names.
 - Update Crawlkit to v0.16.3, retaining the existing Go minimum and SQLite runtime pairing.
 
 ## 0.15.1 - 2026-09-13

docs/commands/messages.md

@@ -35,6 +35,7 @@ discrawl --json messages --channel maintainers --days 3
 - if `tail` is already running, plain `messages` reads the local archive without waiting; `messages --sync` fails fast instead of waiting behind the tail lock
 - `--dm` skips Git snapshot auto-update because DMs are never imported from the shared mirror
 - use either `--last` for the newest matching rows or `--all` for an uncapped oldest-to-newest slice
+- uncapped and large listings retain mention display names even beyond 32,766 messages; reply and mention lookups do not impose a separate row cap
 - ambiguous channel names fail with candidate guild/channel ids; resolve once with `discrawl channels resolve <name> --json` and reuse the numeric id
 
 ## Empty results

internal/store/messages.go

@@ -2,7 +2,11 @@ package store
 
 import (
 	"context"
+	"database/sql"
+	"encoding/json"
+	"maps"
 	"regexp"
+	"slices"
 	"strings"
 	"time"
 )
@@ -198,7 +202,7 @@ func (s *Store) hydrateMessageThreadContext(ctx context.Context, rows []MessageR
 	if len(rows) == 0 {
 		return rows, nil
 	}
-	rootIDs := make([]any, 0, len(rows))
+	rootIDs := make([]string, 0, len(rows))
 	seenRoots := map[string]struct{}{}
 	visible := map[string]struct{}{}
 	for _, row := range rows {
@@ -256,9 +260,9 @@ func (s *Store) hydrateMessageThreadContext(ctx context.Context, rows []MessageR
 		left join guilds g on g.id = m.guild_id and g.deleted_at is null
 		left join channels c on c.id = m.channel_id
 		left join members mem on mem.guild_id = m.guild_id and mem.user_id = m.author_id and mem.deleted_at is null
-		where m.id in (` + placeholders(len(rootIDs)) + `)
+		where m.id in (select value from json_each(?))
 		order by m.created_at asc, m.id asc`
-	contextRows, err := s.db.QueryContext(ctx, query, rootIDs...)
+	contextRows, err := s.queryMessageIDs(ctx, query, rootIDs)
 	if err != nil {
 		return nil, err
 	}
@@ -349,7 +353,7 @@ func (s *Store) resolveMessageDisplayMentions(ctx context.Context, rows []Messag
 	if len(rows) == 0 {
 		return nil
 	}
-	ids := make([]any, 0, len(rows))
+	ids := make([]string, 0, len(rows))
 	indexByID := make(map[string]int, len(rows))
 	for index, row := range rows {
 		id := strings.TrimSpace(row.MessageID)
@@ -362,8 +366,8 @@ func (s *Store) resolveMessageDisplayMentions(ctx context.Context, rows []Messag
 	if len(ids) == 0 {
 		return nil
 	}
-	query := `select message_id, target_type, target_id, target_name from mention_events where message_id in (` + placeholders(len(ids)) + `)`
-	mentionRows, err := s.db.QueryContext(ctx, query, ids...)
+	query := `select message_id, target_type, target_id, target_name from mention_events where message_id in (select value from json_each(?))`
+	mentionRows, err := s.queryMessageIDs(ctx, query, ids)
 	if err != nil {
 		return err
 	}
@@ -458,7 +462,6 @@ func (s *Store) discordMemberDisplayNames(ctx context.Context, ids map[string]st
 	if len(ids) == 0 {
 		return nil, nil
 	}
-	args := mapKeysAsAny(ids)
 	query := `
 		select guild_id, user_id,
 			coalesce(
@@ -469,9 +472,9 @@ func (s *Store) discordMemberDisplayNames(ctx context.Context, ids map[string]st
 				''
 			)
 		from members
-		where deleted_at is null and user_id in (` + placeholders(len(args)) + `)
+		where deleted_at is null and user_id in (select value from json_each(?))
 	`
-	rows, err := s.db.QueryContext(ctx, query, args...)
+	rows, err := s.queryMessageIDs(ctx, query, slices.Collect(maps.Keys(ids)))
 	if err != nil {
 		return nil, err
 	}
@@ -491,9 +494,8 @@ func (s *Store) discordChannelNames(ctx context.Context, ids map[string]struct{}
 	if len(ids) == 0 {
 		return nil, nil
 	}
-	args := mapKeysAsAny(ids)
-	query := `select guild_id, id, coalesce(nullif(name, ''), '') from channels where id in (` + placeholders(len(args)) + `)`
-	rows, err := s.db.QueryContext(ctx, query, args...)
+	query := `select guild_id, id, coalesce(nullif(name, ''), '') from channels where id in (select value from json_each(?))`
+	rows, err := s.queryMessageIDs(ctx, query, slices.Collect(maps.Keys(ids)))
 	if err != nil {
 		return nil, err
 	}
@@ -509,12 +511,13 @@ func (s *Store) discordChannelNames(ctx context.Context, ids map[string]struct{}
 	return out, rows.Err()
 }
 
-func mapKeysAsAny(values map[string]struct{}) []any {
-	out := make([]any, 0, len(values))
-	for value := range values {
-		out = append(out, value)
+func (s *Store) queryMessageIDs(ctx context.Context, query string, ids []string) (*sql.Rows, error) {
+	// Bind one JSON array so uncapped listings do not exhaust SQLite parameters.
+	encoded, err := json.Marshal(ids)
+	if err != nil {
+		return nil, err
 	}
-	return out
+	return s.db.QueryContext(ctx, query, string(encoded))
 }
 
 func rememberResolvedDiscordName(out map[string]string, guildID, id, name string) {

internal/store/messages_large_test.go

@@ -0,0 +1,80 @@
+package store
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMessageHydrationBeyondSQLiteVariableLimit(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	s, err := Open(ctx, filepath.Join(t.TempDir(), "discrawl.db"))
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, s.Close()) })
+
+	// SQLite permits 32766 bound parameters; uncapped archive slices can exceed it.
+	const count = 32767
+	_, err = s.db.ExecContext(ctx, `
+		with recursive sequence(n) as (select 1 union all select n + 1 from sequence where n < ?)
+		insert into messages (id, guild_id, channel_id, message_type, created_at, content, normalized_content, raw_json, updated_at)
+		select printf('m%05d', n), 'g1', 'archive', 0, '2026-09-01T00:00:00.000000000Z',
+		       'ping <@&role>', 'ping <@&role>', '{}', '2026-09-01T00:00:00.000000000Z' from sequence`, count)
+	require.NoError(t, err)
+	_, err = s.db.ExecContext(ctx, `
+		insert into mention_events (message_id, guild_id, channel_id, target_type, target_id, target_name, event_at)
+		select id, guild_id, channel_id, 'role', 'role', 'Readers', created_at from messages`)
+	require.NoError(t, err)
+	require.NoError(t, s.UpsertMember(ctx, MemberRecord{GuildID: "g1", UserID: "u32767", Username: "last-user", RoleIDsJSON: `[]`, RawJSON: `{}`}))
+	require.NoError(t, s.UpsertChannel(ctx, ChannelRecord{ID: "c32767", GuildID: "g1", Kind: "text", Name: "last-channel", RawJSON: `{}`}))
+
+	t.Run("uncapped listing", func(t *testing.T) {
+		rows, err := s.ListMessages(ctx, MessageListOptions{Channel: "archive"})
+		require.NoError(t, err)
+		require.Len(t, rows, count)
+		require.Equal(t, "m00001", rows[0].MessageID)
+		require.Equal(t, "m32767", rows[count-1].MessageID)
+		for _, row := range rows {
+			require.Equal(t, "ping @Readers", row.DisplayContent)
+		}
+	})
+
+	t.Run("reply roots", func(t *testing.T) {
+		replies := make([]MessageRow, count)
+		for i := range replies {
+			replies[i] = MessageRow{MessageID: fmt.Sprintf("reply%d", i), GuildID: "g1", ReplyToMessage: fmt.Sprintf("m%05d", i+1)}
+		}
+		rows, err := s.hydrateMessageThreadContext(ctx, replies, count)
+		require.NoError(t, err)
+		require.Len(t, rows, 2*count)
+		require.Equal(t, replies, rows[:count])
+		require.Equal(t, "m00001", rows[count].MessageID)
+		require.Equal(t, "m32767", rows[2*count-1].MessageID)
+		require.Equal(t, "ping @Readers", rows[2*count-1].DisplayContent)
+	})
+
+	for _, kind := range []string{"user", "channel"} {
+		t.Run(kind+" names", func(t *testing.T) {
+			rows := make([]MessageRow, count)
+			for i := range rows {
+				content := fmt.Sprintf("<@u%d>", i+1)
+				if kind == "channel" {
+					content = fmt.Sprintf("<#c%d>", i+1)
+				}
+				rows[i] = MessageRow{GuildID: "g1", DisplayContent: content}
+			}
+			require.NoError(t, s.resolveInlineDiscordMentions(ctx, rows))
+			if kind == "user" {
+				require.Equal(t, "<@u1>", rows[0].DisplayContent)
+				require.Equal(t, "@last-user", rows[count-1].DisplayContent)
+			} else {
+				require.Equal(t, "<#c1>", rows[0].DisplayContent)
+				require.Equal(t, "#last-channel", rows[count-1].DisplayContent)
+			}
+		})
+	}
+}