diff --git a/docs/changelog.md b/docs/changelog.md index 861c662e6..c410edc56 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -55,6 +55,20 @@ All notable changes to msgvault, grouped by release. cursors, and CardDAV sync runs are recorded and recoverable after a daemon restart. +- Chat media collection now skips attachments from conversations with more + than 20 participants by default on Beeper, Slack, Discord, and Teams. Direct + chats and small groups keep their media; skipped occurrences carry a typed + `participant_threshold` marker instead of a retry marker. Set + `media_max_participants = 0` in the provider table to remove the cap, or + raise it to taste. With large-room volume gone, the per-attachment size + default for Beeper, Slack, and Teams moves from 100 MiB to 250 MiB so long + voice notes, screen recordings, and phone video from direct chats are kept; + Discord stays at 50 MiB, and an explicit `max_media_mb` is unchanged. + Previously over-cap files under 250 MiB are retried by the next + `backfill-*-media` run because the cap changed. The `media_scope`, + `media_max_participants`, `max_media_mb`, and `accounts_config` keys are now + documented for every chat provider. + - Starting in v0.20.0, remote deletion remains permanently opt-in. The invoking CLI can grant durable consent with `[deletion] remote_enabled = true`; `MSGVAULT_ENABLE_REMOTE_DELETE=1` diff --git a/docs/configuration.md b/docs/configuration.md index ef9558c34..3b816df04 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,5 +1,5 @@ --- -last_edited: 2026-08-30 +last_edited: 2026-09-03 title: Configuration description: Configuration file reference, environment variables, and file locations. --- @@ -53,6 +53,9 @@ auto_confirm_identities = false [discord] # Per-attachment download cap (default: 50 MiB) max_media_bytes = 52428800 +# Skip attachments from rooms with more than this many participants +# (default: 20; 0 = no cap). Shared by [beeper], [slack], and [teams]. +media_max_participants = 20 # Trailing edit/delete/reaction repair window (default: seven days) edit_rescan_window = "168h" @@ -463,14 +466,22 @@ add-discord`; tokens and binding labels do not belong in `config.toml`. | Key | Default | Description | |---|---|---| | `max_media_bytes` | `52428800` (50 MiB) | Maximum size of one Discord attachment downloaded during sync or backfill | +| `max_media_mb` | — | Same cap in MiB; when set it takes precedence over `max_media_bytes` | +| `media` | `true` | Download attachment bytes at all | +| `media_scope` | `all` | Which conversations collect media: `all`, `direct` (direct and group chats only), or `none` | +| `media_max_participants` | `20` | Skip media from conversations with more participants than this; `0` disables the cap. See [Media policy](#media-policy) | | `edit_rescan_window` | `168h` (seven days) | Trailing per-channel/thread window refreshed for edits, deletions, and reaction summaries | -Use an exact guild ID for a per-guild filter block: +Use an exact guild ID for a per-guild filter block. The same block also takes +the per-account media overrides (`media`, `max_media_mb`) that the other chat +providers put under `accounts_config`: ```toml [discord.guilds."123456789012345678"] include = ["456789012345678901"] exclude = ["567890123456789012"] +# media = false +# max_media_mb = 25 ``` An empty `include` means every accessible text or announcement channel, thread, @@ -480,6 +491,48 @@ can override an excluded parent; an explicit child exclude can override an included parent. `exclude` wins when the same ID is in both lists. See [Discord](/usage/discord/#configure-media-repairs-and-channel-filters). +### Media policy + +`[beeper]`, `[slack]`, `[discord]`, and `[teams]` share one attachment policy +vocabulary. It decides which chat media is downloaded during sync and backfill; +message text is always archived. + +| Key | Default | Description | +|---|---|---| +| `media` | `true` | Download attachment bytes. `false` archives messages without their media and records a `policy_scope` skip marker | +| `media_scope` | `all` | `all` collects from every conversation; `direct` collects only from direct and group chats (not channels, rooms, or guild channels); `none` collects nothing | +| `media_max_participants` | `20` | Skip media from conversations with more participants than this. Omitting the key applies the default; an explicit `0` removes the cap | +| `max_media_mb` | `250` (Discord `50`) | Per-attachment size cap in MiB. Sized for long voice notes, screen recordings, and phone video from direct chats now that the participant cap keeps large-room volume out | +| `accounts_config` | — | Per-account overrides of `media` and `max_media_mb`, keyed by Beeper accountID, Slack team ID, or Teams account email. Discord uses `[discord.guilds.""]` instead | + +The participant cap exists because most attachment bytes in a real chat +archive come from large rooms whose forwarded videos nobody wants kept. Direct +chats and small groups keep their photos, voice notes, and files. A skipped +occurrence is recorded with a typed marker (`participant_threshold`, +`policy_scope`, `account_policy`, or `size_cap`) that distinguishes a +deliberate skip from a failed download, so the `backfill-*-media` commands do +not retry it unless the policy changes. + +```toml +[beeper] +media_scope = "all" +media_max_participants = 20 +max_media_mb = 250 + +# Keep everything from one account regardless of room size or size cap. +[beeper.accounts_config.signal] +media = true +max_media_mb = 500 + +# Never download from another account. +[beeper.accounts_config.telegram] +media = false +``` + +Policy changes apply to future downloads. Media already stored under an +earlier policy stays until you run `msgvault purge-excluded-media`, which +removes attachment bytes the current policy would no longer collect. + ### `[log]` Structured file logging. Disabled by default. Enable it to get persistent, machine-readable logs for troubleshooting. Every CLI invocation writes a unique `run_id` on every log line so you can trace a single run across shared daily log files. @@ -706,7 +759,13 @@ accounts = [] # accountID include filter (empty = all) exclude_accounts = [] # skip networks archived natively, e.g. ["whatsapp"] rate_limit_qps = 20 # request rate against the local API media = true # download attachment bytes -max_media_mb = 100 # per-attachment download cap (MiB) +media_scope = "all" # all, direct, or none +media_max_participants = 20 # skip media from larger rooms; 0 = no cap +max_media_mb = 250 # per-attachment download cap (MiB) + +# [beeper.accounts_config.signal] # per-account override, keyed by accountID +# media = true +# max_media_mb = 500 ``` | Key | Default | Description | @@ -718,7 +777,10 @@ max_media_mb = 100 # per-attachment download cap (MiB) | `exclude_accounts` | — | Beeper accountIDs to skip (wins over `accounts`) | | `rate_limit_qps` | `20` | Request rate limit against the local API | | `media` | `true` | Download attachment bytes (failed downloads retry via `backfill-beeper-media`) | -| `max_media_mb` | `100` | Per-attachment download cap in MiB (over-cap media leaves a retry marker) | +| `media_scope` | `all` | `all`, `direct`, or `none`; see [Media policy](#media-policy) | +| `media_max_participants` | `20` | Skip media from conversations above this many participants; `0` = no cap | +| `max_media_mb` | `250` | Per-attachment download cap in MiB (over-cap media is recorded as a `size_cap` skip and retried only after the cap changes) | +| `accounts_config` | — | Per-accountID `media` and `max_media_mb` overrides | ### `[slack]` @@ -733,7 +795,12 @@ schedule = "*/30 * * * *" # 5-field cron; empty = manual sync only channels = [] # channel-name include filter (empty = all memberships) exclude_channels = [] # channel names to skip, e.g. ["noise"] media = true # download shared-file bytes -max_media_mb = 100 # per-file download cap (MiB) +media_scope = "all" # all, direct, or none +media_max_participants = 20 # skip files from larger channels; 0 = no cap +max_media_mb = 250 # per-file download cap (MiB) + +# [slack.accounts_config.T0123456] # per-workspace override, keyed by team ID +# media = false ``` | Key | Default | Description | @@ -743,7 +810,36 @@ max_media_mb = 100 # per-file download cap (MiB) | `channels` | all | Channel names to sync (include filter; DMs are never filtered) | | `exclude_channels` | — | Channel names to skip (wins over `channels`) | | `media` | `true` | Download shared-file bytes (failed downloads retry via `backfill-slack-media`) | -| `max_media_mb` | `100` | Per-file download cap in MiB (over-cap files leave a retry marker) | +| `media_scope` | `all` | `all`, `direct` (DMs and group DMs only), or `none`; see [Media policy](#media-policy) | +| `media_max_participants` | `20` | Skip files from conversations above this many members; `0` = no cap | +| `max_media_mb` | `250` | Per-file download cap in MiB (over-cap files are recorded as a `size_cap` skip and retried only after the cap changes) | +| `accounts_config` | — | Per-team-ID `media` and `max_media_mb` overrides | + +### `[teams]` + +Media policy for [Microsoft Teams](/usage/teams/) chats and channels. Teams +sync itself is scheduled through `[[accounts]]`; this table only decides which +attachments are downloaded. + +```toml +[teams] +media = true +media_scope = "all" +media_max_participants = 20 +max_media_mb = 250 + +[teams.accounts_config."user@example.com"] +media = true +max_media_mb = 500 +``` + +| Key | Default | Description | +|---|---|---| +| `media` | `true` | Download attachment and inline hosted-content bytes (failed downloads retry via `backfill-teams-media`) | +| `media_scope` | `all` | `all`, `direct` (chats only, not channels), or `none`; see [Media policy](#media-policy) | +| `media_max_participants` | `20` | Skip media from chats and channels above this many members; `0` = no cap | +| `max_media_mb` | `250` | Per-attachment download cap in MiB | +| `accounts_config` | — | Per-account overrides of `media` and `max_media_mb`, keyed by the Teams account email | ### Granola Sources diff --git a/docs/usage/beeper.md b/docs/usage/beeper.md index f00079899..782d66347 100644 --- a/docs/usage/beeper.md +++ b/docs/usage/beeper.md @@ -1,4 +1,5 @@ --- +last_edited: 2026-09-03 title: Beeper description: Archive every chat network connected to Beeper Desktop via its local API. --- @@ -90,9 +91,14 @@ are only picked up by `--full` runs. - Voice-note transcriptions (when Beeper has them) are appended to the message body so they are searchable. - Attachments (photos, videos, voice notes, files) are downloaded during sync - into msgvault's content-addressed attachment store. Downloads that fail (or - exceed `max_media_mb`) leave a pending marker and the message is archived - anyway; retry them later with `msgvault backfill-beeper-media`. Use + into msgvault's content-addressed attachment store. By default media from + conversations with more than 20 participants is skipped with a typed + `participant_threshold` marker, so direct chats and small groups keep their + media while large rooms do not fill the disk; set + `media_max_participants = 0` to collect from every room. Downloads that fail + leave a pending marker and the message is archived anyway; retry them later + with `msgvault backfill-beeper-media`. Over-cap files (`max_media_mb`) are + recorded as a `size_cap` skip and retried only after the cap changes. Use `--no-media` or `media = false` to skip downloads — note that skipped-by-flag downloads leave no pending markers, so `backfill-beeper-media` will not fetch them later; re-enable media and run `sync-beeper --full` instead. @@ -167,9 +173,20 @@ accounts = [] # accountID include filter (empty = all) exclude_accounts = [] # e.g. ["whatsapp"] — see below rate_limit_qps = 20 # request rate against the local API media = true # download attachment bytes -max_media_mb = 100 # per-attachment size cap +media_scope = "all" # all, direct, or none +media_max_participants = 20 # skip media from larger rooms; 0 = no cap +max_media_mb = 250 # per-attachment size cap + +# [beeper.accounts_config.signal] # per-account override, keyed by accountID +# media = true +# max_media_mb = 500 ``` +See [Media policy](/configuration/#media-policy) for how the scope, participant +cap, size cap, and per-account overrides combine, and +`msgvault purge-excluded-media` for removing media a changed policy would no +longer collect. + ### Overlap with native importers If you already archive a network natively (e.g. `import-whatsapp` or diff --git a/docs/usage/discord.md b/docs/usage/discord.md index 8986cdf36..9081c9dc5 100644 --- a/docs/usage/discord.md +++ b/docs/usage/discord.md @@ -1,4 +1,5 @@ --- +last_edited: 2026-09-03 title: Discord description: Archive Discord guild channels, threads, and attachments through a read-only bot. --- @@ -84,19 +85,28 @@ ambiguous fails instead of guessing. ## Configure media, repairs, and channel filters Discord settings are optional. The defaults download attachments up to 50 MiB -and re-scan the trailing seven days for edits, deletions, and changed reaction -counts: +from channels with at most 20 members, and re-scan the trailing seven days for +edits, deletions, and changed reaction counts: ```toml [discord] max_media_bytes = 52428800 +media_scope = "all" # all, direct, or none +media_max_participants = 20 # skip media from larger channels; 0 = no cap edit_rescan_window = "168h" [discord.guilds."123456789012345678"] include = ["456789012345678901"] exclude = ["567890123456789012"] +# media = false # per-guild media override +# max_media_mb = 25 ``` +Most guild channels have more than 20 members, so the default participant cap +keeps guild media out of the archive unless you raise the cap or set it to `0`. +Skipped attachments carry a typed `participant_threshold` marker rather than a +retry marker; see [Media policy](/configuration/#media-policy). + `include` and `exclude` contain Discord channel, thread, or forum-post IDs. An empty `include` means every accessible message container. Top-level channels match their own IDs. Threads and forum posts inherit their parent's state, but @@ -239,7 +249,7 @@ credentials or call the Discord API. ## Attachment backfill and limits Retry attachment downloads after a transient failure or after raising -`max_media_bytes`: +`max_media_bytes` or `media_max_participants`: ```bash # Scan all archived Discord messages that have attachments. diff --git a/docs/usage/slack.md b/docs/usage/slack.md index 89cac3072..988f98cf6 100644 --- a/docs/usage/slack.md +++ b/docs/usage/slack.md @@ -1,5 +1,5 @@ --- -last_edited: 2026-08-30 +last_edited: 2026-09-03 title: Slack description: Archive Slack workspaces through the Web API or a Slackdump export. --- @@ -150,9 +150,14 @@ the archived body, raw JSON, attachments, or reactions. ### Files Files are downloaded into content-addressed attachment storage, capped at -`max_media_mb` per file. Files hosted outside `files.slack.com` (external -links, connected drives) are recorded as metadata + permalink only. Failed -downloads leave pending markers: +`max_media_mb` per file. By default files shared in conversations with more +than 20 members are skipped with a typed `participant_threshold` marker; DMs, +group DMs, and small channels keep theirs. Set `media_max_participants = 0` +under `[slack]` to collect from every channel, or `media_scope = "direct"` to +collect only from DMs and group DMs (see +[Media policy](/configuration/#media-policy)). Files hosted outside +`files.slack.com` (external links, connected drives) are recorded as metadata + +permalink only. Failed downloads leave pending markers: ```bash msgvault backfill-slack-media @@ -172,11 +177,13 @@ deleting the row or retrying an unreachable file forever. [slack] enabled = true schedule = "*/30 * * * *" +media_max_participants = 20 # default; 0 = collect files from every channel ``` The daemon then syncs every registered workspace on the schedule. See [Configuration](/configuration/#slack) for the full option list -(channel include/exclude filters, media caps). +(channel include/exclude filters, media scope, participant and size caps, +per-workspace `accounts_config` overrides). ## Identity unification diff --git a/docs/usage/teams.md b/docs/usage/teams.md index 1c5ff984c..8951aaca4 100644 --- a/docs/usage/teams.md +++ b/docs/usage/teams.md @@ -1,4 +1,5 @@ --- +last_edited: 2026-09-03 title: Microsoft Teams description: Archive Microsoft Teams chats and channels through delegated Microsoft Graph sync. --- @@ -112,6 +113,28 @@ The backfill scans stored Teams HTML bodies for `hostedContents` URLs and downloads those images into the attachment store. It is idempotent because attachment storage is content-addressed. +## Media Policy + +Attachment downloads follow the shared chat media policy. By default media +from chats and channels with more than 20 members is skipped with a typed +`participant_threshold` marker, while one-to-one and small group chats keep +theirs. Adjust it under `[teams]`: + +```toml +[teams] +media = true +media_scope = "all" # all, direct (chats only), or none +media_max_participants = 20 # 0 = no cap +max_media_mb = 250 + +[teams.accounts_config."user@example.com"] +max_media_mb = 500 +``` + +See [Media policy](/configuration/#media-policy) for the full vocabulary and +`msgvault purge-excluded-media` for removing media a changed policy would no +longer collect. + ## Scheduled Sync `msgvault serve` can schedule Teams syncs through the normal `[[accounts]]` diff --git a/internal/attachmentpolicy/policy.go b/internal/attachmentpolicy/policy.go index f539a5654..1d225c659 100644 --- a/internal/attachmentpolicy/policy.go +++ b/internal/attachmentpolicy/policy.go @@ -3,6 +3,13 @@ package attachmentpolicy import "fmt" +// DefaultChatMaxBytes is the per-attachment size cap chat providers (Beeper, +// Slack, Teams) apply when max_media_mb is unset. It is sized for the media +// worth keeping from direct chats and small groups: long voice notes, screen +// recordings, and phone video routinely clear 100 MiB, and the participant +// cap already keeps large-room volume out. Discord keeps its own lower cap. +const DefaultChatMaxBytes int64 = 250 << 20 + // Scope limits media downloads by conversation kind. type Scope string diff --git a/internal/beeper/media.go b/internal/beeper/media.go index 4e86fc786..01017974e 100644 --- a/internal/beeper/media.go +++ b/internal/beeper/media.go @@ -15,9 +15,9 @@ import ( ) // defaultMaxMediaBytes caps individual attachment downloads (config -// max_media_mb overrides). +// max_media_mb overrides). It mirrors the config-layer default exactly. const ( - defaultMaxMediaBytes = int64(100 << 20) + defaultMaxMediaBytes = attachmentpolicy.DefaultChatMaxBytes beeperAttachmentTypeImage = "img" ) diff --git a/internal/config/config.go b/internal/config/config.go index b7b7d7b20..9f3d80f9d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -336,9 +336,23 @@ type BackupConfig struct { } const ( - DefaultChatMaxMediaBytes int64 = 100 << 20 + // DefaultChatMaxMediaBytes is the per-attachment size cap for Beeper, + // Slack, and Teams when max_media_mb is unset. The importers fall back to + // the same attachmentpolicy constant, so the effective default is one + // number wherever the policy is resolved. + DefaultChatMaxMediaBytes int64 = attachmentpolicy.DefaultChatMaxBytes DefaultDiscordMaxMediaBytes int64 = 50 << 20 DefaultDiscordEditRescanWindow time.Duration = 7 * 24 * time.Hour + + // DefaultMediaMaxParticipants is the participant cap applied to Beeper, + // Slack, Discord, and Teams media collection when a config file omits + // media_max_participants. Media from rooms above this size is skipped + // with a typed participant_threshold marker; direct chats and small + // groups keep theirs. NewDefaultConfig pre-fills the four provider + // fields so a file that omits the key inherits the cap, while an explicit + // media_max_participants = 0 still means "no cap" because TOML decoding + // overwrites the pre-filled value with the operator's zero. + DefaultMediaMaxParticipants = 20 ) // MediaAccountConfig overrides attachment download settings for one provider @@ -688,6 +702,11 @@ func NewDefaultConfig() *Config { Accounts: []AccountSchedule{}, SynctechSMS: SynctechSMSConfig{Sources: []SynctechSMSSource{}}, GCal: []GCalSource{}, + // Group-room media is capped by default; see DefaultMediaMaxParticipants. + Beeper: BeeperConfig{MediaMaxParticipants: DefaultMediaMaxParticipants}, + Slack: SlackConfig{MediaMaxParticipants: DefaultMediaMaxParticipants}, + Discord: DiscordConfig{MediaMaxParticipants: DefaultMediaMaxParticipants}, + Teams: TeamsConfig{MediaMaxParticipants: DefaultMediaMaxParticipants}, } cfg.Attachments.Documents = documentindex.DefaultDocumentsConfig() cfg.Vector.ApplyDefaults() @@ -1225,11 +1244,12 @@ type BeeperConfig struct { RateLimitQPS float64 `toml:"rate_limit_qps"` // Media toggles attachment download (nil/absent = enabled). Media *bool `toml:"media"` - // MaxMediaMB caps individual attachment downloads in MiB (0 = 100). + // MaxMediaMB caps individual attachment downloads in MiB (0 = 250). MaxMediaMB int `toml:"max_media_mb"` // MediaScope is all, direct, or none (empty = all). MediaScope string `toml:"media_scope"` - // MediaMaxParticipants caps eligible conversation membership (0 = no cap). + // MediaMaxParticipants caps eligible conversation membership (omitted = + // DefaultMediaMaxParticipants; explicit 0 = no cap). MediaMaxParticipants int `toml:"media_max_participants"` // AccountsConfig holds per-Beeper-account media overrides. AccountsConfig map[string]MediaAccountConfig `toml:"accounts_config"` @@ -1250,17 +1270,20 @@ type SlackConfig struct { ExcludeChannels []string `toml:"exclude_channels"` // Media toggles file download (nil/absent = enabled). Media *bool `toml:"media"` - // MaxMediaMB caps individual file downloads in MiB (0 = 100). + // MaxMediaMB caps individual file downloads in MiB (0 = 250). MaxMediaMB int `toml:"max_media_mb"` // MediaScope is all, direct, or none (empty = all). MediaScope string `toml:"media_scope"` - // MediaMaxParticipants caps eligible conversation membership (0 = no cap). + // MediaMaxParticipants caps eligible conversation membership (omitted = + // DefaultMediaMaxParticipants; explicit 0 = no cap). MediaMaxParticipants int `toml:"media_max_participants"` // AccountsConfig holds per-workspace media overrides keyed by team ID. AccountsConfig map[string]MediaAccountConfig `toml:"accounts_config"` } // TeamsConfig configures provider-wide and per-account Teams media policy. +// MediaMaxParticipants follows the same omitted-versus-explicit-zero rule as +// the other providers. type TeamsConfig struct { Media *bool `toml:"media"` MediaScope string `toml:"media_scope"` @@ -1292,7 +1315,7 @@ func (b BeeperConfig) MaxMediaBytes() int64 { if b.MaxMediaMB > 0 { return int64(b.MaxMediaMB) << 20 } - return 100 << 20 + return DefaultChatMaxMediaBytes } // MediaPolicy resolves Beeper provider settings and an account override. diff --git a/internal/config/config_beeper_test.go b/internal/config/config_beeper_test.go index 0ecf3cb5d..548a89fd2 100644 --- a/internal/config/config_beeper_test.go +++ b/internal/config/config_beeper_test.go @@ -55,7 +55,7 @@ func TestBeeperConfigDefaults(t *testing.T) { assert.False(cfg.Beeper.Enabled) assert.Empty(cfg.Beeper.Schedule) assert.True(cfg.Beeper.MediaEnabled(), "media defaults on") - assert.Equal(int64(100<<20), cfg.Beeper.MaxMediaBytes(), "default 100 MiB cap") + assert.Equal(DefaultChatMaxMediaBytes, cfg.Beeper.MaxMediaBytes(), "default 250 MiB cap") } func TestBeeperMediaConfig(t *testing.T) { diff --git a/internal/config/config_media_policy_test.go b/internal/config/config_media_policy_test.go index 9015209a7..11f89249f 100644 --- a/internal/config/config_media_policy_test.go +++ b/internal/config/config_media_policy_test.go @@ -10,13 +10,95 @@ import ( "go.kenn.io/msgvault/internal/attachmentpolicy" ) -func TestMediaPolicyDefaultsRemainAllowAll(t *testing.T) { +func TestMediaPolicyDefaultsCapGroupRoomMedia(t *testing.T) { assert := assert.New(t) cfg := NewDefaultConfig() - assert.Equal(attachmentpolicy.Policy{Scope: attachmentpolicy.ScopeAll, MaxBytes: 100 << 20}, cfg.Beeper.MediaPolicy("signal")) - assert.Equal(attachmentpolicy.Policy{Scope: attachmentpolicy.ScopeAll, MaxBytes: 100 << 20}, cfg.Slack.MediaPolicy("T01")) - assert.Equal(attachmentpolicy.Policy{Scope: attachmentpolicy.ScopeAll, MaxBytes: 50 << 20}, cfg.Discord.MediaPolicy("G01")) - assert.Equal(attachmentpolicy.Policy{Scope: attachmentpolicy.ScopeAll, MaxBytes: 100 << 20}, cfg.Teams.MediaPolicy("user@example.com")) + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeAll, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: DefaultChatMaxMediaBytes, + }, cfg.Beeper.MediaPolicy("signal")) + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeAll, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: DefaultChatMaxMediaBytes, + }, cfg.Slack.MediaPolicy("T01")) + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeAll, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: DefaultDiscordMaxMediaBytes, + }, cfg.Discord.MediaPolicy("G01")) + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeAll, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: DefaultChatMaxMediaBytes, + }, cfg.Teams.MediaPolicy("user@example.com")) +} + +func TestLoadMediaPolicyOmittedParticipantCapUsesDefault(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + configPath := filepath.Join(t.TempDir(), "config.toml") + // [beeper] and [discord] set other media keys but omit the participant + // cap; [slack] and [teams] have no table at all. + require.NoError(os.WriteFile(configPath, []byte(` +[beeper] +media_scope = "direct" +max_media_mb = 80 + +[discord] +edit_rescan_window = "24h" +[discord.guilds.G01] +include = ["C01"] +`), 0o644)) + + cfg, err := Load(configPath, "") + require.NoError(err) + + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeDirect, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: 80 << 20, + }, cfg.Beeper.MediaPolicy("signal")) + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeAll, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: DefaultChatMaxMediaBytes, + }, cfg.Slack.MediaPolicy("T01")) + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeAll, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: DefaultDiscordMaxMediaBytes, + }, cfg.Discord.MediaPolicy("G01")) + assert.Equal(attachmentpolicy.Policy{ + Scope: attachmentpolicy.ScopeAll, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: DefaultChatMaxMediaBytes, + }, cfg.Teams.MediaPolicy("user@example.com")) + assert.Equal([]string{"C01"}, cfg.Discord.Guilds["G01"].Include, "guild filters survive the default") +} + +func TestLoadMediaPolicyExplicitZeroDisablesParticipantCap(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + configPath := filepath.Join(t.TempDir(), "config.toml") + require.NoError(os.WriteFile(configPath, []byte(` +[beeper] +media_max_participants = 0 + +[slack] +media_max_participants = 0 + +[discord] +media_max_participants = 0 + +[teams] +media_max_participants = 0 +`), 0o644)) + + cfg, err := Load(configPath, "") + require.NoError(err) + + for name, policy := range map[string]attachmentpolicy.Policy{ + "beeper": cfg.Beeper.MediaPolicy("signal"), + "slack": cfg.Slack.MediaPolicy("T01"), + "discord": cfg.Discord.MediaPolicy("G01"), + "teams": cfg.Teams.MediaPolicy("user@example.com"), + } { + assert.Zero(policy.MaxParticipants, "%s: explicit zero means no cap", name) + assert.Equal(attachmentpolicy.ScopeAll, policy.Scope, name) + assert.Empty(policy.DisabledReason, name) + } + // A 300-person room is admitted once the cap is disabled. + room := attachmentpolicy.Conversation{Type: "group_chat", ParticipantCount: 300} + assert.True(cfg.Beeper.MediaPolicy("signal").Allows(room, 1<<20)) + assert.Equal(attachmentpolicy.SkipParticipantThreshold, + NewDefaultConfig().Beeper.MediaPolicy("signal").Evaluate(room, 1<<20), + "the default policy skips the same room") } func TestLoadMediaPolicyAccountOverrides(t *testing.T) { @@ -67,8 +149,10 @@ max_media_mb = 40 Scope: attachmentpolicy.ScopeDirect, MaxParticipants: 5, MaxBytes: 80 << 20, DisabledReason: attachmentpolicy.SkipPolicyScope, }, cfg.Beeper.MediaPolicy("telegram")) + // [slack] and [teams] omit media_max_participants, so the account + // overrides layer on top of the default cap. assert.Equal(attachmentpolicy.Policy{ - Scope: attachmentpolicy.ScopeNone, MaxBytes: 90 << 20, + Scope: attachmentpolicy.ScopeNone, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: 90 << 20, DisabledReason: attachmentpolicy.SkipAccountPolicy, }, cfg.Slack.MediaPolicy("T01")) assert.Equal(attachmentpolicy.Policy{ @@ -76,7 +160,7 @@ max_media_mb = 40 DisabledReason: attachmentpolicy.SkipAccountPolicy, }, cfg.Discord.MediaPolicy("G01")) assert.Equal(attachmentpolicy.Policy{ - Scope: attachmentpolicy.ScopeDirect, MaxBytes: 40 << 20, + Scope: attachmentpolicy.ScopeDirect, MaxParticipants: DefaultMediaMaxParticipants, MaxBytes: 40 << 20, }, cfg.Teams.MediaPolicy("user@example.com")) } diff --git a/internal/slack/media.go b/internal/slack/media.go index a3cc443eb..9b4f28408 100644 --- a/internal/slack/media.go +++ b/internal/slack/media.go @@ -18,8 +18,8 @@ import ( ) // defaultMaxMediaBytes caps individual file downloads (config max_media_mb -// overrides). -const defaultMaxMediaBytes = int64(100 << 20) +// overrides). It mirrors the config-layer default exactly. +const defaultMaxMediaBytes = attachmentpolicy.DefaultChatMaxBytes // mediaHost is the only host file bytes are ever fetched from with the // bearer token. Message JSON is attacker-influenceable (any workspace member diff --git a/internal/slack/media_test.go b/internal/slack/media_test.go index 6894a1bdd..36fb7fb8e 100644 --- a/internal/slack/media_test.go +++ b/internal/slack/media_test.go @@ -594,7 +594,7 @@ func TestMediaTimeoutScalesWithCap(t *testing.T) { // on slow links; the media bound must scale with the size cap (~128 // KiB/s floor rate) above a generous minimum, and never be infinite. assert.Equal(10*time.Minute, mediaTimeout(1<<20), "small caps get the floor") - assert.Equal(800*time.Second, mediaTimeout(100<<20), "default 100 MiB cap ≈ 13m20s") + assert.Equal(2000*time.Second, mediaTimeout(defaultMaxMediaBytes), "default 250 MiB cap ≈ 33m20s") assert.Equal(8192*time.Second, mediaTimeout(1<<30), "bigger caps scale up") assert.Greater(mediaTimeout(1), time.Minute, "never anywhere near the 60s API deadline") } diff --git a/internal/teams/importer.go b/internal/teams/importer.go index c10e8881b..c6bb848a9 100644 --- a/internal/teams/importer.go +++ b/internal/teams/importer.go @@ -1092,7 +1092,7 @@ func (imp *Importer) downloadInlineImages(ctx context.Context, messageID int64, policy := opts.MediaPolicy maxBytes := policy.MaxBytes if maxBytes <= 0 { - maxBytes = 100 << 20 + maxBytes = attachmentpolicy.DefaultChatMaxBytes } policy.MaxBytes = maxBytes