From 0f1c386361574a97bfe0a998aed144c55b3221bf Mon Sep 17 00:00:00 2001 From: Ricardo Canales Date: Sat, 6 Jun 2026 21:39:45 -0700 Subject: [PATCH 1/7] =?UTF-8?q?feat(csv-import):=20Wave=200=20foundations?= =?UTF-8?q?=20=E2=80=94=20textmatch,=20staging=20schema,=20content=5Fhash,?= =?UTF-8?q?=20fingerprint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Groundwork for CSV import v2 (drag-anywhere, dedupe, account detection, profiles): - Extract pure name-similarity scorer into tag-free internal/textmatch so the CSV classifier can reuse the exact sync-matcher logic without importing sync (cycle-free). Matcher delegates; behavior unchanged (existing tests pass). - Migrations (additive, shared-DB safe): - transactions.content_hash (account-independent dedup) + partial transactions_dedup_lookup index on (account_id, date, amount). - csv_import_profiles (saved recipes keyed by header fingerprint). - csv_import_sessions + csv_import_rows (durable staging so preview == apply). - sqlc queries for the three tables (rows via :copyfrom for 50k-row inserts) plus UpsertTransactionV2 returning (xmax = 0) AS inserted for reliable insert-vs-update counts. - internal/provider/csv/fingerprint.go: HeaderFingerprint, ExtractMask (card column + filename), FilenameTokens, InstitutionTokens. Unit tested. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitignore | 1 + ...260607043318_transactions_content_hash.sql | 25 +++ .../20260607043418_csv_import_profiles.sql | 46 ++++++ .../20260607043518_csv_import_sessions.sql | 81 ++++++++++ internal/db/queries/csv_import_profiles.sql | 59 ++++++++ internal/db/queries/csv_import_rows.sql | 73 +++++++++ internal/db/queries/csv_import_sessions.sql | 68 +++++++++ internal/db/queries/transactions.sql | 53 +++++++ internal/provider/csv/fingerprint.go | 143 ++++++++++++++++++ internal/provider/csv/fingerprint_test.go | 65 ++++++++ internal/sync/matcher.go | 74 ++------- internal/textmatch/similarity.go | 98 ++++++++++++ internal/textmatch/similarity_test.go | 77 ++++++++++ 13 files changed, 799 insertions(+), 64 deletions(-) create mode 100644 internal/db/migrations/20260607043318_transactions_content_hash.sql create mode 100644 internal/db/migrations/20260607043418_csv_import_profiles.sql create mode 100644 internal/db/migrations/20260607043518_csv_import_sessions.sql create mode 100644 internal/db/queries/csv_import_profiles.sql create mode 100644 internal/db/queries/csv_import_rows.sql create mode 100644 internal/db/queries/csv_import_sessions.sql create mode 100644 internal/provider/csv/fingerprint.go create mode 100644 internal/provider/csv/fingerprint_test.go create mode 100644 internal/textmatch/similarity.go create mode 100644 internal/textmatch/similarity_test.go diff --git a/.gitignore b/.gitignore index cefaec719..98ff8602c 100644 --- a/.gitignore +++ b/.gitignore @@ -31,6 +31,7 @@ vendor/ internal/db/db.go internal/db/models.go internal/db/*.sql.go +internal/db/copyfrom.go # templ generated files — *.templ files are source, *_templ.go are artifacts # produced by `make templ` (or `templ generate`). Keep generated Go out of diff --git a/internal/db/migrations/20260607043318_transactions_content_hash.sql b/internal/db/migrations/20260607043318_transactions_content_hash.sql new file mode 100644 index 000000000..54f8d8899 --- /dev/null +++ b/internal/db/migrations/20260607043318_transactions_content_hash.sql @@ -0,0 +1,25 @@ +-- +goose Up + +-- content_hash: an account-INDEPENDENT fingerprint of a transaction's core +-- content (date | amount | normalized description). The existing dedup key +-- `provider_transaction_id` bakes in the account id, so it only catches exact +-- re-imports into the SAME account. The CSV import v2 classifier needs to detect +-- duplicates against ALL transactions already in a resolved account — including +-- Plaid/Teller rows — so it compares on this account-independent hash plus a +-- date+amount+name fuzzy pass. +-- +-- Nullable + additive: CSV v2 writes it going forward; legacy provider rows stay +-- NULL (the classifier falls back to date+amount+name for those). Safe on the +-- shared dev DB alongside running servers. +ALTER TABLE transactions ADD COLUMN IF NOT EXISTS content_hash TEXT; + +-- Fast candidate lookup for classification: "all live transactions in this +-- account near this date with this amount". Partial on deleted_at because the +-- classifier only ever compares against live rows. +CREATE INDEX IF NOT EXISTS transactions_dedup_lookup + ON transactions (account_id, date, amount) + WHERE deleted_at IS NULL; + +-- +goose Down +DROP INDEX IF EXISTS transactions_dedup_lookup; +ALTER TABLE transactions DROP COLUMN IF EXISTS content_hash; diff --git a/internal/db/migrations/20260607043418_csv_import_profiles.sql b/internal/db/migrations/20260607043418_csv_import_profiles.sql new file mode 100644 index 000000000..30a95a97c --- /dev/null +++ b/internal/db/migrations/20260607043418_csv_import_profiles.sql @@ -0,0 +1,46 @@ +-- +goose Up + +-- csv_import_profiles: saved CSV "recipes" so repeat imports from the same +-- source are one click. Keyed by a header fingerprint (a stable hash of the +-- file's normalized header row). On a future import whose fingerprint matches, +-- the flow auto-applies the saved column mapping / date format / sign / currency +-- and pre-selects `default_account_id`, collapsing straight to a deduped preview. +-- +-- A profile is upserted on every successful apply: created the first time a given +-- header layout is imported, then bumped (times_used / last_used_at) and updated +-- to remember the most recently resolved account. One profile per +-- (user, header_fingerprint). +-- +-- Additive-only: one new table. Safe on the shared dev DB. +CREATE TABLE csv_import_profiles ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + short_id TEXT NOT NULL UNIQUE, + user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE, + name TEXT NOT NULL, + header_fingerprint TEXT NOT NULL, + headers JSONB NOT NULL DEFAULT '[]'::jsonb, + detected_template TEXT NULL, + column_mapping JSONB NOT NULL DEFAULT '{}'::jsonb, + date_format TEXT NOT NULL DEFAULT '', + delimiter TEXT NOT NULL DEFAULT ',', + positive_is_debit BOOLEAN NOT NULL DEFAULT FALSE, + has_debit_credit BOOLEAN NOT NULL DEFAULT FALSE, + iso_currency_code TEXT NOT NULL DEFAULT 'USD', + default_account_id UUID NULL REFERENCES accounts (id) ON DELETE SET NULL, + institution_hint TEXT NULL, + mask_hint TEXT NULL, + times_used INTEGER NOT NULL DEFAULT 0, + last_used_at TIMESTAMPTZ NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + UNIQUE (user_id, header_fingerprint) +); + +CREATE TRIGGER set_short_id_csv_import_profiles + BEFORE INSERT ON csv_import_profiles + FOR EACH ROW EXECUTE FUNCTION set_short_id(); + +CREATE INDEX csv_import_profiles_user_idx ON csv_import_profiles (user_id); + +-- +goose Down +DROP TABLE IF EXISTS csv_import_profiles; diff --git a/internal/db/migrations/20260607043518_csv_import_sessions.sql b/internal/db/migrations/20260607043518_csv_import_sessions.sql new file mode 100644 index 000000000..07ddf061f --- /dev/null +++ b/internal/db/migrations/20260607043518_csv_import_sessions.sql @@ -0,0 +1,81 @@ +-- +goose Up + +-- csv_import_sessions + csv_import_rows: durable staging for the CSV import v2 +-- flow. The current wizard holds parsed rows in the HTTP session, which can't +-- satisfy "preview exactly what will happen, then apply that exact set" — the +-- classification depends on live DB state, a 50k-row classified set with edits +-- can't live in a cookie, and apply needs to be atomic + idempotent. +-- +-- A session is created on drop, classified once, persisted here, edited in place, +-- and applied in a single transaction. Rows carry their parsed values + a +-- per-row classification (new / exact_dup / probable_dup / conflict / error / +-- needs_account) and the user's include/exclude intent. +-- +-- Additive-only: two new tables. Safe on the shared dev DB. +CREATE TABLE csv_import_sessions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + short_id TEXT NOT NULL UNIQUE, + user_id UUID NOT NULL REFERENCES users (id) ON DELETE CASCADE, + -- analyzing | awaiting_account | previewed | applying | applied | failed | abandoned + status TEXT NOT NULL DEFAULT 'analyzing', + filename TEXT NOT NULL DEFAULT '', + file_sha256 TEXT NOT NULL DEFAULT '', + delimiter TEXT NOT NULL DEFAULT ',', + headers JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_blob BYTEA NULL, -- original bytes; nulled after apply + row_count INTEGER NOT NULL DEFAULT 0, + resolved_account_id UUID NULL REFERENCES accounts (id) ON DELETE SET NULL, + resolved_connection_id UUID NULL REFERENCES bank_connections (id) ON DELETE SET NULL, + detected_template TEXT NULL, + column_mapping JSONB NULL, + date_format TEXT NOT NULL DEFAULT '', + positive_is_debit BOOLEAN NOT NULL DEFAULT FALSE, + has_debit_credit BOOLEAN NOT NULL DEFAULT FALSE, + iso_currency_code TEXT NOT NULL DEFAULT 'USD', + profile_id UUID NULL REFERENCES csv_import_profiles (id) ON DELETE SET NULL, + result JSONB NULL, -- snapshot of CSVImportResult after apply + sync_log_id UUID NULL REFERENCES sync_logs (id) ON DELETE SET NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NULL -- TTL for abandoned-session cleanup +); + +CREATE TRIGGER set_short_id_csv_import_sessions + BEFORE INSERT ON csv_import_sessions + FOR EACH ROW EXECUTE FUNCTION set_short_id(); + +CREATE INDEX csv_import_sessions_user_idx ON csv_import_sessions (user_id); +-- Sweeper lookup: abandoned/expired sessions that were never applied. +CREATE INDEX csv_import_sessions_expires_idx ON csv_import_sessions (expires_at) + WHERE status <> 'applied'; + +CREATE TABLE csv_import_rows ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + session_id UUID NOT NULL REFERENCES csv_import_sessions (id) ON DELETE CASCADE, + row_index INTEGER NOT NULL, + raw JSONB NOT NULL DEFAULT '[]'::jsonb, + parsed_date DATE NULL, + parsed_amount NUMERIC(12,2) NULL, + parsed_desc TEXT NULL, + parsed_merchant TEXT NULL, + parsed_category TEXT NULL, + -- new | exact_dup | probable_dup | conflict | error | needs_account + classification TEXT NOT NULL DEFAULT 'new', + match_txn_id UUID NULL REFERENCES transactions (id) ON DELETE SET NULL, + match_score INTEGER NULL, + match_reason TEXT NULL, + parse_error TEXT NULL, + content_hash TEXT NULL, + provider_txn_id TEXT NULL, + include BOOLEAN NOT NULL DEFAULT TRUE, + user_edited BOOLEAN NOT NULL DEFAULT FALSE, + UNIQUE (session_id, row_index) +); + +-- Summary counts (GROUP BY classification) + filtered pagination per session. +CREATE INDEX csv_import_rows_session_class_idx + ON csv_import_rows (session_id, classification); + +-- +goose Down +DROP TABLE IF EXISTS csv_import_rows; +DROP TABLE IF EXISTS csv_import_sessions; diff --git a/internal/db/queries/csv_import_profiles.sql b/internal/db/queries/csv_import_profiles.sql new file mode 100644 index 000000000..27c286c97 --- /dev/null +++ b/internal/db/queries/csv_import_profiles.sql @@ -0,0 +1,59 @@ +-- name: UpsertCSVImportProfile :one +-- Create a profile for this (user, header layout), or update the existing one. +-- `name` is intentionally NOT in the conflict SET list so a user's rename +-- survives future imports. times_used is bumped and last_used_at refreshed. +INSERT INTO csv_import_profiles ( + user_id, name, header_fingerprint, headers, detected_template, + column_mapping, date_format, delimiter, positive_is_debit, has_debit_credit, + iso_currency_code, default_account_id, institution_hint, mask_hint, + times_used, last_used_at +) VALUES ( + $1, $2, $3, $4, $5, + $6, $7, $8, $9, $10, + $11, $12, $13, $14, + 1, NOW() +) +ON CONFLICT (user_id, header_fingerprint) DO UPDATE SET + headers = EXCLUDED.headers, + detected_template = EXCLUDED.detected_template, + column_mapping = EXCLUDED.column_mapping, + date_format = EXCLUDED.date_format, + delimiter = EXCLUDED.delimiter, + positive_is_debit = EXCLUDED.positive_is_debit, + has_debit_credit = EXCLUDED.has_debit_credit, + iso_currency_code = EXCLUDED.iso_currency_code, + default_account_id = EXCLUDED.default_account_id, + institution_hint = EXCLUDED.institution_hint, + mask_hint = EXCLUDED.mask_hint, + times_used = csv_import_profiles.times_used + 1, + last_used_at = NOW(), + updated_at = NOW() +RETURNING *; + +-- name: GetCSVImportProfileByFingerprint :one +SELECT * FROM csv_import_profiles +WHERE user_id = $1 AND header_fingerprint = $2; + +-- name: GetCSVImportProfile :one +SELECT * FROM csv_import_profiles WHERE id = $1; + +-- name: GetCSVImportProfileByShortID :one +SELECT * FROM csv_import_profiles WHERE short_id = $1; + +-- name: ListCSVImportProfilesByUser :many +SELECT * FROM csv_import_profiles +WHERE user_id = $1 +ORDER BY last_used_at DESC NULLS LAST, created_at DESC; + +-- name: ListCSVImportProfiles :many +SELECT * FROM csv_import_profiles +ORDER BY last_used_at DESC NULLS LAST, created_at DESC; + +-- name: RenameCSVImportProfile :one +UPDATE csv_import_profiles +SET name = $2, updated_at = NOW() +WHERE id = $1 +RETURNING *; + +-- name: DeleteCSVImportProfile :exec +DELETE FROM csv_import_profiles WHERE id = $1; diff --git a/internal/db/queries/csv_import_rows.sql b/internal/db/queries/csv_import_rows.sql new file mode 100644 index 000000000..6befd4dc5 --- /dev/null +++ b/internal/db/queries/csv_import_rows.sql @@ -0,0 +1,73 @@ +-- name: CreateCSVImportRows :copyfrom +-- Bulk-insert fully-parsed + classified rows (up to 50k). Classification is +-- computed in Go, so (re)classification is a DeleteCSVImportRows + this copy. +INSERT INTO csv_import_rows ( + session_id, row_index, raw, parsed_date, parsed_amount, parsed_desc, + parsed_merchant, parsed_category, classification, match_txn_id, match_score, + match_reason, parse_error, content_hash, provider_txn_id, include, user_edited +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 +); + +-- name: DeleteCSVImportRows :exec +DELETE FROM csv_import_rows WHERE session_id = $1; + +-- name: GetCSVImportRow :one +SELECT * FROM csv_import_rows WHERE id = $1; + +-- name: GetCSVImportRowsPage :many +-- Paginated preview. Empty $2 returns all classifications. +SELECT * FROM csv_import_rows +WHERE session_id = $1 + AND ($2 = '' OR classification = $2) +ORDER BY row_index +LIMIT $3 OFFSET $4; + +-- name: CountCSVImportRowsByClassification :many +SELECT classification, COUNT(*) AS count +FROM csv_import_rows +WHERE session_id = $1 +GROUP BY classification; + +-- name: ListIncludedCSVImportRows :many +-- The exact set that apply will upsert: included, non-error rows in row order. +SELECT * FROM csv_import_rows +WHERE session_id = $1 + AND include = TRUE + AND classification <> 'error' + AND classification <> 'needs_account' +ORDER BY row_index; + +-- name: UpdateCSVImportRow :one +-- Inline edit of a single row (also re-derives its hash/classification in Go). +UPDATE csv_import_rows +SET parsed_date = $2, + parsed_amount = $3, + parsed_desc = $4, + parsed_merchant = $5, + parsed_category = $6, + classification = $7, + match_txn_id = $8, + match_score = $9, + match_reason = $10, + parse_error = $11, + content_hash = $12, + provider_txn_id = $13, + include = $14, + user_edited = TRUE +WHERE id = $1 +RETURNING *; + +-- name: SetCSVImportRowInclude :exec +UPDATE csv_import_rows SET include = $2 WHERE id = $1; + +-- name: SetCSVImportRowsIncludeByClassification :exec +UPDATE csv_import_rows +SET include = $3 +WHERE session_id = $1 AND classification = $2; + +-- name: SetCSVImportRowsCategoryAll :exec +-- Bulk "set category" across all included rows in the session. +UPDATE csv_import_rows +SET parsed_category = $2, user_edited = TRUE +WHERE session_id = $1 AND include = TRUE; diff --git a/internal/db/queries/csv_import_sessions.sql b/internal/db/queries/csv_import_sessions.sql new file mode 100644 index 000000000..6a347ec64 --- /dev/null +++ b/internal/db/queries/csv_import_sessions.sql @@ -0,0 +1,68 @@ +-- name: CreateCSVImportSession :one +INSERT INTO csv_import_sessions ( + user_id, status, filename, file_sha256, delimiter, headers, raw_blob, + row_count, detected_template, column_mapping, date_format, + positive_is_debit, has_debit_credit, iso_currency_code, profile_id, expires_at +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, + $8, $9, $10, $11, + $12, $13, $14, $15, $16 +) +RETURNING *; + +-- name: GetCSVImportSession :one +SELECT * FROM csv_import_sessions WHERE id = $1; + +-- name: GetCSVImportSessionByShortID :one +SELECT * FROM csv_import_sessions WHERE short_id = $1; + +-- name: UpdateCSVImportSessionStatus :exec +UPDATE csv_import_sessions +SET status = $2, updated_at = NOW() +WHERE id = $1; + +-- name: ResolveCSVImportSessionAccount :one +-- Bind the chosen target account/connection + currency and advance status. +UPDATE csv_import_sessions +SET resolved_account_id = $2, + resolved_connection_id = $3, + iso_currency_code = $4, + status = $5, + updated_at = NOW() +WHERE id = $1 +RETURNING *; + +-- name: UpdateCSVImportSessionParse :one +-- Re-map columns / sign / date-format / template (triggers a reclassify). +UPDATE csv_import_sessions +SET column_mapping = $2, + date_format = $3, + positive_is_debit = $4, + has_debit_credit = $5, + detected_template = $6, + updated_at = NOW() +WHERE id = $1 +RETURNING *; + +-- name: SetCSVImportSessionProfile :exec +UPDATE csv_import_sessions +SET profile_id = $2, updated_at = NOW() +WHERE id = $1; + +-- name: FinalizeCSVImportSession :exec +-- After a successful apply: snapshot result, link the sync log, drop the blob. +UPDATE csv_import_sessions +SET status = 'applied', + result = $2, + sync_log_id = $3, + raw_blob = NULL, + updated_at = NOW() +WHERE id = $1; + +-- name: ListExpiredCSVImportSessions :many +SELECT * FROM csv_import_sessions +WHERE status <> 'applied' AND expires_at IS NOT NULL AND expires_at < NOW() +ORDER BY expires_at; + +-- name: DeleteCSVImportSession :exec +DELETE FROM csv_import_sessions WHERE id = $1; diff --git a/internal/db/queries/transactions.sql b/internal/db/queries/transactions.sql index e351ef249..48773c46a 100644 --- a/internal/db/queries/transactions.sql +++ b/internal/db/queries/transactions.sql @@ -48,6 +48,59 @@ ON CONFLICT (provider_transaction_id) DO UPDATE SET END RETURNING *; +-- name: UpsertTransactionV2 :one +-- Like UpsertTransaction but also stores the account-independent content_hash +-- (used by CSV import v2 dedup) and returns whether the row was newly inserted +-- (xmax = 0 on INSERT, non-zero on the ON CONFLICT UPDATE path) so callers get +-- reliable insert-vs-update counts without timestamp heuristics. +INSERT INTO transactions ( + account_id, provider_transaction_id, provider_pending_transaction_id, + amount, iso_currency_code, date, authorized_date, + datetime, authorized_datetime, provider_name, provider_merchant_name, + provider_category_primary, provider_category_detailed, provider_category_confidence, + provider_payment_channel, pending, category_id, provider_raw, merchant_key, + content_hash +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20 +) +ON CONFLICT (provider_transaction_id) DO UPDATE SET + account_id = EXCLUDED.account_id, + merchant_key = EXCLUDED.merchant_key, + provider_pending_transaction_id = EXCLUDED.provider_pending_transaction_id, + amount = EXCLUDED.amount, + iso_currency_code = EXCLUDED.iso_currency_code, + date = EXCLUDED.date, + authorized_date = EXCLUDED.authorized_date, + datetime = EXCLUDED.datetime, + authorized_datetime = EXCLUDED.authorized_datetime, + provider_name = EXCLUDED.provider_name, + provider_merchant_name = EXCLUDED.provider_merchant_name, + provider_category_primary = EXCLUDED.provider_category_primary, + provider_category_detailed = EXCLUDED.provider_category_detailed, + provider_category_confidence = EXCLUDED.provider_category_confidence, + provider_payment_channel = EXCLUDED.provider_payment_channel, + pending = EXCLUDED.pending, + provider_raw = COALESCE(EXCLUDED.provider_raw, transactions.provider_raw), + content_hash = COALESCE(EXCLUDED.content_hash, transactions.content_hash), + category_id = CASE + WHEN transactions.category_override <> 'none' THEN transactions.category_id + WHEN EXCLUDED.category_id IS NOT NULL THEN EXCLUDED.category_id + ELSE transactions.category_id + END, + deleted_at = NULL, + updated_at = CASE + WHEN transactions.amount IS DISTINCT FROM EXCLUDED.amount + OR transactions.provider_name IS DISTINCT FROM EXCLUDED.provider_name + OR transactions.pending IS DISTINCT FROM EXCLUDED.pending + OR transactions.provider_merchant_name IS DISTINCT FROM EXCLUDED.provider_merchant_name + OR transactions.provider_category_primary IS DISTINCT FROM EXCLUDED.provider_category_primary + OR transactions.provider_category_detailed IS DISTINCT FROM EXCLUDED.provider_category_detailed + OR transactions.deleted_at IS NOT NULL + THEN NOW() + ELSE transactions.updated_at + END +RETURNING *, (xmax = 0) AS inserted; + -- name: SoftDeleteTransactionByExternalID :exec UPDATE transactions SET deleted_at = NOW() WHERE provider_transaction_id = $1 AND deleted_at IS NULL; diff --git a/internal/provider/csv/fingerprint.go b/internal/provider/csv/fingerprint.go new file mode 100644 index 000000000..1ed47248c --- /dev/null +++ b/internal/provider/csv/fingerprint.go @@ -0,0 +1,143 @@ +//go:build !lite + +package csv + +import ( + "crypto/sha256" + "encoding/hex" + "regexp" + "strings" +) + +// NormalizeHeaders lowercases and trims each header. The order is preserved +// because column mapping is index-based — a bank reordering its columns is a +// genuinely different layout that should not reuse the same saved profile. +func NormalizeHeaders(headers []string) []string { + out := make([]string, len(headers)) + for i, h := range headers { + out[i] = strings.ToLower(strings.TrimSpace(h)) + } + return out +} + +// HeaderFingerprint returns a stable hash of the normalized, ordered header row. +// It is the key used to look up a saved CSV import profile: two files from the +// same source export the same header layout and therefore the same fingerprint. +func HeaderFingerprint(headers []string) string { + norm := NormalizeHeaders(headers) + sum := sha256.Sum256([]byte(strings.Join(norm, "\x1f"))) + return hex.EncodeToString(sum[:]) +} + +// maskHeaderRe matches header names that plausibly carry a card/account number. +var maskHeaderRe = regexp.MustCompile(`(?i)\b(card\s*no\.?|card\s*number|account\s*(no\.?|number)|last\s*4)\b`) + +// digitsRe extracts runs of digits. +var digitsRe = regexp.MustCompile(`\d+`) + +// filenameMaskRe finds a 4-digit group preceded by a mask-ish hint in a +// filename, e.g. "x4321", "ending 4321", "acct_4321", "-4321". +// The trailing (?:\D|$) (rather than \b) rejects the case where the 4 digits are +// part of a longer run — "_" counts as a word char so \b would mis-fire on +// "4321_". RE2 has no lookahead, so we consume one non-digit outside the group. +var filenameMaskRe = regexp.MustCompile(`(?i)(?:x|ending|acct|account|card|no|#|-|_)\s*[-_ ]?(\d{4})(?:\D|$)`) + +// lastFour returns the trailing 4 digits of s (digits only), or "" if there are +// fewer than 4 digits. +func lastFour(s string) string { + var b strings.Builder + for _, r := range s { + if r >= '0' && r <= '9' { + b.WriteByte(byte(r)) + } + } + d := b.String() + if len(d) < 4 { + return "" + } + return d[len(d)-4:] +} + +// ExtractMask attempts to recover the account's last-4 ("mask") from the file — +// first from a card/account-number column (using a sample of data rows), then +// from the filename. Returns "" when nothing convincing is found. +func ExtractMask(filename string, headers []string, sampleRows [][]string) string { + // 1. A dedicated card/account-number column. + for i, h := range headers { + if !maskHeaderRe.MatchString(h) { + continue + } + for _, row := range sampleRows { + if i >= len(row) { + continue + } + if m := lastFour(row[i]); m != "" { + return m + } + } + } + + // 2. The filename, with a mask-ish hint in front of the 4 digits. + base := filename + if idx := strings.LastIndexAny(base, `/\`); idx >= 0 { + base = base[idx+1:] + } + if m := filenameMaskRe.FindStringSubmatch(base); m != nil { + return m[1] + } + + return "" +} + +// stopTokens are filename fragments that carry no account-identifying signal. +var stopTokens = map[string]bool{ + "csv": true, "tsv": true, "txt": true, "transactions": true, + "transaction": true, "activity": true, "statement": true, "export": true, + "download": true, "history": true, "data": true, "report": true, +} + +// tokenSplitRe splits a filename into alphabetic/numeric tokens. +var tokenSplitRe = regexp.MustCompile(`[^a-zA-Z0-9]+`) + +// FilenameTokens returns lowercased, meaningful tokens from a filename (no +// extension, no generic words, no pure numbers). Used to match a file against +// an account's name/nickname. +func FilenameTokens(filename string) []string { + base := filename + if idx := strings.LastIndexAny(base, `/\`); idx >= 0 { + base = base[idx+1:] + } + if idx := strings.LastIndex(base, "."); idx >= 0 { + base = base[:idx] + } + var out []string + for _, tok := range tokenSplitRe.Split(strings.ToLower(base), -1) { + if len(tok) < 3 || stopTokens[tok] { + continue + } + if digitsRe.MatchString(tok) && !regexp.MustCompile(`[a-z]`).MatchString(tok) { + continue // pure number + } + out = append(out, tok) + } + return out +} + +// InstitutionTokens returns lowercased significant tokens of an institution / +// account name, suitable for substring overlap scoring. Drops short and generic +// words ("the", "card", "checking", ...). +func InstitutionTokens(name string) []string { + generic := map[string]bool{ + "the": true, "card": true, "credit": true, "checking": true, + "savings": true, "account": true, "bank": true, "of": true, + "and": true, "debit": true, "rewards": true, "cash": true, + } + var out []string + for _, tok := range tokenSplitRe.Split(strings.ToLower(strings.TrimSpace(name)), -1) { + if len(tok) < 3 || generic[tok] { + continue + } + out = append(out, tok) + } + return out +} diff --git a/internal/provider/csv/fingerprint_test.go b/internal/provider/csv/fingerprint_test.go new file mode 100644 index 000000000..1765a7ba5 --- /dev/null +++ b/internal/provider/csv/fingerprint_test.go @@ -0,0 +1,65 @@ +//go:build !lite + +package csv + +import ( + "reflect" + "testing" +) + +func TestHeaderFingerprintStableAndOrderSensitive(t *testing.T) { + a := HeaderFingerprint([]string{"Date", "Amount", "Description"}) + // Case/whitespace insensitive, same order → same fingerprint. + b := HeaderFingerprint([]string{" date ", "AMOUNT", "description"}) + if a != b { + t.Errorf("fingerprint should ignore case/whitespace: %s != %s", a, b) + } + // Different column order → different fingerprint (mapping is index-based). + c := HeaderFingerprint([]string{"Amount", "Date", "Description"}) + if a == c { + t.Error("fingerprint should be order-sensitive") + } +} + +func TestExtractMaskFromColumn(t *testing.T) { + headers := []string{"Transaction Date", "Card No.", "Description", "Debit"} + rows := [][]string{ + {"2026-01-02", "", "STARBUCKS", "5.00"}, // empty mask cell + {"2026-01-03", "1234567890124321", "AMZN", ""}, // full PAN → last4 + } + if got := ExtractMask("capone.csv", headers, rows); got != "4321" { + t.Errorf("ExtractMask column = %q, want 4321", got) + } +} + +func TestExtractMaskFromFilename(t *testing.T) { + headers := []string{"Date", "Amount", "Description"} + cases := map[string]string{ + "Chase_x4321_Activity.csv": "4321", + "statement-ending 9876.csv": "9876", + "acct_5555_2026.csv": "5555", + "plain_transactions.csv": "", + "/Users/me/Downloads/card-0001": "0001", + } + for fn, want := range cases { + if got := ExtractMask(fn, headers, nil); got != want { + t.Errorf("ExtractMask(%q) = %q, want %q", fn, got, want) + } + } +} + +func TestFilenameTokens(t *testing.T) { + got := FilenameTokens("Chase_Sapphire_transactions_2026.csv") + want := []string{"chase", "sapphire"} + if !reflect.DeepEqual(got, want) { + t.Errorf("FilenameTokens = %v, want %v", got, want) + } +} + +func TestInstitutionTokens(t *testing.T) { + got := InstitutionTokens("Chase Sapphire Credit Card") + want := []string{"chase", "sapphire"} + if !reflect.DeepEqual(got, want) { + t.Errorf("InstitutionTokens = %v, want %v", got, want) + } +} diff --git a/internal/sync/matcher.go b/internal/sync/matcher.go index 7feca75a7..440f32b87 100644 --- a/internal/sync/matcher.go +++ b/internal/sync/matcher.go @@ -10,6 +10,7 @@ import ( "breadbox/internal/db" "breadbox/internal/pgconv" + "breadbox/internal/textmatch" "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" @@ -240,40 +241,11 @@ func pickBestCandidate(depName, depMerchant string, candidates []matchCandidate) } // nameSimilarityScore computes how similar two transactions' names are. -// Returns score (0-3) and which fields matched. +// Returns score (0-3) and which fields matched. The scoring lives in the +// tag-free internal/textmatch package so the CSV import classifier can reuse +// the exact same logic without importing the sync package. func nameSimilarityScore(depName, depMerchant, priName, priMerchant string) (int, []string) { - // Exact merchant_name match (highest signal). - // EqualFold covers case-insensitive equality without allocating. - if depMerchant != "" && priMerchant != "" && - strings.EqualFold(depMerchant, priMerchant) { - return 3, []string{"merchant_name"} - } - - // Merchant name contains or is contained. - if depMerchant != "" && priMerchant != "" { - depMerchantLower := strings.ToLower(depMerchant) - priMerchantLower := strings.ToLower(priMerchant) - if strings.Contains(depMerchantLower, priMerchantLower) || - strings.Contains(priMerchantLower, depMerchantLower) { - return 2, []string{"merchant_name"} - } - } - - // Exact name match. - if strings.EqualFold(depName, priName) { - return 2, []string{"name"} - } - - // Name contains or is contained. - depNameLower := strings.ToLower(depName) - priNameLower := strings.ToLower(priName) - if strings.Contains(depNameLower, priNameLower) || - strings.Contains(priNameLower, depNameLower) { - return 1, []string{"name"} - } - - // No name similarity — still valid (date + amount matched). - return 0, nil + return textmatch.Score(depName, depMerchant, priName, priMerchant) } // nameSimilarityScoreLowered is the same as nameSimilarityScore but accepts @@ -285,37 +257,11 @@ func nameSimilarityScoreLowered( depMerchant, depMerchantLower, priName, priMerchant string, ) (int, []string) { - // Exact merchant_name match (highest signal). - if depMerchant != "" && priMerchant != "" && - strings.EqualFold(depMerchant, priMerchant) { - return 3, []string{"merchant_name"} - } - - // Merchant name contains or is contained. We reuse depMerchantLower - // across candidates and only lower priMerchant on demand. - if depMerchant != "" && priMerchant != "" { - priMerchantLower := strings.ToLower(priMerchant) - if strings.Contains(depMerchantLower, priMerchantLower) || - strings.Contains(priMerchantLower, depMerchantLower) { - return 2, []string{"merchant_name"} - } - } - - // Exact name match. - if strings.EqualFold(depName, priName) { - return 2, []string{"name"} - } - - // Name contains or is contained. Reuse the pre-lowered depName and - // lower priName on demand. - priNameLower := strings.ToLower(priName) - if strings.Contains(depNameLower, priNameLower) || - strings.Contains(priNameLower, depNameLower) { - return 1, []string{"name"} - } - - // No name similarity — still valid (date + amount matched). - return 0, nil + return textmatch.ScoreLowered( + depName, depNameLower, + depMerchant, depMerchantLower, + priName, priMerchant, + ) } // buildMatchedOn returns which name fields matched between two transactions. diff --git a/internal/textmatch/similarity.go b/internal/textmatch/similarity.go new file mode 100644 index 000000000..7dc02d790 --- /dev/null +++ b/internal/textmatch/similarity.go @@ -0,0 +1,98 @@ +// Package textmatch holds pure, dependency-free string-similarity scoring used +// to decide whether two transactions describe the same thing. It is shared by +// the sync matcher (cross-connection duplicate reconciliation) and the CSV +// import classifier (dedup against existing transactions). +// +// This package is deliberately tag-free (no build constraints, no imports of +// server-only packages) so it compiles under every build tag and unit-tests +// fast without a DB. +package textmatch + +import "strings" + +// Score computes how similar two transactions' names are. It returns a score +// (0-3, higher is a stronger match) and which fields matched ("merchant_name" +// and/or "name"). The comparison is case-insensitive and symmetric. +// +// 3 — merchant names match exactly +// 2 — merchant names contain one another, OR names match exactly +// 1 — names contain one another +// 0 — no name similarity (caller may still treat as a match on date+amount) +// +// The first pair (a*) and second pair (b*) are interchangeable; the returned +// field labels reflect which field produced the match, not which side. +func Score(aName, aMerchant, bName, bMerchant string) (int, []string) { + // Exact merchant_name match (highest signal). EqualFold covers + // case-insensitive equality without allocating. + if aMerchant != "" && bMerchant != "" && + strings.EqualFold(aMerchant, bMerchant) { + return 3, []string{"merchant_name"} + } + + // Merchant name contains or is contained. + if aMerchant != "" && bMerchant != "" { + aMerchantLower := strings.ToLower(aMerchant) + bMerchantLower := strings.ToLower(bMerchant) + if strings.Contains(aMerchantLower, bMerchantLower) || + strings.Contains(bMerchantLower, aMerchantLower) { + return 2, []string{"merchant_name"} + } + } + + // Exact name match. + if strings.EqualFold(aName, bName) { + return 2, []string{"name"} + } + + // Name contains or is contained. + aNameLower := strings.ToLower(aName) + bNameLower := strings.ToLower(bName) + if strings.Contains(aNameLower, bNameLower) || + strings.Contains(bNameLower, aNameLower) { + return 1, []string{"name"} + } + + // No name similarity — still valid (caller matched on date + amount). + return 0, nil +} + +// ScoreLowered is identical to Score but accepts pre-lowered first-side strings +// so a caller iterating over many candidates can lower the (shared) first side +// exactly once. The scoring behavior is identical to Score. +func ScoreLowered( + aName, aNameLower, + aMerchant, aMerchantLower, + bName, bMerchant string, +) (int, []string) { + // Exact merchant_name match (highest signal). + if aMerchant != "" && bMerchant != "" && + strings.EqualFold(aMerchant, bMerchant) { + return 3, []string{"merchant_name"} + } + + // Merchant name contains or is contained. We reuse aMerchantLower across + // candidates and only lower bMerchant on demand. + if aMerchant != "" && bMerchant != "" { + bMerchantLower := strings.ToLower(bMerchant) + if strings.Contains(aMerchantLower, bMerchantLower) || + strings.Contains(bMerchantLower, aMerchantLower) { + return 2, []string{"merchant_name"} + } + } + + // Exact name match. + if strings.EqualFold(aName, bName) { + return 2, []string{"name"} + } + + // Name contains or is contained. Reuse the pre-lowered aName and lower + // bName on demand. + bNameLower := strings.ToLower(bName) + if strings.Contains(aNameLower, bNameLower) || + strings.Contains(bNameLower, aNameLower) { + return 1, []string{"name"} + } + + // No name similarity — still valid (caller matched on date + amount). + return 0, nil +} diff --git a/internal/textmatch/similarity_test.go b/internal/textmatch/similarity_test.go new file mode 100644 index 000000000..c99b5377a --- /dev/null +++ b/internal/textmatch/similarity_test.go @@ -0,0 +1,77 @@ +package textmatch + +import ( + "reflect" + "strings" + "testing" +) + +func TestScore(t *testing.T) { + tests := []struct { + name string + aName, aMerchant, bName, bMerc string + wantScore int + wantFields []string + }{ + {"merchant exact", "card 1234", "Starbucks", "POS 9", "starbucks", 3, []string{"merchant_name"}}, + {"merchant contains", "x", "Starbucks Coffee", "y", "starbucks", 2, []string{"merchant_name"}}, + {"name exact, no merchant", "AMAZON", "", "amazon", "", 2, []string{"name"}}, + {"name contains", "AMAZON MKTPLACE US", "", "amazon", "", 1, []string{"name"}}, + {"no similarity", "WALMART", "", "TARGET", "", 0, nil}, + {"merchant beats name", "AMAZON", "Whole Foods", "AMAZON", "whole foods market", 2, []string{"merchant_name"}}, + {"empty merchant falls to name", "Uber Trip", "", "UBER", "", 1, []string{"name"}}, + {"both empty", "", "", "", "", 2, []string{"name"}}, // EqualFold("","") is true + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotScore, gotFields := Score(tt.aName, tt.aMerchant, tt.bName, tt.bMerc) + if gotScore != tt.wantScore { + t.Errorf("Score() score = %d, want %d", gotScore, tt.wantScore) + } + if !reflect.DeepEqual(gotFields, tt.wantFields) { + t.Errorf("Score() fields = %v, want %v", gotFields, tt.wantFields) + } + }) + } +} + +func TestScoreSymmetric(t *testing.T) { + // Swapping the two sides must yield the same score (field labels are + // field-based, not side-based, so they match too). + cases := [][4]string{ + {"AMAZON MKTPLACE", "", "amazon", ""}, + {"x", "Starbucks Coffee", "y", "starbucks"}, + {"WALMART", "", "TARGET", ""}, + } + for _, c := range cases { + s1, f1 := Score(c[0], c[1], c[2], c[3]) + s2, f2 := Score(c[2], c[3], c[0], c[1]) + if s1 != s2 || !reflect.DeepEqual(f1, f2) { + t.Errorf("asymmetric for %v: (%d,%v) vs (%d,%v)", c, s1, f1, s2, f2) + } + } +} + +func TestScoreLoweredParity(t *testing.T) { + // ScoreLowered must score identically to Score for every input — it is + // purely an allocation optimization. + cases := [][4]string{ + {"card 1234", "Starbucks", "POS 9", "starbucks"}, + {"AMAZON MKTPLACE US", "", "amazon", ""}, + {"WALMART", "Walmart", "TARGET", "Target"}, + {"Uber Trip", "", "UBER", ""}, + {"", "", "", ""}, + } + for _, c := range cases { + wantScore, wantFields := Score(c[0], c[1], c[2], c[3]) + gotScore, gotFields := ScoreLowered( + c[0], strings.ToLower(c[0]), + c[1], strings.ToLower(c[1]), + c[2], c[3], + ) + if gotScore != wantScore || !reflect.DeepEqual(gotFields, wantFields) { + t.Errorf("parity mismatch for %v: lowered=(%d,%v) direct=(%d,%v)", + c, gotScore, gotFields, wantScore, wantFields) + } + } +} From 34646a7d127014d0637f38a11a9a7c9f59e75972 Mon Sep 17 00:00:00 2001 From: Ricardo Canales Date: Sat, 6 Jun 2026 21:48:01 -0700 Subject: [PATCH 2/7] =?UTF-8?q?feat(csv-import):=20Wave=201=20=E2=80=94=20?= =?UTF-8?q?dedup=20classifier=20+=20deterministic=20account=20matcher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - internal/service/csv_classify.go: ParseCSVRows (mapping → parsed rows, errors preserved for inline-fix) + ClassifyCSVRows. Loads the resolved account's live transactions in the date window (provider-agnostic — dedupes vs Plaid/Teller too) and classifies each row as new / exact_dup (provider id or content_hash) / probable_dup (same amount, date within tolerance, name similar via textmatch) / error. content_hash + stable provider_txn_id derived per row. - internal/service/csv_account_match.go: weighted, explainable account ranker — saved-profile default account (auto), mask/last4 (+40), transaction overlap (≤+40), institution name (+15), filename hint (+5). Preselect only when the top is strong AND a clear leader, or a profile matched. - csv dedup helpers: GenerateContentHash (account-independent) + GenerateExternalIDWithOccurrence (force-import disambiguator). - sqlc: ListAccountTransactionsForDedup, ListTransactionKeysInRange. - Integration tests: all-new, exact dup by provider id, cross-provider probable dup, nearby-date/different-name stays new, parse error; overlap ranking, mask+overlap preselect, profile preselect. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/db/queries/transactions.sql | 17 + internal/provider/csv/dedup.go | 37 ++ internal/service/csv_account_match.go | 307 +++++++++++++++ .../csv_account_match_integration_test.go | 123 ++++++ internal/service/csv_classify.go | 350 ++++++++++++++++++ .../service/csv_classify_integration_test.go | 108 ++++++ 6 files changed, 942 insertions(+) create mode 100644 internal/service/csv_account_match.go create mode 100644 internal/service/csv_account_match_integration_test.go create mode 100644 internal/service/csv_classify.go create mode 100644 internal/service/csv_classify_integration_test.go diff --git a/internal/db/queries/transactions.sql b/internal/db/queries/transactions.sql index 48773c46a..b0560dff6 100644 --- a/internal/db/queries/transactions.sql +++ b/internal/db/queries/transactions.sql @@ -101,6 +101,23 @@ ON CONFLICT (provider_transaction_id) DO UPDATE SET END RETURNING *, (xmax = 0) AS inserted; +-- name: ListAccountTransactionsForDedup :many +-- Live transactions in an account within a date window — the candidate set the +-- CSV import classifier compares incoming rows against (provider-agnostic, so it +-- also dedupes against Plaid/Teller rows already in the account). +SELECT id, date, amount, provider_name, provider_merchant_name, + content_hash, provider_transaction_id +FROM transactions +WHERE account_id = $1 AND deleted_at IS NULL AND date BETWEEN $2 AND $3; + +-- name: ListTransactionKeysInRange :many +-- (account_id, date, amount) for all live transactions in a date window, across +-- every account — used to score how well an uploaded CSV overlaps each candidate +-- account during account auto-detection. +SELECT account_id, date, amount +FROM transactions +WHERE deleted_at IS NULL AND account_id IS NOT NULL AND date BETWEEN $1 AND $2; + -- name: SoftDeleteTransactionByExternalID :exec UPDATE transactions SET deleted_at = NOW() WHERE provider_transaction_id = $1 AND deleted_at IS NULL; diff --git a/internal/provider/csv/dedup.go b/internal/provider/csv/dedup.go index 17da67c96..1a73cdf6c 100644 --- a/internal/provider/csv/dedup.go +++ b/internal/provider/csv/dedup.go @@ -13,6 +13,10 @@ import ( // GenerateExternalID creates a deterministic external transaction ID by // hashing SHA-256(accountID|YYYY-MM-DD|amount|lower(trim(description))). +// +// The account id is baked in so this id is stable for idempotent upserts into a +// RESOLVED account: re-importing the same file into the same account reproduces +// identical ids and UpsertTransaction no-ops. func GenerateExternalID(accountID string, date time.Time, amount decimal.Decimal, description string) string { input := fmt.Sprintf("%s|%s|%s|%s", accountID, @@ -23,3 +27,36 @@ func GenerateExternalID(accountID string, date time.Time, amount decimal.Decimal hash := sha256.Sum256([]byte(input)) return fmt.Sprintf("%x", hash[:]) } + +// GenerateExternalIDWithOccurrence is GenerateExternalID with a disambiguator +// appended, used when a user force-imports a row that is byte-for-byte identical +// to one already imported (same date+amount+description). Occurrence 0 is +// identical to GenerateExternalID so the common path stays idempotent. +func GenerateExternalIDWithOccurrence(accountID string, date time.Time, amount decimal.Decimal, description string, occurrence int) string { + if occurrence == 0 { + return GenerateExternalID(accountID, date, amount, description) + } + input := fmt.Sprintf("%s|%s|%s|%s|#%d", + accountID, + date.Format("2006-01-02"), + amount.String(), + strings.ToLower(strings.TrimSpace(description)), + occurrence, + ) + hash := sha256.Sum256([]byte(input)) + return fmt.Sprintf("%x", hash[:]) +} + +// GenerateContentHash creates an account-INDEPENDENT fingerprint of a +// transaction's core content: SHA-256(YYYY-MM-DD|amount|lower(trim(desc))). It +// lets the CSV import classifier detect that a row already exists in a target +// account regardless of which provider (CSV, Plaid, Teller) created it. +func GenerateContentHash(date time.Time, amount decimal.Decimal, description string) string { + input := fmt.Sprintf("%s|%s|%s", + date.Format("2006-01-02"), + amount.String(), + strings.ToLower(strings.TrimSpace(description)), + ) + hash := sha256.Sum256([]byte(input)) + return fmt.Sprintf("%x", hash[:]) +} diff --git a/internal/service/csv_account_match.go b/internal/service/csv_account_match.go new file mode 100644 index 000000000..88b991ce8 --- /dev/null +++ b/internal/service/csv_account_match.go @@ -0,0 +1,307 @@ +//go:build !lite + +package service + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +// Account-detection scoring weights. Deterministic and explainable — every point +// awarded carries a human-readable reason. +const ( + scoreMaskMatch = 40 // file's last-4 matches the account mask + scoreOverlapMax = 40 // proportional to how many rows already exist there + scoreInstitution = 15 // template bank name appears in the account name + scoreFilenameHint = 5 // filename mentions the account + overlapSampleSize = 200 + preselectMinScore = 70 // top must reach this… + preselectMinGap = 20 // …and lead the runner-up by this + confidenceHighScore = 70 + confidenceMediumScore = 40 +) + +// CSVAccountMatch is one ranked account suggestion for an uploaded file. +type CSVAccountMatch struct { + AccountID string `json:"account_id"` + ShortID string `json:"short_id"` + AccountName string `json:"account_name"` + Institution string `json:"institution"` + Mask string `json:"mask"` + Currency string `json:"currency"` + Score int `json:"score"` + Confidence string `json:"confidence"` // high | medium | low + Reasons []string `json:"reasons"` + ProfileMatch bool `json:"profile_match"` +} + +// CSVAccountSuggestion is the full account-detection result: ranked matches plus +// whether one is confident enough to pre-select. +type CSVAccountSuggestion struct { + Matches []CSVAccountMatch `json:"matches"` + Preselect string `json:"preselect"` // account id to auto-select, "" = ask the user + ProfileID string `json:"profile_id"` +} + +// CSVDetectionSignals are the file-derived inputs to account detection. +type CSVDetectionSignals struct { + Filename string + Headers []string + DetectedTemplate string // bank template name, "" if none matched + Mask string // last-4 extracted from the file/filename, "" if none +} + +// MatchCSVAccounts ranks existing accounts for an uploaded file using the file +// signals plus how strongly the parsed rows overlap each account's existing +// transactions. A saved profile (matched by header fingerprint) short-circuits +// to its remembered account. +func (s *Service) MatchCSVAccounts( + ctx context.Context, + userID pgtype.UUID, + signals CSVDetectionSignals, + parsed []CSVParsedRow, +) (*CSVAccountSuggestion, error) { + accounts, err := s.Queries.ListAccounts(ctx) + if err != nil { + return nil, fmt.Errorf("list accounts: %w", err) + } + + // Profile lookup by header fingerprint (the strongest possible signal). + fingerprint := csvpkg.HeaderFingerprint(signals.Headers) + var profileAccountID, profileID string + if prof, err := s.Queries.GetCSVImportProfileByFingerprint(ctx, db.GetCSVImportProfileByFingerprintParams{ + UserID: userID, + HeaderFingerprint: fingerprint, + }); err == nil { + profileID = formatUUID(prof.ID) + if prof.DefaultAccountID.Valid { + profileAccountID = formatUUID(prof.DefaultAccountID) + } + } else if !errors.Is(err, pgx.ErrNoRows) { + s.Logger.Warn("csv account match: profile lookup failed", "error", err) + } + + overlap := s.overlapRatios(ctx, accounts, parsed) + fileTokens := csvpkg.FilenameTokens(signals.Filename) + + matches := make([]CSVAccountMatch, 0, len(accounts)) + for _, a := range accounts { + accID := formatUUID(a.ID) + mask := pgconv.TextOr(a.Mask, "") + institution := pgconv.TextOr(a.InstitutionName, "") + + m := CSVAccountMatch{ + AccountID: accID, + ShortID: a.ShortID, + AccountName: a.Name, + Institution: institution, + Mask: mask, + Currency: pgconv.TextOr(a.IsoCurrencyCode, "USD"), + } + + // Mask / last-4. + if signals.Mask != "" && mask != "" && signals.Mask == mask { + m.Score += scoreMaskMatch + m.Reasons = append(m.Reasons, "card ending "+mask+" matches") + } + + // Transaction overlap. + if ratio := overlap[accID]; ratio > 0 { + pts := int(ratio*float64(scoreOverlapMax) + 0.5) + if pts > 0 { + m.Score += pts + m.Reasons = append(m.Reasons, fmt.Sprintf("%d%% of rows already here", int(ratio*100+0.5))) + } + } + + // Institution name (template bank name appears in the account name). + if signals.DetectedTemplate != "" && institutionOverlap(signals.DetectedTemplate, a.Name, institution) { + m.Score += scoreInstitution + m.Reasons = append(m.Reasons, "institution looks like "+signals.DetectedTemplate) + } + + // Filename mentions the account. + if tokensOverlap(fileTokens, csvpkg.InstitutionTokens(a.Name)) { + m.Score += scoreFilenameHint + m.Reasons = append(m.Reasons, "filename mentions this account") + } + + // Profile-remembered account wins outright. + if profileAccountID != "" && accID == profileAccountID { + m.ProfileMatch = true + m.Score = 100 + m.Reasons = append([]string{"saved import profile"}, m.Reasons...) + } + + m.Confidence = confidenceFor(m.Score) + matches = append(matches, m) + } + + sort.SliceStable(matches, func(i, j int) bool { + if matches[i].Score != matches[j].Score { + return matches[i].Score > matches[j].Score + } + return matches[i].AccountName < matches[j].AccountName + }) + + return &CSVAccountSuggestion{ + Matches: matches, + Preselect: choosePreselect(matches, profileAccountID), + ProfileID: profileID, + }, nil +} + +// overlapRatios returns, per account id, the fraction of sampled parseable rows +// whose (date, amount) already exists in that account. +func (s *Service) overlapRatios(ctx context.Context, accounts []db.ListAccountsRow, parsed []CSVParsedRow) map[string]float64 { + sample := sampleParsedRows(parsed, overlapSampleSize) + if len(sample) == 0 { + return nil + } + + var minDate, maxDate time.Time + for i, r := range sample { + if i == 0 || r.Date.Before(minDate) { + minDate = r.Date + } + if i == 0 || r.Date.After(maxDate) { + maxDate = r.Date + } + } + + keys, err := s.Queries.ListTransactionKeysInRange(ctx, db.ListTransactionKeysInRangeParams{ + Date: pgconv.Date(minDate), + Date_2: pgconv.Date(maxDate), + }) + if err != nil { + s.Logger.Warn("csv account match: overlap query failed", "error", err) + return nil + } + + // Per-account set of date|amount keys present in the DB. + present := map[string]map[string]struct{}{} + for _, k := range keys { + if !k.AccountID.Valid { + continue + } + acc := formatUUID(k.AccountID) + set := present[acc] + if set == nil { + set = map[string]struct{}{} + present[acc] = set + } + set[dateAmountKey(k.Date.Time, numericToDecimal(k.Amount).String())] = struct{}{} + } + + ratios := make(map[string]float64, len(accounts)) + for _, a := range accounts { + acc := formatUUID(a.ID) + set := present[acc] + if len(set) == 0 { + continue + } + matched := 0 + for _, r := range sample { + if _, ok := set[dateAmountKey(r.Date, r.Amount.String())]; ok { + matched++ + } + } + if matched > 0 { + ratios[acc] = float64(matched) / float64(len(sample)) + } + } + return ratios +} + +func dateAmountKey(d time.Time, amount string) string { + return d.Format("2006-01-02") + "|" + amount +} + +// sampleParsedRows returns up to n parseable rows, strided to spread coverage +// across the file rather than clustering at the start. +func sampleParsedRows(parsed []CSVParsedRow, n int) []CSVParsedRow { + valid := make([]CSVParsedRow, 0, len(parsed)) + for _, r := range parsed { + if r.ParseError == "" { + valid = append(valid, r) + } + } + if len(valid) <= n { + return valid + } + out := make([]CSVParsedRow, 0, n) + stride := float64(len(valid)) / float64(n) + for i := 0; i < n; i++ { + out = append(out, valid[int(float64(i)*stride)]) + } + return out +} + +func institutionOverlap(template, name, institution string) bool { + tmplTokens := csvpkg.InstitutionTokens(template) + target := csvpkg.InstitutionTokens(name) + target = append(target, csvpkg.InstitutionTokens(institution)...) + return tokensOverlap(tmplTokens, target) +} + +func tokensOverlap(a, b []string) bool { + if len(a) == 0 || len(b) == 0 { + return false + } + set := make(map[string]struct{}, len(b)) + for _, t := range b { + set[t] = struct{}{} + } + for _, t := range a { + if _, ok := set[t]; ok { + return true + } + } + return false +} + +func confidenceFor(score int) string { + switch { + case score >= confidenceHighScore: + return "high" + case score >= confidenceMediumScore: + return "medium" + default: + return "low" + } +} + +// choosePreselect returns the account id to auto-select, or "" to ask the user. +// A profile-remembered account always pre-selects; otherwise the top match must +// be both strong and a clear leader. +func choosePreselect(matches []CSVAccountMatch, profileAccountID string) string { + if profileAccountID != "" { + for _, m := range matches { + if m.AccountID == profileAccountID { + return profileAccountID + } + } + } + if len(matches) == 0 { + return "" + } + top := matches[0] + second := 0 + if len(matches) > 1 { + second = matches[1].Score + } + if top.Score >= preselectMinScore && top.Score-second >= preselectMinGap { + return top.AccountID + } + return "" +} diff --git a/internal/service/csv_account_match_integration_test.go b/internal/service/csv_account_match_integration_test.go new file mode 100644 index 000000000..73699e8e1 --- /dev/null +++ b/internal/service/csv_account_match_integration_test.go @@ -0,0 +1,123 @@ +//go:build integration && !lite + +package service_test + +import ( + "context" + "testing" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + "breadbox/internal/service" + "breadbox/internal/testutil" +) + +// overlapRows builds CSV rows matching the given (date, amount) pairs. +func overlapRows(pairs [][2]string) [][]string { + rows := make([][]string, len(pairs)) + for i, p := range pairs { + rows[i] = []string{p[0], p[1], "TXN " + p[0]} + } + return rows +} + +func TestAccountMatch_OverlapRanksCorrectAccount(t *testing.T) { + svc, q, _ := newService(t) + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + accA := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Checking A") + accB := testutil.MustCreateAccount(t, q, conn.ID, "extB", "Savings B") + + // Seed account B with three transactions; A gets an unrelated one. + testutil.MustCreateTransaction(t, q, accB.ID, "b1", "TXN 2026-04-01", 1000, "2026-04-01") + testutil.MustCreateTransaction(t, q, accB.ID, "b2", "TXN 2026-04-02", 2000, "2026-04-02") + testutil.MustCreateTransaction(t, q, accB.ID, "b3", "TXN 2026-04-03", 3000, "2026-04-03") + testutil.MustCreateTransaction(t, q, accA.ID, "a1", "OTHER", 9999, "2026-04-01") + + rows := overlapRows([][2]string{{"2026-04-01", "10.00"}, {"2026-04-02", "20.00"}, {"2026-04-03", "30.00"}}) + parsed := service.ParseCSVRows(classifyCfg, rows) + + sug, err := svc.MatchCSVAccounts(context.Background(), user.ID, service.CSVDetectionSignals{}, parsed) + if err != nil { + t.Fatalf("MatchCSVAccounts: %v", err) + } + if len(sug.Matches) == 0 || sug.Matches[0].AccountID != pgconv.FormatUUID(accB.ID) { + t.Fatalf("expected account B on top, got %+v", sug.Matches) + } + if sug.Matches[0].Score < 40 { + t.Errorf("full overlap should score the overlap max, got %d", sug.Matches[0].Score) + } +} + +func TestAccountMatch_PreselectOnMaskPlusOverlap(t *testing.T) { + svc, q, pool := newService(t) + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + acc := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Chase Checking") + // Give the account a mask. + if _, err := pool.Exec(context.Background(), "UPDATE accounts SET mask = '4321' WHERE id = $1", acc.ID); err != nil { + t.Fatalf("set mask: %v", err) + } + testutil.MustCreateTransaction(t, q, acc.ID, "t1", "TXN 2026-05-01", 1000, "2026-05-01") + testutil.MustCreateTransaction(t, q, acc.ID, "t2", "TXN 2026-05-02", 2000, "2026-05-02") + + rows := overlapRows([][2]string{{"2026-05-01", "10.00"}, {"2026-05-02", "20.00"}}) + parsed := service.ParseCSVRows(classifyCfg, rows) + + sug, err := svc.MatchCSVAccounts(context.Background(), user.ID, service.CSVDetectionSignals{Mask: "4321"}, parsed) + if err != nil { + t.Fatalf("MatchCSVAccounts: %v", err) + } + if sug.Preselect != pgconv.FormatUUID(acc.ID) { + t.Fatalf("expected preselect of the matched account, got %q (top score %d)", sug.Preselect, topScore(sug)) + } +} + +func TestAccountMatch_ProfilePreselect(t *testing.T) { + svc, q, _ := newService(t) + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + acc := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Chase Checking") + + headers := []string{"Date", "Amount", "Description"} + fp := csvpkg.HeaderFingerprint(headers) + _, err := q.UpsertCSVImportProfile(context.Background(), db.UpsertCSVImportProfileParams{ + UserID: user.ID, + Name: "Chase CSV", + HeaderFingerprint: fp, + Headers: []byte(`["Date","Amount","Description"]`), + ColumnMapping: []byte(`{"date":0,"amount":1,"description":2}`), + DateFormat: "2006-01-02", + Delimiter: ",", + IsoCurrencyCode: "USD", + DefaultAccountID: acc.ID, + }) + if err != nil { + t.Fatalf("seed profile: %v", err) + } + + rows := overlapRows([][2]string{{"2026-06-01", "10.00"}}) + parsed := service.ParseCSVRows(classifyCfg, rows) + + sug, err := svc.MatchCSVAccounts(context.Background(), user.ID, service.CSVDetectionSignals{Headers: headers}, parsed) + if err != nil { + t.Fatalf("MatchCSVAccounts: %v", err) + } + if sug.ProfileID == "" { + t.Error("expected ProfileID to be set on a fingerprint hit") + } + if sug.Preselect != pgconv.FormatUUID(acc.ID) { + t.Fatalf("profile should preselect its default account, got %q", sug.Preselect) + } + if len(sug.Matches) == 0 || !sug.Matches[0].ProfileMatch { + t.Error("top match should be flagged ProfileMatch") + } +} + +func topScore(s *service.CSVAccountSuggestion) int { + if len(s.Matches) == 0 { + return 0 + } + return s.Matches[0].Score +} diff --git a/internal/service/csv_classify.go b/internal/service/csv_classify.go new file mode 100644 index 000000000..c7b823b45 --- /dev/null +++ b/internal/service/csv_classify.go @@ -0,0 +1,350 @@ +//go:build !lite + +package service + +import ( + "context" + "fmt" + "math/big" + "strings" + "time" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + "breadbox/internal/textmatch" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/shopspring/decimal" +) + +// CSVRowClassification is the per-row verdict produced when an analyzed CSV is +// compared against the transactions already in a resolved target account. +type CSVRowClassification string + +const ( + CSVRowNew CSVRowClassification = "new" // not present — import it (default include) + CSVRowExactDup CSVRowClassification = "exact_dup" // identical row already exists (default exclude) + CSVRowProbableDup CSVRowClassification = "probable_dup" // same date+amount, similar name (default exclude) + CSVRowConflict CSVRowClassification = "conflict" // would materially change an existing row + CSVRowError CSVRowClassification = "error" // unparseable (default exclude) + CSVRowNeedsAccount CSVRowClassification = "needs_account" // no account resolved yet +) + +// DefaultDedupToleranceDays is the +/- day window for the fuzzy duplicate pass. +const DefaultDedupToleranceDays = 1 + +// CSVParsedRow is a single CSV data row after applying the column mapping. +type CSVParsedRow struct { + RowIndex int + Raw []string + Date time.Time + Amount decimal.Decimal + Desc string + Merchant string + Category string + ParseError string // non-empty → classification is forced to error +} + +// CSVClassifiedRow is a parsed row plus its verdict against a target account. +type CSVClassifiedRow struct { + CSVParsedRow + Classification CSVRowClassification + MatchTxnID string // existing transaction this row matched (exact/probable) + MatchScore int // name-similarity score (0-3) for probable matches + MatchReason string + ContentHash string + ProviderTxnID string // stable upsert id, set once the account is known + Include bool // default user intent +} + +// CSVParseConfig captures everything needed to turn raw rows into parsed rows. +type CSVParseConfig struct { + ColumnMapping map[string]int + DateFormat string + PositiveIsDebit bool + HasDebitCredit bool +} + +// ParseCSVRows applies the column mapping to raw rows, producing CSVParsedRow +// values. Rows that fail to parse carry a ParseError instead of being dropped, +// so the preview can show and let the user fix them. This mirrors the per-row +// logic in ImportCSV but never touches the DB. +func ParseCSVRows(cfg CSVParseConfig, rows [][]string) []CSVParsedRow { + dateCol := cfg.ColumnMapping["date"] + amountCol := cfg.ColumnMapping["amount"] + descCol := cfg.ColumnMapping["description"] + catCol, hasCat := cfg.ColumnMapping["category"] + merchantCol, hasMerchant := cfg.ColumnMapping["merchant_name"] + debitCol := cfg.ColumnMapping["debit"] + creditCol := cfg.ColumnMapping["credit"] + + out := make([]CSVParsedRow, 0, len(rows)) + for i, row := range rows { + pr := CSVParsedRow{RowIndex: i, Raw: row} + + maxCol := max(dateCol, descCol) + if cfg.HasDebitCredit { + maxCol = max(maxCol, max(debitCol, creditCol)) + } else { + maxCol = max(maxCol, amountCol) + } + if maxCol >= len(row) { + pr.ParseError = "not enough columns" + out = append(out, pr) + continue + } + + dateVal, err := csvpkg.ParseDate(row[dateCol], cfg.DateFormat) + if err != nil { + pr.ParseError = err.Error() + out = append(out, pr) + continue + } + pr.Date = dateVal + + var amount decimal.Decimal + if cfg.HasDebitCredit { + amount, err = csvpkg.ParseDualColumns(row[debitCol], row[creditCol]) + } else { + amount, err = csvpkg.ParseAmount(row[amountCol]) + } + if err != nil { + pr.ParseError = err.Error() + out = append(out, pr) + continue + } + pr.Amount = csvpkg.NormalizeSign(amount, cfg.PositiveIsDebit) + + desc := strings.TrimSpace(row[descCol]) + if desc == "" { + pr.ParseError = "empty description" + out = append(out, pr) + continue + } + if len(desc) > 500 { + desc = desc[:500] + } + pr.Desc = desc + + if hasCat && catCol < len(row) { + pr.Category = strings.TrimSpace(row[catCol]) + } + if hasMerchant && merchantCol < len(row) { + m := strings.TrimSpace(row[merchantCol]) + if len(m) > 200 { + m = m[:200] + } + pr.Merchant = m + } + + out = append(out, pr) + } + return out +} + +// dedupCandidate is an existing transaction reduced to the fields the classifier +// compares against. +type dedupCandidate struct { + id string + date time.Time + amount decimal.Decimal + name string + merchant string + contentHash string + providerTxnID string +} + +// ClassifyCSVRows compares parsed rows against the live transactions in the +// resolved account and assigns each a classification. accountIDStr is the +// canonical account UUID string (used to derive the stable provider_transaction_id). +func (s *Service) ClassifyCSVRows( + ctx context.Context, + accountID pgtype.UUID, + accountIDStr string, + rows []CSVParsedRow, + toleranceDays int, +) ([]CSVClassifiedRow, error) { + if toleranceDays < 0 { + toleranceDays = 0 + } + + // Date window covering all parseable rows, padded by the tolerance. + var minDate, maxDate time.Time + haveDate := false + for _, r := range rows { + if r.ParseError != "" { + continue + } + if !haveDate || r.Date.Before(minDate) { + minDate = r.Date + } + if !haveDate || r.Date.After(maxDate) { + maxDate = r.Date + } + haveDate = true + } + + // Existing-row indexes. + byProviderTxnID := map[string]dedupCandidate{} + byContentHash := map[string]dedupCandidate{} + byAmount := map[string][]dedupCandidate{} + + if haveDate { + tol := time.Duration(toleranceDays) * 24 * time.Hour + existing, err := s.Queries.ListAccountTransactionsForDedup(ctx, db.ListAccountTransactionsForDedupParams{ + AccountID: accountID, + Date: pgconv.Date(minDate.Add(-tol)), + Date_2: pgconv.Date(maxDate.Add(tol)), + }) + if err != nil { + return nil, fmt.Errorf("load dedup candidates: %w", err) + } + for _, e := range existing { + cand := dedupCandidate{ + id: formatUUID(e.ID), + date: e.Date.Time, + amount: numericToDecimal(e.Amount), + name: e.ProviderName, + merchant: pgconv.TextOr(e.ProviderMerchantName, ""), + contentHash: pgconv.TextOr(e.ContentHash, ""), + providerTxnID: e.ProviderTransactionID, + } + byProviderTxnID[cand.providerTxnID] = cand + if cand.contentHash != "" { + byContentHash[cand.contentHash] = cand + } + byAmount[cand.amount.String()] = append(byAmount[cand.amount.String()], cand) + } + } + + out := make([]CSVClassifiedRow, 0, len(rows)) + for _, pr := range rows { + cr := CSVClassifiedRow{CSVParsedRow: pr} + + if pr.ParseError != "" { + cr.Classification = CSVRowError + cr.Include = false + out = append(out, cr) + continue + } + + cr.ContentHash = csvpkg.GenerateContentHash(pr.Date, pr.Amount, pr.Desc) + cr.ProviderTxnID = csvpkg.GenerateExternalID(accountIDStr, pr.Date, pr.Amount, pr.Desc) + + // Exact duplicate: identical row already imported (same provider id) or + // identical content already present from any source. + if cand, ok := byProviderTxnID[cr.ProviderTxnID]; ok { + cr.Classification = CSVRowExactDup + cr.MatchTxnID = cand.id + cr.MatchReason = "already imported" + cr.Include = false + out = append(out, cr) + continue + } + if cand, ok := byContentHash[cr.ContentHash]; ok { + cr.Classification = CSVRowExactDup + cr.MatchTxnID = cand.id + cr.MatchReason = "identical transaction already exists" + cr.Include = false + out = append(out, cr) + continue + } + + // Probable duplicate: same amount, date within tolerance, name similar. + best, bestScore, bestReason := bestFuzzyMatch(pr, byAmount[pr.Amount.String()], toleranceDays) + if best != nil { + cr.Classification = CSVRowProbableDup + cr.MatchTxnID = best.id + cr.MatchScore = bestScore + cr.MatchReason = bestReason + cr.Include = false + out = append(out, cr) + continue + } + + cr.Classification = CSVRowNew + cr.Include = true + out = append(out, cr) + } + return out, nil +} + +// bestFuzzyMatch finds the strongest same-amount, within-tolerance candidate for +// a parsed row. Returns nil when none qualifies. A candidate with an exact +// date+amount match qualifies even with no name overlap (score 0). +func bestFuzzyMatch(pr CSVParsedRow, sameAmount []dedupCandidate, toleranceDays int) (*dedupCandidate, int, string) { + descLower := strings.ToLower(pr.Desc) + merchLower := strings.ToLower(pr.Merchant) + + var best *dedupCandidate + bestScore := -1 + bestExact := false + var bestReason string + + for i := range sameAmount { + cand := &sameAmount[i] + dayDiff := daysBetween(pr.Date, cand.date) + if dayDiff > toleranceDays { + continue + } + exactDate := dayDiff == 0 + score, fields := textmatch.ScoreLowered(pr.Desc, descLower, pr.Merchant, merchLower, cand.name, cand.merchant) + + // Only a same-date+amount row qualifies on its own with no name overlap. + if score == 0 && !exactDate { + continue + } + + // Prefer higher name score; tie-break toward an exact-date match. + better := score > bestScore || (score == bestScore && exactDate && !bestExact) + if best == nil || better { + best = cand + bestScore = score + bestExact = exactDate + bestReason = fuzzyReason(score, fields, exactDate) + } + } + if best == nil { + return nil, 0, "" + } + return best, bestScore, bestReason +} + +func fuzzyReason(score int, fields []string, exactDate bool) string { + datePart := "same amount" + if exactDate { + datePart = "same date & amount" + } else { + datePart = "same amount, nearby date" + } + if score >= 2 && len(fields) > 0 { + return fmt.Sprintf("%s, %s matches", datePart, strings.ReplaceAll(fields[0], "_", " ")) + } + if score == 1 { + return datePart + ", similar name" + } + return datePart +} + +// daysBetween returns the absolute whole-day difference between two dates. +func daysBetween(a, b time.Time) int { + ay, am, ad := a.Date() + by, bm, bd := b.Date() + da := time.Date(ay, am, ad, 0, 0, 0, 0, time.UTC) + db := time.Date(by, bm, bd, 0, 0, 0, 0, time.UTC) + diff := int(da.Sub(db).Hours() / 24) + if diff < 0 { + diff = -diff + } + return diff +} + +// numericToDecimal converts a pgtype.Numeric to a shopspring Decimal. Invalid / +// NaN / infinity values become zero (they never occur for transaction amounts). +func numericToDecimal(n pgtype.Numeric) decimal.Decimal { + if !n.Valid || n.NaN || n.Int == nil || n.InfinityModifier != pgtype.Finite { + return decimal.Zero + } + return decimal.NewFromBigInt(new(big.Int).Set(n.Int), n.Exp) +} diff --git a/internal/service/csv_classify_integration_test.go b/internal/service/csv_classify_integration_test.go new file mode 100644 index 000000000..3d6593177 --- /dev/null +++ b/internal/service/csv_classify_integration_test.go @@ -0,0 +1,108 @@ +//go:build integration && !lite + +package service_test + +import ( + "context" + "testing" + + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + "breadbox/internal/service" + "breadbox/internal/testutil" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/shopspring/decimal" +) + +// classifyCfg is the column mapping used across these tests: date|amount|desc. +var classifyCfg = service.CSVParseConfig{ + ColumnMapping: map[string]int{"date": 0, "amount": 1, "description": 2}, + DateFormat: "2006-01-02", + PositiveIsDebit: true, // positive already = debit (Breadbox convention) +} + +func classify(t *testing.T, svc *service.Service, acct pgtype.UUID, rows [][]string) []service.CSVClassifiedRow { + t.Helper() + parsed := service.ParseCSVRows(classifyCfg, rows) + out, err := svc.ClassifyCSVRows(context.Background(), acct, pgconv.FormatUUID(acct), parsed, service.DefaultDedupToleranceDays) + if err != nil { + t.Fatalf("ClassifyCSVRows: %v", err) + } + return out +} + +func TestClassify_AllNewIntoEmptyAccount(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + + rows := [][]string{ + {"2026-01-10", "12.50", "STARBUCKS"}, + {"2026-01-11", "40.00", "SHELL GAS"}, + } + got := classify(t, svc, acct, rows) + for _, r := range got { + if r.Classification != service.CSVRowNew || !r.Include { + t.Errorf("row %d: got %s include=%v, want new/included", r.RowIndex, r.Classification, r.Include) + } + } +} + +func TestClassify_ExactDupByProviderID(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + acctStr := pgconv.FormatUUID(acct) + + // Pre-seed a transaction whose provider id is exactly what re-importing the + // same row would generate. + date := testutil.MustParseDate("2026-01-15") + extID := csvpkg.GenerateExternalID(acctStr, date, decimal.RequireFromString("5.00"), "STARBUCKS") + testutil.MustCreateTransaction(t, q, acct, extID, "STARBUCKS", 500, "2026-01-15") + + got := classify(t, svc, acct, [][]string{{"2026-01-15", "5.00", "STARBUCKS"}}) + if got[0].Classification != service.CSVRowExactDup || got[0].Include { + t.Fatalf("got %s include=%v, want exact_dup/excluded", got[0].Classification, got[0].Include) + } +} + +func TestClassify_ProbableDupCrossProvider(t *testing.T) { + svc, q, _ := newService(t) + // A Teller-sourced txn (no content_hash) with same date+amount and similar + // name. Re-importing a CSV that overlaps must flag it as a probable dup. + user := testutil.MustCreateUser(t, q, "Bob") + conn := testutil.MustCreateTellerConnection(t, q, user.ID, "teller_1") + acct := testutil.MustCreateAccount(t, q, conn.ID, "tlr_acct", "Teller Checking") + testutil.MustCreateTransaction(t, q, acct.ID, "teller_txn_1", "STARBUCKS STORE 123", 500, "2026-02-01") + + // Same amount/date, name is a superset — should fuzzy-match. + got := classify(t, svc, acct.ID, [][]string{{"2026-02-01", "5.00", "STARBUCKS"}}) + if got[0].Classification != service.CSVRowProbableDup || got[0].Include { + t.Fatalf("got %s include=%v reason=%q, want probable_dup/excluded", + got[0].Classification, got[0].Include, got[0].MatchReason) + } + if got[0].MatchTxnID == "" { + t.Error("probable dup should record the matched txn id") + } +} + +func TestClassify_NearbyDateNoNameStaysNew(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + // Existing txn: same amount, 5 days away (outside tolerance), different name. + testutil.MustCreateTransaction(t, q, acct, "x1", "WALMART", 500, "2026-03-01") + + got := classify(t, svc, acct, [][]string{{"2026-03-06", "5.00", "TARGET"}}) + if got[0].Classification != service.CSVRowNew { + t.Fatalf("got %s, want new (amount matches but date far + name differs)", got[0].Classification) + } +} + +func TestClassify_ParseErrorRow(t *testing.T) { + svc, q, _ := newService(t) + acct := seedTxnFixture(t, q) + got := classify(t, svc, acct, [][]string{{"not-a-date", "5.00", "X"}}) + if got[0].Classification != service.CSVRowError || got[0].Include { + t.Fatalf("got %s include=%v, want error/excluded", got[0].Classification, got[0].Include) + } +} + From 18055812a8040c7204b6149d075c2ffdfea4a484 Mon Sep 17 00:00:00 2001 From: Ricardo Canales Date: Sat, 6 Jun 2026 21:56:09 -0700 Subject: [PATCH 3/7] =?UTF-8?q?feat(csv-import):=20Wave=202=20=E2=80=94=20?= =?UTF-8?q?session=20lifecycle=20+=20atomic=20idempotent=20apply?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit internal/service/csv_import_v2.go — the durable staging flow: - CreateImportSession: parse file, detect template/mapping/date-format/currency, apply a saved profile (auto mapping + auto-resolve to its account), run account detection, persist the session + rows. Auto-classifies when an account is confidently chosen; otherwise stages rows as needs_account. - ResolveImportAccount: bind an existing account or mint a new CSV connection+ account, reclassify every row, status → previewed. - ListImportRows: server-paginated rows + per-classification summary counts. - EditImportRow: inline-edit a row and reclassify just it. - BulkImportOp: include/exclude by classification, set-category, remap (re-parse + reclassify all). - ApplyImportSession: one tx — upsert the exact included set via UpsertTransactionV2 (reliable insert/update counts), occurrence-disambiguate genuine same-key duplicates, stamp session currency, write a sync_log, upsert the source profile, finalize (snapshot result, drop blob). Idempotent: a re-drop into the same account upserts to no-ops. New row column category_id (migration) for user-chosen Breadbox category applied at import; CountIncludedCSVImportRows query. Integration tests: full lifecycle, profile-driven re-drop (dedup 2 + add 1), double-import idempotency, account-change-before-confirm. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...0260607043618_csv_import_rows_category.sql | 11 + internal/db/queries/csv_import_rows.sql | 15 +- internal/service/csv_import_v2.go | 1033 +++++++++++++++++ .../service/csv_import_v2_integration_test.go | 178 +++ 4 files changed, 1234 insertions(+), 3 deletions(-) create mode 100644 internal/db/migrations/20260607043618_csv_import_rows_category.sql create mode 100644 internal/service/csv_import_v2.go create mode 100644 internal/service/csv_import_v2_integration_test.go diff --git a/internal/db/migrations/20260607043618_csv_import_rows_category.sql b/internal/db/migrations/20260607043618_csv_import_rows_category.sql new file mode 100644 index 000000000..bd4dcd575 --- /dev/null +++ b/internal/db/migrations/20260607043618_csv_import_rows_category.sql @@ -0,0 +1,11 @@ +-- +goose Up + +-- category_id: the Breadbox category a user assigns to a staged row (via the +-- preview's bulk "set category" or inline edit). NULL = let transaction rules +-- categorize at/after import, matching the legacy CSV behavior. Distinct from +-- parsed_category, which is the raw provider category string from the file. +ALTER TABLE csv_import_rows + ADD COLUMN IF NOT EXISTS category_id UUID NULL REFERENCES categories (id) ON DELETE SET NULL; + +-- +goose Down +ALTER TABLE csv_import_rows DROP COLUMN IF EXISTS category_id; diff --git a/internal/db/queries/csv_import_rows.sql b/internal/db/queries/csv_import_rows.sql index 6befd4dc5..04f15ee45 100644 --- a/internal/db/queries/csv_import_rows.sql +++ b/internal/db/queries/csv_import_rows.sql @@ -4,9 +4,10 @@ INSERT INTO csv_import_rows ( session_id, row_index, raw, parsed_date, parsed_amount, parsed_desc, parsed_merchant, parsed_category, classification, match_txn_id, match_score, - match_reason, parse_error, content_hash, provider_txn_id, include, user_edited + match_reason, parse_error, content_hash, provider_txn_id, include, user_edited, + category_id ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18 ); -- name: DeleteCSVImportRows :exec @@ -29,6 +30,13 @@ FROM csv_import_rows WHERE session_id = $1 GROUP BY classification; +-- name: CountIncludedCSVImportRows :one +SELECT COUNT(*) FROM csv_import_rows +WHERE session_id = $1 + AND include = TRUE + AND classification <> 'error' + AND classification <> 'needs_account'; + -- name: ListIncludedCSVImportRows :many -- The exact set that apply will upsert: included, non-error rows in row order. SELECT * FROM csv_import_rows @@ -54,6 +62,7 @@ SET parsed_date = $2, content_hash = $12, provider_txn_id = $13, include = $14, + category_id = $15, user_edited = TRUE WHERE id = $1 RETURNING *; @@ -69,5 +78,5 @@ WHERE session_id = $1 AND classification = $2; -- name: SetCSVImportRowsCategoryAll :exec -- Bulk "set category" across all included rows in the session. UPDATE csv_import_rows -SET parsed_category = $2, user_edited = TRUE +SET category_id = $2, user_edited = TRUE WHERE session_id = $1 AND include = TRUE; diff --git a/internal/service/csv_import_v2.go b/internal/service/csv_import_v2.go new file mode 100644 index 000000000..51c1220dd --- /dev/null +++ b/internal/service/csv_import_v2.go @@ -0,0 +1,1033 @@ +//go:build !lite + +package service + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "breadbox/internal/db" + "breadbox/internal/pgconv" + csvpkg "breadbox/internal/provider/csv" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/shopspring/decimal" +) + +// importSessionTTL is how long an un-applied session lives before the sweeper +// may reap it. +const importSessionTTL = 24 * time.Hour + +// Session status values. +const ( + importStatusAnalyzing = "analyzing" + importStatusAwaitingAccount = "awaiting_account" + importStatusPreviewed = "previewed" + importStatusApplied = "applied" +) + +// ErrImportSessionNotFound is returned when a session id/short id doesn't resolve. +var ErrImportSessionNotFound = errors.New("import session not found") + +// ImportSession is the service-layer view of a csv_import_sessions row. +type ImportSession struct { + ID string `json:"id"` + ShortID string `json:"short_id"` + UserID string `json:"user_id"` + Status string `json:"status"` + Filename string `json:"filename"` + Delimiter string `json:"delimiter"` + Headers []string `json:"headers"` + RowCount int `json:"row_count"` + ResolvedAccountID string `json:"resolved_account_id"` + ResolvedConnectionID string `json:"resolved_connection_id"` + DetectedTemplate string `json:"detected_template"` + ColumnMapping map[string]int `json:"column_mapping"` + DateFormat string `json:"date_format"` + PositiveIsDebit bool `json:"positive_is_debit"` + HasDebitCredit bool `json:"has_debit_credit"` + IsoCurrencyCode string `json:"iso_currency_code"` + ProfileID string `json:"profile_id"` + Result *CSVImportResult `json:"result,omitempty"` +} + +// ImportRowView is the service-layer view of one staged row. +type ImportRowView struct { + ID string `json:"id"` + RowIndex int `json:"row_index"` + Raw []string `json:"raw"` + Date string `json:"date"` + Amount string `json:"amount"` + Desc string `json:"description"` + Merchant string `json:"merchant"` + Category string `json:"category"` + Classification string `json:"classification"` + MatchTxnID string `json:"match_txn_id"` + MatchScore int `json:"match_score"` + MatchReason string `json:"match_reason"` + ParseError string `json:"parse_error"` + Include bool `json:"include"` + UserEdited bool `json:"user_edited"` +} + +// ImportSummary is the per-classification breakdown for a session. +type ImportSummary struct { + Total int `json:"total"` + Counts map[string]int `json:"counts"` + IncludedCount int `json:"included_count"` +} + +// ImportAnalysis is returned from CreateImportSession: the session plus account +// suggestions plus a first summary. +type ImportAnalysis struct { + Session *ImportSession `json:"session"` + Accounts *CSVAccountSuggestion `json:"accounts"` + Summary ImportSummary `json:"summary"` +} + +// CreateImportSessionParams are the inputs for analyzing a dropped file. +type CreateImportSessionParams struct { + UserID string + Filename string + Data []byte +} + +// CreateImportSession parses + analyzes a dropped CSV, persists a staging +// session, applies any saved profile, runs account detection, and (when an +// account can be confidently chosen) classifies rows so the preview is ready. +func (s *Service) CreateImportSession(ctx context.Context, p CreateImportSessionParams) (*ImportAnalysis, error) { + userID, err := s.resolveUserID(ctx, p.UserID) + if err != nil { + return nil, fmt.Errorf("invalid user: %w", err) + } + + pf, err := csvpkg.ParseFile(p.Data) + if err != nil { + return nil, fmt.Errorf("parse csv: %w", err) + } + + cfg, detectedTemplate := detectParseConfig(pf) + currency := detectCurrency(pf.Rows, cfg.ColumnMapping) + + // Apply a saved profile (keyed by header fingerprint), if any. + fingerprint := csvpkg.HeaderFingerprint(pf.Headers) + var profileID pgtype.UUID + if prof, perr := s.Queries.GetCSVImportProfileByFingerprint(ctx, db.GetCSVImportProfileByFingerprintParams{ + UserID: userID, + HeaderFingerprint: fingerprint, + }); perr == nil { + profileID = prof.ID + if m := decodeMapping(prof.ColumnMapping); len(m) > 0 { + cfg.ColumnMapping = m + } + cfg.DateFormat = prof.DateFormat + cfg.PositiveIsDebit = prof.PositiveIsDebit + cfg.HasDebitCredit = prof.HasDebitCredit + if prof.IsoCurrencyCode != "" { + currency = prof.IsoCurrencyCode + } + if prof.DetectedTemplate.Valid { + detectedTemplate = prof.DetectedTemplate.String + } + } else if !errors.Is(perr, pgx.ErrNoRows) { + s.Logger.Warn("csv import: profile lookup failed", "error", perr) + } + + headersJSON, _ := json.Marshal(pf.Headers) + mappingJSON, _ := json.Marshal(cfg.ColumnMapping) + sum := sha256.Sum256(p.Data) + + sess, err := s.Queries.CreateCSVImportSession(ctx, db.CreateCSVImportSessionParams{ + UserID: userID, + Status: importStatusAnalyzing, + Filename: p.Filename, + FileSha256: hex.EncodeToString(sum[:]), + Delimiter: string(pf.Delimiter), + Headers: headersJSON, + RawBlob: p.Data, + RowCount: int32(len(pf.Rows)), + DetectedTemplate: pgconv.TextIfNotEmpty(detectedTemplate), + ColumnMapping: mappingJSON, + DateFormat: cfg.DateFormat, + PositiveIsDebit: cfg.PositiveIsDebit, + HasDebitCredit: cfg.HasDebitCredit, + IsoCurrencyCode: currency, + ProfileID: profileID, + ExpiresAt: pgconv.Timestamptz(time.Now().Add(importSessionTTL)), + }) + if err != nil { + return nil, fmt.Errorf("create session: %w", err) + } + + parsed := ParseCSVRows(cfg, pf.Rows) + + // Account detection. + suggestion, err := s.MatchCSVAccounts(ctx, userID, CSVDetectionSignals{ + Filename: p.Filename, + Headers: pf.Headers, + DetectedTemplate: detectedTemplate, + Mask: csvpkg.ExtractMask(p.Filename, pf.Headers, sampleRaw(pf.Rows, 50)), + }, parsed) + if err != nil { + return nil, err + } + + // If we can confidently pick an account, resolve + classify now so the + // preview is immediately ready; otherwise stage the rows pending an account. + if suggestion.Preselect != "" { + if _, err := s.resolveAndClassify(ctx, sess, suggestion.Preselect, currency, cfg, pf.Rows); err != nil { + return nil, err + } + } else { + if err := s.stageNeedsAccount(ctx, sess.ID, cfg, pf.Rows); err != nil { + return nil, err + } + _ = s.Queries.UpdateCSVImportSessionStatus(ctx, db.UpdateCSVImportSessionStatusParams{ + ID: sess.ID, + Status: importStatusAwaitingAccount, + }) + } + + out, err := s.GetImportSession(ctx, sess.ShortID) + if err != nil { + return nil, err + } + summary, err := s.importSummary(ctx, sess.ID) + if err != nil { + return nil, err + } + return &ImportAnalysis{Session: out, Accounts: suggestion, Summary: summary}, nil +} + +// ResolveImportAccountParams selects (or creates) the target account. +type ResolveImportAccountParams struct { + AccountID string // existing account (UUID or short id) + + // Create-new (used when AccountID is empty). + CreateNew bool + NewName string + NewType string + NewSubtype string + NewCurrency string +} + +// ResolveImportAccount binds the session to an account (existing or newly +// created) and reclassifies every row against it. Status → previewed. +func (s *Service) ResolveImportAccount(ctx context.Context, sessionIDOrShort string, p ResolveImportAccountParams) (*ImportSession, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + rows, err := s.rawRows(sess) + if err != nil { + return nil, err + } + cfg := sessionParseConfig(sess) + + accountID := p.AccountID + currency := sess.IsoCurrencyCode + if p.CreateNew { + name := strings.TrimSpace(p.NewName) + if name == "" { + name = "CSV Import" + } + cur := p.NewCurrency + if cur == "" { + cur = currency + } + _, acctID, err := s.createCSVConnectionAccount(ctx, sess.UserID, name, cur, orDefault(p.NewType, "depository"), orDefault(p.NewSubtype, "checking")) + if err != nil { + return nil, err + } + accountID = formatUUID(acctID) + currency = cur + } else { + // Adopt the chosen account's currency when the file didn't detect one. + if accID, err := s.resolveAccountID(ctx, accountID); err == nil { + if acc, err := s.Queries.GetAccount(ctx, accID); err == nil { + if c := pgconv.TextOr(acc.IsoCurrencyCode, ""); c != "" { + currency = c + } + } + } + } + + if _, err := s.resolveAndClassify(ctx, sess, accountID, currency, cfg, rows); err != nil { + return nil, err + } + return s.GetImportSession(ctx, sess.ShortID) +} + +// resolveAndClassify binds the account on the session, classifies all rows +// against it, and persists them. Returns the number of rows persisted. +func (s *Service) resolveAndClassify(ctx context.Context, sess db.CsvImportSession, accountIDOrStr, currency string, cfg CSVParseConfig, rows [][]string) (int, error) { + accountID, err := s.resolveAccountID(ctx, accountIDOrStr) + if err != nil { + return 0, fmt.Errorf("resolve account: %w", err) + } + acctStr := formatUUID(accountID) + + acc, err := s.Queries.GetAccount(ctx, accountID) + if err != nil { + return 0, fmt.Errorf("get account: %w", err) + } + + parsed := ParseCSVRows(cfg, rows) + classified, err := s.ClassifyCSVRows(ctx, accountID, acctStr, parsed, DefaultDedupToleranceDays) + if err != nil { + return 0, err + } + + tx, err := s.Pool.Begin(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback(ctx) + q := s.Queries.WithTx(tx) + + if _, err := q.ResolveCSVImportSessionAccount(ctx, db.ResolveCSVImportSessionAccountParams{ + ID: sess.ID, + ResolvedAccountID: accountID, + ResolvedConnectionID: acc.ConnectionID, + IsoCurrencyCode: currency, + Status: importStatusPreviewed, + }); err != nil { + return 0, err + } + if err := q.DeleteCSVImportRows(ctx, sess.ID); err != nil { + return 0, err + } + if _, err := q.CreateCSVImportRows(ctx, classifiedToParams(sess.ID, classified)); err != nil { + return 0, err + } + if err := tx.Commit(ctx); err != nil { + return 0, err + } + return len(classified), nil +} + +// stageNeedsAccount persists parsed rows in the needs_account state (no account +// resolved yet) so the row count + parse errors are visible before resolution. +func (s *Service) stageNeedsAccount(ctx context.Context, sessionID pgtype.UUID, cfg CSVParseConfig, rows [][]string) error { + parsed := ParseCSVRows(cfg, rows) + staged := make([]CSVClassifiedRow, 0, len(parsed)) + for _, pr := range parsed { + cr := CSVClassifiedRow{CSVParsedRow: pr, Include: pr.ParseError == ""} + if pr.ParseError != "" { + cr.Classification = CSVRowError + } else { + cr.Classification = CSVRowNeedsAccount + cr.ContentHash = csvpkg.GenerateContentHash(pr.Date, pr.Amount, pr.Desc) + } + staged = append(staged, cr) + } + tx, err := s.Pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + q := s.Queries.WithTx(tx) + if err := q.DeleteCSVImportRows(ctx, sessionID); err != nil { + return err + } + if _, err := q.CreateCSVImportRows(ctx, classifiedToParams(sessionID, staged)); err != nil { + return err + } + return tx.Commit(ctx) +} + +// ListImportRows returns a page of staged rows plus the full per-classification +// summary. classFilter "" returns all classifications. +func (s *Service) ListImportRows(ctx context.Context, sessionIDOrShort, classFilter string, page, pageSize int) ([]ImportRowView, ImportSummary, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, ImportSummary{}, err + } + if page < 1 { + page = 1 + } + if pageSize <= 0 || pageSize > 500 { + pageSize = 100 + } + rows, err := s.Queries.GetCSVImportRowsPage(ctx, db.GetCSVImportRowsPageParams{ + SessionID: sess.ID, + Column2: classFilter, + Limit: int32(pageSize), + Offset: int32((page - 1) * pageSize), + }) + if err != nil { + return nil, ImportSummary{}, err + } + views := make([]ImportRowView, len(rows)) + for i, r := range rows { + views[i] = rowToView(r) + } + summary, err := s.importSummary(ctx, sess.ID) + if err != nil { + return nil, ImportSummary{}, err + } + return views, summary, nil +} + +// EditImportRowParams carries the editable fields of a single row. +type EditImportRowParams struct { + Date string // "2006-01-02" + Amount string // decimal string + Desc string + Merchant string + Include *bool +} + +// EditImportRow updates a single row's parsed fields and reclassifies just that +// row against the resolved account. +func (s *Service) EditImportRow(ctx context.Context, sessionIDOrShort, rowID string, p EditImportRowParams) (*ImportRowView, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + if !sess.ResolvedAccountID.Valid { + return nil, errors.New("resolve an account before editing rows") + } + rowUUID, err := pgconv.ParseUUID(rowID) + if err != nil { + return nil, fmt.Errorf("invalid row id: %w", err) + } + existing, err := s.Queries.GetCSVImportRow(ctx, rowUUID) + if err != nil { + return nil, fmt.Errorf("get row: %w", err) + } + + pr := CSVParsedRow{RowIndex: int(existing.RowIndex)} + _ = json.Unmarshal(existing.Raw, &pr.Raw) + + date, derr := time.Parse("2006-01-02", strings.TrimSpace(p.Date)) + amount, aerr := decimal.NewFromString(strings.TrimSpace(p.Amount)) + pr.Desc = strings.TrimSpace(p.Desc) + pr.Merchant = strings.TrimSpace(p.Merchant) + switch { + case derr != nil: + pr.ParseError = "unparseable date" + case aerr != nil: + pr.ParseError = "unparseable amount" + case pr.Desc == "": + pr.ParseError = "empty description" + default: + pr.Date, pr.Amount = date, amount + } + + acctStr := formatUUID(sess.ResolvedAccountID) + classified, err := s.ClassifyCSVRows(ctx, sess.ResolvedAccountID, acctStr, []CSVParsedRow{pr}, DefaultDedupToleranceDays) + if err != nil { + return nil, err + } + cr := classified[0] + + include := cr.Include + if p.Include != nil { + include = *p.Include + } + + updated, err := s.Queries.UpdateCSVImportRow(ctx, db.UpdateCSVImportRowParams{ + ID: rowUUID, + ParsedDate: dateOrNull(cr.Date, cr.ParseError == ""), + ParsedAmount: numericOrNull(cr.Amount, cr.ParseError == ""), + ParsedDesc: pgconv.TextIfNotEmpty(cr.Desc), + ParsedMerchant: pgconv.TextIfNotEmpty(cr.Merchant), + ParsedCategory: pgconv.TextIfNotEmpty(cr.Category), + Classification: string(cr.Classification), + MatchTxnID: optionalUUID(cr.MatchTxnID), + MatchScore: pgconv.Int4(int32(cr.MatchScore)), + MatchReason: pgconv.TextIfNotEmpty(cr.MatchReason), + ParseError: pgconv.TextIfNotEmpty(cr.ParseError), + ContentHash: pgconv.TextIfNotEmpty(cr.ContentHash), + ProviderTxnID: pgconv.TextIfNotEmpty(cr.ProviderTxnID), + Include: include, + CategoryID: existing.CategoryID, + }) + if err != nil { + return nil, err + } + v := rowToView(updated) + return &v, nil +} + +// ImportBulkOp describes a bulk preview action. +type ImportBulkOp struct { + Op string // "include" | "exclude" | "set_category" | "remap" + Classification string // for include/exclude scoping ("" = all dup classes) + CategoryID string // for set_category + + // for remap + ColumnMapping map[string]int + DateFormat string + PositiveIsDebit bool + HasDebitCredit bool +} + +// BulkImportOp applies a bulk action to a session's staged rows. +func (s *Service) BulkImportOp(ctx context.Context, sessionIDOrShort string, op ImportBulkOp) error { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return err + } + switch op.Op { + case "include", "exclude": + include := op.Op == "include" + classes := []string{op.Classification} + if op.Classification == "" { + classes = []string{string(CSVRowExactDup), string(CSVRowProbableDup)} + } + for _, c := range classes { + if err := s.Queries.SetCSVImportRowsIncludeByClassification(ctx, db.SetCSVImportRowsIncludeByClassificationParams{ + SessionID: sess.ID, + Classification: c, + Include: include, + }); err != nil { + return err + } + } + return nil + case "set_category": + var catID pgtype.UUID + if op.CategoryID != "" { + id, err := s.resolveCategoryID(ctx, op.CategoryID) + if err != nil { + return fmt.Errorf("resolve category: %w", err) + } + catID = id + } + return s.Queries.SetCSVImportRowsCategoryAll(ctx, db.SetCSVImportRowsCategoryAllParams{ + SessionID: sess.ID, + CategoryID: catID, + }) + case "remap": + if !sess.ResolvedAccountID.Valid { + return errors.New("resolve an account before remapping") + } + mappingJSON, _ := json.Marshal(op.ColumnMapping) + if _, err := s.Queries.UpdateCSVImportSessionParse(ctx, db.UpdateCSVImportSessionParseParams{ + ID: sess.ID, + ColumnMapping: mappingJSON, + DateFormat: op.DateFormat, + PositiveIsDebit: op.PositiveIsDebit, + HasDebitCredit: op.HasDebitCredit, + DetectedTemplate: sess.DetectedTemplate, + }); err != nil { + return err + } + fresh, err := s.Queries.GetCSVImportSession(ctx, sess.ID) + if err != nil { + return err + } + rows, err := s.rawRows(fresh) + if err != nil { + return err + } + _, err = s.resolveAndClassify(ctx, fresh, formatUUID(fresh.ResolvedAccountID), fresh.IsoCurrencyCode, sessionParseConfig(fresh), rows) + return err + default: + return fmt.Errorf("unknown bulk op %q", op.Op) + } +} + +// ApplyImportSession upserts the exact included set into the resolved account in +// one transaction, idempotently, and saves/updates the source profile. +func (s *Service) ApplyImportSession(ctx context.Context, sessionIDOrShort string, actor Actor) (*CSVImportResult, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + if sess.Status == importStatusApplied { + return nil, errors.New("session already applied") + } + if !sess.ResolvedAccountID.Valid || !sess.ResolvedConnectionID.Valid { + return nil, errors.New("resolve an account before applying") + } + acctStr := formatUUID(sess.ResolvedAccountID) + + included, err := s.Queries.ListIncludedCSVImportRows(ctx, sess.ID) + if err != nil { + return nil, err + } + + var uncategorizedID pgtype.UUID + _ = s.Pool.QueryRow(ctx, "SELECT id FROM categories WHERE slug = 'uncategorized'").Scan(&uncategorizedID) + + tx, err := s.Pool.Begin(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) + q := s.Queries.WithTx(tx) + + now := time.Now().UTC() + syncLog, err := q.CreateSyncLog(ctx, db.CreateSyncLogParams{ + ConnectionID: sess.ResolvedConnectionID, + Trigger: db.SyncTriggerManual, + Status: db.SyncStatusInProgress, + StartedAt: pgconv.Timestamptz(now), + }) + if err != nil { + return nil, fmt.Errorf("create sync log: %w", err) + } + + result := &CSVImportResult{ + ConnectionID: formatUUID(sess.ResolvedConnectionID), + AccountID: acctStr, + TotalRows: int(sess.RowCount), + } + + seen := map[string]int{} // provider_txn_id base → next occurrence + for _, r := range included { + amount := numericToDecimal(r.ParsedAmount) + desc := pgconv.TextOr(r.ParsedDesc, "") + date := r.ParsedDate.Time + + // Disambiguate genuine same-key duplicates so "import anyway" inserts a + // distinct row instead of no-op upserting onto the original. + base := pgconv.TextOr(r.ProviderTxnID, csvpkg.GenerateExternalID(acctStr, date, amount, desc)) + occ := seen[base] + if r.Classification == string(CSVRowExactDup) && occ == 0 { + occ = 1 // explicit re-import of an identical row → force a new id + } + seen[base] = occ + 1 + providerTxnID := csvpkg.GenerateExternalIDWithOccurrence(acctStr, date, amount, desc, occ) + + categoryID := uncategorizedID + if r.CategoryID.Valid { + categoryID = r.CategoryID + } + + var amountNumeric pgtype.Numeric + _ = amountNumeric.Scan(amount.String()) + + up, err := q.UpsertTransactionV2(ctx, db.UpsertTransactionV2Params{ + AccountID: sess.ResolvedAccountID, + ProviderTransactionID: providerTxnID, + Amount: amountNumeric, + IsoCurrencyCode: pgconv.Text(sess.IsoCurrencyCode), + Date: pgconv.Date(date), + ProviderName: desc, + ProviderMerchantName: r.ParsedMerchant, + ProviderCategoryPrimary: r.ParsedCategory, + ProviderPaymentChannel: pgconv.Text("other"), + Pending: false, + CategoryID: categoryID, + ProviderRaw: r.Raw, + ContentHash: pgconv.TextIfNotEmpty(pgconv.TextOr(r.ContentHash, csvpkg.GenerateContentHash(date, amount, desc))), + }) + if err != nil { + result.SkippedCount++ + result.SkipReasons = append(result.SkipReasons, fmt.Sprintf("row %d: %s", r.RowIndex+2, err.Error())) + continue + } + if up.Inserted { + result.NewCount++ + } else { + result.UpdatedCount++ + } + } + + syncStatus := db.SyncStatusSuccess + var errMsg pgtype.Text + if result.NewCount+result.UpdatedCount == 0 && result.SkippedCount > 0 { + syncStatus = db.SyncStatusError + errMsg = pgconv.Text(fmt.Sprintf("all %d rows skipped", result.SkippedCount)) + } + if err := q.UpdateSyncLog(ctx, db.UpdateSyncLogParams{ + ID: syncLog.ID, + Status: syncStatus, + CompletedAt: pgconv.Timestamptz(time.Now().UTC()), + AddedCount: int32(result.NewCount), + ModifiedCount: int32(result.UpdatedCount), + RemovedCount: 0, + ErrorMessage: errMsg, + }); err != nil { + s.Logger.Error("csv import: update sync log", "error", err) + } + + // Save / refresh the source profile so future imports of this layout are + // one click. Best-effort — never fails the apply. + if err := s.upsertProfileFromSession(ctx, q, sess); err != nil { + s.Logger.Warn("csv import: profile upsert failed", "error", err) + } + + resultJSON, _ := json.Marshal(result) + if err := q.FinalizeCSVImportSession(ctx, db.FinalizeCSVImportSessionParams{ + ID: sess.ID, + Result: resultJSON, + SyncLogID: syncLog.ID, + }); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return result, nil +} + +// GetImportSession returns the service view of a session by id or short id. +func (s *Service) GetImportSession(ctx context.Context, sessionIDOrShort string) (*ImportSession, error) { + sess, err := s.getSession(ctx, sessionIDOrShort) + if err != nil { + return nil, err + } + v := sessionToView(sess) + return &v, nil +} + +// --- helpers --- + +func (s *Service) getSession(ctx context.Context, idOrShort string) (db.CsvImportSession, error) { + if len(idOrShort) == 8 { + sess, err := s.Queries.GetCSVImportSessionByShortID(ctx, idOrShort) + if err != nil { + return db.CsvImportSession{}, ErrImportSessionNotFound + } + return sess, nil + } + uid, err := pgconv.ParseUUID(idOrShort) + if err != nil { + return db.CsvImportSession{}, ErrImportSessionNotFound + } + sess, err := s.Queries.GetCSVImportSession(ctx, uid) + if err != nil { + return db.CsvImportSession{}, ErrImportSessionNotFound + } + return sess, nil +} + +func (s *Service) importSummary(ctx context.Context, sessionID pgtype.UUID) (ImportSummary, error) { + counts, err := s.Queries.CountCSVImportRowsByClassification(ctx, sessionID) + if err != nil { + return ImportSummary{}, err + } + out := ImportSummary{Counts: map[string]int{}} + for _, c := range counts { + out.Counts[c.Classification] = int(c.Count) + out.Total += int(c.Count) + } + included, err := s.Queries.CountIncludedCSVImportRows(ctx, sessionID) + if err != nil { + return ImportSummary{}, err + } + out.IncludedCount = int(included) + return out, nil +} + +// rawRows re-parses the stored file blob into rows. +func (s *Service) rawRows(sess db.CsvImportSession) ([][]string, error) { + if len(sess.RawBlob) == 0 { + return nil, errors.New("session file is no longer available") + } + pf, err := csvpkg.ParseFile(sess.RawBlob) + if err != nil { + return nil, fmt.Errorf("re-parse csv: %w", err) + } + return pf.Rows, nil +} + +// createCSVConnectionAccount mints a fresh CSV connection + account (generalized +// from the legacy ImportCSV new-import branch). +func (s *Service) createCSVConnectionAccount(ctx context.Context, userID pgtype.UUID, name, currency, accType, subtype string) (pgtype.UUID, pgtype.UUID, error) { + b := make([]byte, 16) + _, _ = rand.Read(b) + externalID := fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) + + conn, err := s.Queries.CreateBankConnection(ctx, db.CreateBankConnectionParams{ + UserID: userID, + Provider: db.ProviderTypeCsv, + InstitutionName: pgconv.Text(name), + ExternalID: pgconv.Text(externalID), + Status: db.ConnectionStatusActive, + }) + if err != nil { + return pgtype.UUID{}, pgtype.UUID{}, fmt.Errorf("create connection: %w", err) + } + acct, err := s.Queries.UpsertAccount(ctx, db.UpsertAccountParams{ + ConnectionID: conn.ID, + ExternalAccountID: externalID, + Name: name, + Type: accType, + Subtype: pgconv.Text(subtype), + IsoCurrencyCode: pgconv.Text(currency), + }) + if err != nil { + return pgtype.UUID{}, pgtype.UUID{}, fmt.Errorf("create account: %w", err) + } + return conn.ID, acct.ID, nil +} + +func (s *Service) upsertProfileFromSession(ctx context.Context, q *db.Queries, sess db.CsvImportSession) error { + headers := decodeHeaders(sess.Headers) + name := strings.TrimSpace(sess.Filename) + if name == "" { + name = "CSV import" + } + _, err := q.UpsertCSVImportProfile(ctx, db.UpsertCSVImportProfileParams{ + UserID: sess.UserID, + Name: name, + HeaderFingerprint: csvpkg.HeaderFingerprint(headers), + Headers: sess.Headers, + DetectedTemplate: sess.DetectedTemplate, + ColumnMapping: sess.ColumnMapping, + DateFormat: sess.DateFormat, + Delimiter: sess.Delimiter, + PositiveIsDebit: sess.PositiveIsDebit, + HasDebitCredit: sess.HasDebitCredit, + IsoCurrencyCode: sess.IsoCurrencyCode, + DefaultAccountID: sess.ResolvedAccountID, + }) + return err +} + +func classifiedToParams(sessionID pgtype.UUID, rows []CSVClassifiedRow) []db.CreateCSVImportRowsParams { + out := make([]db.CreateCSVImportRowsParams, len(rows)) + for i, cr := range rows { + rawJSON, _ := json.Marshal(cr.Raw) + ok := cr.ParseError == "" + out[i] = db.CreateCSVImportRowsParams{ + SessionID: sessionID, + RowIndex: int32(cr.RowIndex), + Raw: rawJSON, + ParsedDate: dateOrNull(cr.Date, ok), + ParsedAmount: numericOrNull(cr.Amount, ok), + ParsedDesc: pgconv.TextIfNotEmpty(cr.Desc), + ParsedMerchant: pgconv.TextIfNotEmpty(cr.Merchant), + ParsedCategory: pgconv.TextIfNotEmpty(cr.Category), + Classification: string(cr.Classification), + MatchTxnID: optionalUUID(cr.MatchTxnID), + MatchScore: pgconv.Int4(int32(cr.MatchScore)), + MatchReason: pgconv.TextIfNotEmpty(cr.MatchReason), + ParseError: pgconv.TextIfNotEmpty(cr.ParseError), + ContentHash: pgconv.TextIfNotEmpty(cr.ContentHash), + ProviderTxnID: pgconv.TextIfNotEmpty(cr.ProviderTxnID), + Include: cr.Include, + UserEdited: false, + } + } + return out +} + +func rowToView(r db.CsvImportRow) ImportRowView { + var raw []string + _ = json.Unmarshal(r.Raw, &raw) + v := ImportRowView{ + ID: formatUUID(r.ID), + RowIndex: int(r.RowIndex), + Raw: raw, + Desc: pgconv.TextOr(r.ParsedDesc, ""), + Merchant: pgconv.TextOr(r.ParsedMerchant, ""), + Category: pgconv.TextOr(r.ParsedCategory, ""), + Classification: r.Classification, + MatchReason: pgconv.TextOr(r.MatchReason, ""), + ParseError: pgconv.TextOr(r.ParseError, ""), + Include: r.Include, + UserEdited: r.UserEdited, + } + if r.MatchTxnID.Valid { + v.MatchTxnID = formatUUID(r.MatchTxnID) + } + if r.MatchScore.Valid { + v.MatchScore = int(r.MatchScore.Int32) + } + if r.ParsedDate.Valid { + v.Date = r.ParsedDate.Time.Format("2006-01-02") + } + if r.ParsedAmount.Valid { + v.Amount = numericToDecimal(r.ParsedAmount).String() + } + return v +} + +func sessionToView(s db.CsvImportSession) ImportSession { + v := ImportSession{ + ID: formatUUID(s.ID), + ShortID: s.ShortID, + UserID: formatUUID(s.UserID), + Status: s.Status, + Filename: s.Filename, + Delimiter: s.Delimiter, + Headers: decodeHeaders(s.Headers), + RowCount: int(s.RowCount), + DetectedTemplate: pgconv.TextOr(s.DetectedTemplate, ""), + ColumnMapping: decodeMapping(s.ColumnMapping), + DateFormat: s.DateFormat, + PositiveIsDebit: s.PositiveIsDebit, + HasDebitCredit: s.HasDebitCredit, + IsoCurrencyCode: s.IsoCurrencyCode, + } + if s.ResolvedAccountID.Valid { + v.ResolvedAccountID = formatUUID(s.ResolvedAccountID) + } + if s.ResolvedConnectionID.Valid { + v.ResolvedConnectionID = formatUUID(s.ResolvedConnectionID) + } + if s.ProfileID.Valid { + v.ProfileID = formatUUID(s.ProfileID) + } + if len(s.Result) > 0 { + var res CSVImportResult + if json.Unmarshal(s.Result, &res) == nil { + v.Result = &res + } + } + return v +} + +func sessionParseConfig(s db.CsvImportSession) CSVParseConfig { + return CSVParseConfig{ + ColumnMapping: decodeMapping(s.ColumnMapping), + DateFormat: s.DateFormat, + PositiveIsDebit: s.PositiveIsDebit, + HasDebitCredit: s.HasDebitCredit, + } +} + +// detectParseConfig builds a parse config from a parsed file using the bank +// template (if matched) or generic column detection. Returns the template name. +func detectParseConfig(pf *csvpkg.ParsedFile) (CSVParseConfig, string) { + cfg := CSVParseConfig{ColumnMapping: map[string]int{}} + if t := csvpkg.DetectTemplate(pf.Headers); t != nil && t.HeaderPatterns != nil { + idx := func(name string) (int, bool) { + for i, h := range pf.Headers { + if strings.EqualFold(strings.TrimSpace(h), name) { + return i, true + } + } + return 0, false + } + if i, ok := idx(t.DateColumn); ok { + cfg.ColumnMapping["date"] = i + } + if i, ok := idx(t.AmountColumn); ok { + cfg.ColumnMapping["amount"] = i + } + if i, ok := idx(t.DescriptionColumn); ok { + cfg.ColumnMapping["description"] = i + } + if t.CategoryColumn != "" { + if i, ok := idx(t.CategoryColumn); ok { + cfg.ColumnMapping["category"] = i + } + } + if t.MerchantColumn != "" { + if i, ok := idx(t.MerchantColumn); ok { + cfg.ColumnMapping["merchant_name"] = i + } + } + if t.HasDebitCredit { + if i, ok := idx(t.DebitColumn); ok { + cfg.ColumnMapping["debit"] = i + } + if i, ok := idx(t.CreditColumn); ok { + cfg.ColumnMapping["credit"] = i + } + } + cfg.DateFormat = t.DateFormat + cfg.PositiveIsDebit = t.PositiveIsDebit + cfg.HasDebitCredit = t.HasDebitCredit + return cfg, t.Name + } + + cfg.ColumnMapping = csvpkg.DetectColumns(pf.Headers) + if dateCol, ok := cfg.ColumnMapping["date"]; ok { + samples := make([]string, 0, 20) + for _, r := range pf.Rows { + if dateCol < len(r) { + samples = append(samples, r[dateCol]) + } + if len(samples) >= 20 { + break + } + } + if df, err := csvpkg.DetectDateFormat(samples); err == nil { + cfg.DateFormat = df + } + } + return cfg, "" +} + +// detectCurrency guesses the ISO currency from symbols in the amount column, +// defaulting to USD. +func detectCurrency(rows [][]string, mapping map[string]int) string { + col, ok := mapping["amount"] + if !ok { + return "USD" + } + for i, r := range rows { + if i >= 50 || col >= len(r) { + continue + } + switch { + case strings.Contains(r[col], "€"): + return "EUR" + case strings.Contains(r[col], "£"): + return "GBP" + case strings.Contains(r[col], "¥"): + return "JPY" + } + } + return "USD" +} + +func decodeHeaders(b []byte) []string { + var h []string + _ = json.Unmarshal(b, &h) + return h +} + +func decodeMapping(b []byte) map[string]int { + m := map[string]int{} + _ = json.Unmarshal(b, &m) + return m +} + +func sampleRaw(rows [][]string, n int) [][]string { + if len(rows) <= n { + return rows + } + return rows[:n] +} + +func dateOrNull(t time.Time, ok bool) pgtype.Date { + if !ok { + return pgtype.Date{} + } + return pgconv.Date(t) +} + +func numericOrNull(d decimal.Decimal, ok bool) pgtype.Numeric { + if !ok { + return pgtype.Numeric{} + } + var n pgtype.Numeric + _ = n.Scan(d.String()) + return n +} + +func optionalUUID(idStr string) pgtype.UUID { + if idStr == "" { + return pgtype.UUID{} + } + u, err := pgconv.ParseUUID(idStr) + if err != nil { + return pgtype.UUID{} + } + return u +} + +func orDefault(v, def string) string { + if strings.TrimSpace(v) == "" { + return def + } + return v +} diff --git a/internal/service/csv_import_v2_integration_test.go b/internal/service/csv_import_v2_integration_test.go new file mode 100644 index 000000000..332256fef --- /dev/null +++ b/internal/service/csv_import_v2_integration_test.go @@ -0,0 +1,178 @@ +//go:build integration && !lite + +package service_test + +import ( + "context" + "testing" + + "breadbox/internal/pgconv" + "breadbox/internal/service" + "breadbox/internal/testutil" +) + +func countTxns(t *testing.T, svc *service.Service, accountID string) int { + t.Helper() + uid, err := pgconv.ParseUUID(accountID) + if err != nil { + t.Fatalf("parse account id: %v", err) + } + var n int + if err := svc.Pool.QueryRow(context.Background(), + "SELECT count(*) FROM transactions WHERE account_id = $1 AND deleted_at IS NULL", uid).Scan(&n); err != nil { + t.Fatalf("count txns: %v", err) + } + return n +} + +func TestImportV2_FullLifecycleAndProfileRedrop(t *testing.T) { + svc, q, _ := newService(t) + ctx := context.Background() + user := testutil.MustCreateUser(t, q, "Alice") + + file1 := []byte("Date,Amount,Description\n2026-07-01,10.00,COFFEE\n2026-07-02,20.00,LUNCH\n") + + // Analyze — no accounts yet, so it awaits account resolution. + an, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "chase.csv", Data: file1, + }) + if err != nil { + t.Fatalf("CreateImportSession: %v", err) + } + if an.Session.Status != "awaiting_account" { + t.Fatalf("status = %q, want awaiting_account", an.Session.Status) + } + + // Resolve by creating a new account. + sess, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{ + CreateNew: true, NewName: "Chase Checking", + }) + if err != nil { + t.Fatalf("ResolveImportAccount: %v", err) + } + if sess.Status != "previewed" || sess.ResolvedAccountID == "" { + t.Fatalf("after resolve: status=%q account=%q", sess.Status, sess.ResolvedAccountID) + } + acctA := sess.ResolvedAccountID + + _, summary, err := svc.ListImportRows(ctx, sess.ShortID, "", 1, 100) + if err != nil { + t.Fatalf("ListImportRows: %v", err) + } + if summary.Counts["new"] != 2 || summary.IncludedCount != 2 { + t.Fatalf("summary = %+v, want 2 new/included", summary) + } + + // Apply. + res, err := svc.ApplyImportSession(ctx, sess.ShortID, service.SystemActor()) + if err != nil { + t.Fatalf("ApplyImportSession: %v", err) + } + if res.NewCount != 2 || res.UpdatedCount != 0 { + t.Fatalf("apply result = %+v, want 2 new", res) + } + if n := countTxns(t, svc, acctA); n != 2 { + t.Fatalf("account has %d txns, want 2", n) + } + + // Re-drop the same file plus one new row. The saved profile should now + // auto-resolve to the same account, dedupe the two, and stage one new row. + file2 := []byte("Date,Amount,Description\n2026-07-01,10.00,COFFEE\n2026-07-02,20.00,LUNCH\n2026-07-03,30.00,DINNER\n") + an2, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "chase.csv", Data: file2, + }) + if err != nil { + t.Fatalf("CreateImportSession redrop: %v", err) + } + if an2.Session.Status != "previewed" { + t.Fatalf("redrop status = %q, want previewed (profile auto-resolve)", an2.Session.Status) + } + if an2.Session.ResolvedAccountID != acctA { + t.Fatalf("redrop resolved to %q, want original account %q", an2.Session.ResolvedAccountID, acctA) + } + if an2.Summary.Counts["exact_dup"] != 2 || an2.Summary.Counts["new"] != 1 { + t.Fatalf("redrop summary = %+v, want 2 exact_dup + 1 new", an2.Summary) + } + + res2, err := svc.ApplyImportSession(ctx, an2.Session.ShortID, service.SystemActor()) + if err != nil { + t.Fatalf("apply redrop: %v", err) + } + if res2.NewCount != 1 { + t.Fatalf("redrop apply = %+v, want 1 new", res2) + } + if n := countTxns(t, svc, acctA); n != 3 { + t.Fatalf("after redrop account has %d txns, want 3", n) + } +} + +func TestImportV2_ApplyIsIdempotent(t *testing.T) { + svc, q, _ := newService(t) + ctx := context.Background() + user := testutil.MustCreateUser(t, q, "Alice") + file := []byte("Date,Amount,Description\n2026-08-01,5.00,A\n2026-08-02,6.00,B\n") + + an, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "f.csv", Data: file, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + sess, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{CreateNew: true, NewName: "Acct"}) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if _, err := svc.ApplyImportSession(ctx, sess.ShortID, service.SystemActor()); err != nil { + t.Fatalf("apply1: %v", err) + } + // A fresh session over the same file into the same account must add nothing. + an2, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "f.csv", Data: file, + }) + if err != nil { + t.Fatalf("create2: %v", err) + } + res2, err := svc.ApplyImportSession(ctx, an2.Session.ShortID, service.SystemActor()) + if err != nil { + t.Fatalf("apply2: %v", err) + } + if res2.NewCount != 0 { + t.Fatalf("second apply added %d rows, want 0 (idempotent)", res2.NewCount) + } + if n := countTxns(t, svc, sess.ResolvedAccountID); n != 2 { + t.Fatalf("account has %d txns after double import, want 2", n) + } +} + +func TestImportV2_AccountChangeBeforeConfirm(t *testing.T) { + svc, q, _ := newService(t) + ctx := context.Background() + user := testutil.MustCreateUser(t, q, "Alice") + conn := testutil.MustCreateConnection(t, q, user.ID, "item_1") + accA := testutil.MustCreateAccount(t, q, conn.ID, "extA", "Account A") + accB := testutil.MustCreateAccount(t, q, conn.ID, "extB", "Account B") + + file := []byte("Date,Amount,Description\n2026-09-01,12.00,X\n") + an, err := svc.CreateImportSession(ctx, service.CreateImportSessionParams{ + UserID: pgconv.FormatUUID(user.ID), Filename: "f.csv", Data: file, + }) + if err != nil { + t.Fatalf("create: %v", err) + } + // Resolve to A, then change to B before applying. + if _, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{AccountID: pgconv.FormatUUID(accA.ID)}); err != nil { + t.Fatalf("resolve A: %v", err) + } + if _, err := svc.ResolveImportAccount(ctx, an.Session.ShortID, service.ResolveImportAccountParams{AccountID: pgconv.FormatUUID(accB.ID)}); err != nil { + t.Fatalf("resolve B: %v", err) + } + if _, err := svc.ApplyImportSession(ctx, an.Session.ShortID, service.SystemActor()); err != nil { + t.Fatalf("apply: %v", err) + } + if n := countTxns(t, svc, pgconv.FormatUUID(accB.ID)); n != 1 { + t.Fatalf("account B has %d txns, want 1", n) + } + if n := countTxns(t, svc, pgconv.FormatUUID(accA.ID)); n != 0 { + t.Fatalf("account A has %d txns, want 0 (import went to B)", n) + } +} From ebc7c4cdb8db032384e3d9a5a7afe605b8ba6069 Mon Sep 17 00:00:00 2001 From: Ricardo Canales Date: Sat, 6 Jun 2026 22:00:37 -0700 Subject: [PATCH 4/7] =?UTF-8?q?feat(csv-import):=20Wave=203=20=E2=80=94=20?= =?UTF-8?q?admin=20HTTP=20endpoints=20for=20the=20v2=20flow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit internal/admin/csv_import_v2.go + routes under /-/csv/v2/* (session-cookie auth, admin-only, so no OpenAPI surface / drift impact): - POST /-/csv/v2/sessions multipart analyze → ImportAnalysis - POST /-/csv/v2/sessions/{id}/resolve bind/create account → session - GET /-/csv/v2/sessions/{id}/rows paginated rows + summary + session - PATCH /-/csv/v2/sessions/{id}/rows/{rid} inline edit - POST /-/csv/v2/sessions/{id}/bulk include/exclude/set-category/remap - POST /-/csv/v2/sessions/{id}/apply apply → CSVImportResult 30MB upload cap; single-household-member default user; error envelope mapping (404 for unknown session, 422 otherwise). Integration test drives analyze → resolve(create-new) → rows → apply through a real chi router and asserts 2 new transactions. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/admin/csv_import_v2.go | 229 ++++++++++++++++++ .../admin/csv_import_v2_integration_test.go | 103 ++++++++ internal/admin/router.go | 8 + 3 files changed, 340 insertions(+) create mode 100644 internal/admin/csv_import_v2.go create mode 100644 internal/admin/csv_import_v2_integration_test.go diff --git a/internal/admin/csv_import_v2.go b/internal/admin/csv_import_v2.go new file mode 100644 index 000000000..b73d916fe --- /dev/null +++ b/internal/admin/csv_import_v2.go @@ -0,0 +1,229 @@ +//go:build !headless && !lite + +package admin + +import ( + "errors" + "io" + "net/http" + "strconv" + + "breadbox/internal/app" + "breadbox/internal/pgconv" + "breadbox/internal/service" + + "github.com/alexedwards/scs/v2" + "github.com/go-chi/chi/v5" +) + +// maxCSVV2UploadSize caps the drag-drop upload. Larger than the legacy wizard's +// 10MB because v2 persists the raw bytes and supports up to 50k rows. +const maxCSVV2UploadSize = 30 << 20 // 30MB + +// CSVV2CreateSessionHandler serves POST /-/csv/v2/sessions. +// Accepts a multipart file (field "file") + optional "user_id"; analyzes the +// file, runs account detection, and returns the import analysis. +func CSVV2CreateSessionHandler(a *app.App, sm *scs.SessionManager, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, maxCSVV2UploadSize) + if err := r.ParseMultipartForm(maxCSVV2UploadSize); err != nil { + writeError(w, http.StatusBadRequest, "FILE_TOO_LARGE", "File too large (max 30MB)") + return + } + file, header, err := r.FormFile("file") + if err != nil { + writeError(w, http.StatusBadRequest, "NO_FILE", "No file uploaded") + return + } + defer file.Close() + raw, err := io.ReadAll(file) + if err != nil { + writeError(w, http.StatusBadRequest, "READ_FAILED", "Failed to read file") + return + } + + userID, err := defaultImportUser(r, svc, r.FormValue("user_id")) + if err != nil { + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", err.Error()) + return + } + + filename := r.FormValue("filename") + if filename == "" && header != nil { + filename = header.Filename + } + + analysis, err := svc.CreateImportSession(r.Context(), service.CreateImportSessionParams{ + UserID: userID, + Filename: filename, + Data: raw, + }) + if err != nil { + a.Logger.Debug("csv v2 analyze failed", "error", err) + writeError(w, http.StatusUnprocessableEntity, "PARSE_FAILED", humanizeCSVError(err)) + return + } + writeJSON(w, http.StatusOK, analysis) + } +} + +// CSVV2ResolveHandler serves POST /-/csv/v2/sessions/{id}/resolve. +func CSVV2ResolveHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + AccountID string `json:"account_id"` + CreateNew bool `json:"create_new"` + NewName string `json:"new_name"` + NewType string `json:"new_type"` + NewSubtype string `json:"new_subtype"` + NewCurrency string `json:"new_currency"` + } + if !decodeJSON(w, r, &req) { + return + } + if !req.CreateNew && req.AccountID == "" { + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", "account_id or create_new is required") + return + } + sess, err := svc.ResolveImportAccount(r.Context(), chi.URLParam(r, "id"), service.ResolveImportAccountParams{ + AccountID: req.AccountID, + CreateNew: req.CreateNew, + NewName: req.NewName, + NewType: req.NewType, + NewSubtype: req.NewSubtype, + NewCurrency: req.NewCurrency, + }) + if err != nil { + writeImportErr(w, a, "resolve account", err) + return + } + writeJSON(w, http.StatusOK, sess) + } +} + +// CSVV2RowsHandler serves GET /-/csv/v2/sessions/{id}/rows. +func CSVV2RowsHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + pageSize, _ := strconv.Atoi(r.URL.Query().Get("page_size")) + rows, summary, err := svc.ListImportRows(r.Context(), id, r.URL.Query().Get("status"), page, pageSize) + if err != nil { + writeImportErr(w, a, "list rows", err) + return + } + sess, err := svc.GetImportSession(r.Context(), id) + if err != nil { + writeImportErr(w, a, "get session", err) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "session": sess, + "rows": rows, + "summary": summary, + }) + } +} + +// CSVV2EditRowHandler serves PATCH /-/csv/v2/sessions/{id}/rows/{rowId}. +func CSVV2EditRowHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Date string `json:"date"` + Amount string `json:"amount"` + Desc string `json:"description"` + Merchant string `json:"merchant"` + Include *bool `json:"include"` + } + if !decodeJSON(w, r, &req) { + return + } + row, err := svc.EditImportRow(r.Context(), chi.URLParam(r, "id"), chi.URLParam(r, "rowId"), service.EditImportRowParams{ + Date: req.Date, + Amount: req.Amount, + Desc: req.Desc, + Merchant: req.Merchant, + Include: req.Include, + }) + if err != nil { + writeImportErr(w, a, "edit row", err) + return + } + writeJSON(w, http.StatusOK, row) + } +} + +// CSVV2BulkHandler serves POST /-/csv/v2/sessions/{id}/bulk. +func CSVV2BulkHandler(a *app.App, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var req struct { + Op string `json:"op"` + Classification string `json:"classification"` + CategoryID string `json:"category_id"` + ColumnMapping map[string]int `json:"column_mapping"` + DateFormat string `json:"date_format"` + PositiveIsDebit bool `json:"positive_is_debit"` + HasDebitCredit bool `json:"has_debit_credit"` + } + if !decodeJSON(w, r, &req) { + return + } + if err := svc.BulkImportOp(r.Context(), chi.URLParam(r, "id"), service.ImportBulkOp{ + Op: req.Op, + Classification: req.Classification, + CategoryID: req.CategoryID, + ColumnMapping: req.ColumnMapping, + DateFormat: req.DateFormat, + PositiveIsDebit: req.PositiveIsDebit, + HasDebitCredit: req.HasDebitCredit, + }); err != nil { + writeImportErr(w, a, "bulk op", err) + return + } + // Return the refreshed summary so the client can update counts. + _, summary, err := svc.ListImportRows(r.Context(), chi.URLParam(r, "id"), "", 1, 1) + if err != nil { + writeImportErr(w, a, "summary", err) + return + } + writeJSON(w, http.StatusOK, map[string]any{"summary": summary}) + } +} + +// CSVV2ApplyHandler serves POST /-/csv/v2/sessions/{id}/apply. +func CSVV2ApplyHandler(a *app.App, sm *scs.SessionManager, svc *service.Service) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + result, err := svc.ApplyImportSession(r.Context(), chi.URLParam(r, "id"), ActorFromSession(sm, r)) + if err != nil { + writeImportErr(w, a, "apply", err) + return + } + writeJSON(w, http.StatusOK, result) + } +} + +// defaultImportUser resolves the household user for a new import: the explicit +// value if given, otherwise the sole household member. +func defaultImportUser(r *http.Request, svc *service.Service, explicit string) (string, error) { + if explicit != "" { + return explicit, nil + } + users, err := svc.Queries.ListUsers(r.Context()) + if err != nil { + return "", errors.New("failed to list household members") + } + if len(users) == 0 { + return "", errors.New("create a household member before importing") + } + return pgconv.FormatUUID(users[0].ID), nil +} + +// writeImportErr maps a service error to an HTTP response. +func writeImportErr(w http.ResponseWriter, a *app.App, op string, err error) { + if errors.Is(err, service.ErrImportSessionNotFound) { + writeError(w, http.StatusNotFound, "NOT_FOUND", "Import session not found") + return + } + a.Logger.Debug("csv v2 "+op+" failed", "error", err) + writeError(w, http.StatusUnprocessableEntity, "VALIDATION_ERROR", err.Error()) +} diff --git a/internal/admin/csv_import_v2_integration_test.go b/internal/admin/csv_import_v2_integration_test.go new file mode 100644 index 000000000..36538f2c2 --- /dev/null +++ b/internal/admin/csv_import_v2_integration_test.go @@ -0,0 +1,103 @@ +//go:build integration && !headless && !lite + +package admin + +import ( + "bytes" + "encoding/json" + "log/slog" + "mime/multipart" + "net/http" + "net/http/httptest" + "testing" + + "breadbox/internal/app" + "breadbox/internal/service" + "breadbox/internal/testutil" + + "github.com/alexedwards/scs/v2" + "github.com/go-chi/chi/v5" +) + +// TestCSVV2Flow_HTTP drives the v2 endpoints through a real chi router: +// analyze (multipart) → resolve (create new account) → list rows → apply. +func TestCSVV2Flow_HTTP(t *testing.T) { + pool, q := testutil.ServicePool(t) + svc := service.New(q, pool, nil, slog.Default()) + a := &app.App{DB: pool, Queries: q, Logger: slog.Default()} + sm := scs.New() // defaults to an in-memory store + + testutil.MustCreateUser(t, q, "Alice") + + r := chi.NewRouter() + r.Use(sm.LoadAndSave) + r.Post("/-/csv/v2/sessions", CSVV2CreateSessionHandler(a, sm, svc)) + r.Post("/-/csv/v2/sessions/{id}/resolve", CSVV2ResolveHandler(a, svc)) + r.Get("/-/csv/v2/sessions/{id}/rows", CSVV2RowsHandler(a, svc)) + r.Post("/-/csv/v2/sessions/{id}/apply", CSVV2ApplyHandler(a, sm, svc)) + srv := httptest.NewServer(r) + defer srv.Close() + + // 1. Analyze via multipart upload. + var body bytes.Buffer + mw := multipart.NewWriter(&body) + fw, _ := mw.CreateFormFile("file", "chase.csv") + fw.Write([]byte("Date,Amount,Description\n2026-10-01,10.00,COFFEE\n2026-10-02,20.00,LUNCH\n")) + mw.Close() + + resp, err := http.Post(srv.URL+"/-/csv/v2/sessions", mw.FormDataContentType(), &body) + if err != nil { + t.Fatalf("analyze post: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("analyze status %d", resp.StatusCode) + } + var analysis service.ImportAnalysis + json.NewDecoder(resp.Body).Decode(&analysis) + resp.Body.Close() + shortID := analysis.Session.ShortID + if shortID == "" { + t.Fatal("no session short id returned") + } + + // 2. Resolve by creating a new account. + post := func(path string, payload any) *http.Response { + b, _ := json.Marshal(payload) + rsp, err := http.Post(srv.URL+path, "application/json", bytes.NewReader(b)) + if err != nil { + t.Fatalf("post %s: %v", path, err) + } + return rsp + } + rr := post("/-/csv/v2/sessions/"+shortID+"/resolve", map[string]any{"create_new": true, "new_name": "Chase"}) + if rr.StatusCode != http.StatusOK { + t.Fatalf("resolve status %d", rr.StatusCode) + } + rr.Body.Close() + + // 3. List rows. + rowsResp, err := http.Get(srv.URL + "/-/csv/v2/sessions/" + shortID + "/rows") + if err != nil { + t.Fatalf("rows get: %v", err) + } + var rowsBody struct { + Summary service.ImportSummary `json:"summary"` + } + json.NewDecoder(rowsResp.Body).Decode(&rowsBody) + rowsResp.Body.Close() + if rowsBody.Summary.IncludedCount != 2 { + t.Fatalf("included = %d, want 2", rowsBody.Summary.IncludedCount) + } + + // 4. Apply. + ar := post("/-/csv/v2/sessions/"+shortID+"/apply", map[string]any{}) + if ar.StatusCode != http.StatusOK { + t.Fatalf("apply status %d", ar.StatusCode) + } + var result service.CSVImportResult + json.NewDecoder(ar.Body).Decode(&result) + ar.Body.Close() + if result.NewCount != 2 { + t.Fatalf("apply new count = %d, want 2", result.NewCount) + } +} diff --git a/internal/admin/router.go b/internal/admin/router.go index d50169b8f..412bb6307 100644 --- a/internal/admin/router.go +++ b/internal/admin/router.go @@ -578,6 +578,14 @@ func NewAdminRouter(a *app.App, sm *scs.SessionManager, tr *TemplateRenderer, sv r.Post("/csv/preview", CSVPreviewHandler(a, sm)) r.Post("/csv/import", CSVImportHandler(a, sm, svc)) + // CSV import v2 — drag-anywhere flow (analyze → resolve → preview → apply). + r.Post("/csv/v2/sessions", CSVV2CreateSessionHandler(a, sm, svc)) + r.Post("/csv/v2/sessions/{id}/resolve", CSVV2ResolveHandler(a, svc)) + r.Get("/csv/v2/sessions/{id}/rows", CSVV2RowsHandler(a, svc)) + r.Patch("/csv/v2/sessions/{id}/rows/{rowId}", CSVV2EditRowHandler(a, svc)) + r.Post("/csv/v2/sessions/{id}/bulk", CSVV2BulkHandler(a, svc)) + r.Post("/csv/v2/sessions/{id}/apply", CSVV2ApplyHandler(a, sm, svc)) + // API key + OAuth client revoke/delete — admin only. r.Delete("/api-keys/{id}", RevokeAPIKeyHandler(svc)) r.Delete("/oauth-clients/{id}", RevokeOAuthClientHandler(svc)) From 4d0317650fc9e32909dfa5e8821a770fb74c323a Mon Sep 17 00:00:00 2001 From: Ricardo Canales Date: Sat, 6 Jun 2026 22:09:42 -0700 Subject: [PATCH 5/7] =?UTF-8?q?feat(csv-import):=20Wave=204=20=E2=80=94=20?= =?UTF-8?q?drop-anywhere=20overlay=20+=20full-screen=20flow=20modal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UI for the v2 flow, daisy-first, matching the Workflows-page quality bar: - base.html: $store.csvImport store + document-level drag-and-drop (dragenter depth counter, file-only overlay, [data-dropzone] opt-out, multi-file → first + toast). Full-window dashed drop overlay. Mounts components.ImportModal in the shared global-overlay region (renders on every page, hidden until a drop). - components.ImportModal: large daisy flow — analyzing / account picker (ranked suggestions w/ confidence + reason chips + saved-profile badge + create-new) / preview (classification filter join-tabs, summary counts, server-paginated table table-sm with include checkboxes, classification badges, inline row edit, bulk include/exclude) / success. - static/js/admin/components/csv_import_v2.js: the csvImport Alpine factory — watches the store, drives analyze→resolve→rows→edit/bulk→apply against the /-/csv/v2/* endpoints (CSRF auto-injected), toasts on apply. - Registered ImportModal in the renderComponent bridge. - Sandboxified: SectionCSVImport in /design (patterns) — drop overlay, account suggestions, classification badges, preview table, success. Screenshot-verified. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/admin/templates.go | 3 + .../templates/components/import_modal.templ | 269 ++++++++++++++++++ .../components/pages/design_sections.templ | 98 +++++++ .../components/pages/design_types.go | 7 + internal/templates/layout/base.html | 98 +++++++ static/js/admin/components/csv_import_v2.js | 254 +++++++++++++++++ 6 files changed, 729 insertions(+) create mode 100644 internal/templates/components/import_modal.templ create mode 100644 static/js/admin/components/csv_import_v2.js diff --git a/internal/admin/templates.go b/internal/admin/templates.go index 494f6da56..adef0cae1 100644 --- a/internal/admin/templates.go +++ b/internal/admin/templates.go @@ -93,6 +93,9 @@ var componentRegistry = map[string]componentAdapter{ "ThemeToggle": func(_ any) (templ.Component, error) { return components.ThemeToggle(), nil }, + "ImportModal": func(_ any) (templ.Component, error) { + return components.ImportModal(), nil + }, "TopbarUserMenu": func(data any) (templ.Component, error) { m, ok := data.(map[string]any) if !ok { diff --git a/internal/templates/components/import_modal.templ b/internal/templates/components/import_modal.templ new file mode 100644 index 000000000..ddefd8637 --- /dev/null +++ b/internal/templates/components/import_modal.templ @@ -0,0 +1,269 @@ +package components + +// ImportModal is the drop-anywhere CSV import flow: a large daisy modal that +// walks analyze → resolve account → preview/edit → apply → result. It is +// mounted once globally from base.html and driven by the $store.csvImport store +// (drop overlay + open/close) plus the csvImport Alpine factory +// (static/js/admin/components/csv_import_v2.js). +templ ImportModal() { + + +} + +templ importModalAnalyzing() { +
+ +
Analyzing your file…
+
+} + +templ importModalError() { +
+
+ @LucideIcon("alert-circle", "w-5 h-5") + +
+
+ +
+
+} + +// Account selection: ranked suggestions + create-new. +templ importModalAccount() { +
+
+
Which account is this?
+
We ranked your accounts by how well the file matches. Pick one, or create a new account.
+
+
+ +
No existing accounts yet — create one below.
+
+ + +
+} + +// Preview: summary chips + classification filter + paginated table. +templ importModalPreview() { +
+ +
+
+ + + + + +
+
+ + +
+
+ +
+ + + + + + + + + + + + + + +
DateAmountDescriptionStatus
+
No rows in this view.
+
+ +
+
+ + + +
+
+
+} + +templ importModalResult() { +
+
+ @LucideIcon("check", "w-7 h-7 text-success") +
+
+
Import complete
+
+ added + · updated + · skipped +
+
+ +
+} diff --git a/internal/templates/components/pages/design_sections.templ b/internal/templates/components/pages/design_sections.templ index 55fc955ca..cd8c2a15d 100644 --- a/internal/templates/components/pages/design_sections.templ +++ b/internal/templates/components/pages/design_sections.templ @@ -3508,3 +3508,101 @@ templ designSettingsCreateButton() { templ designSettingsSaveButton(label string) { } + +// SectionCSVImport showcases the drop-anywhere CSV import flow's building +// blocks (static markup — the live flow is components.ImportModal driven by the +// csvImport Alpine factory + $store.csvImport). Variant matrix: drop overlay, +// ranked account picker rows, classification badges, a preview table slice, and +// the success screen. +templ SectionCSVImport() { +
+ @designExample("Drop overlay", "Shown full-window when a file is dragged over any page.") { +
+
+
+ @components.LucideIcon("upload-cloud", "w-10 h-10 text-primary") +
Drop your CSV to import
+
We'll detect the account and preview changes before saving.
+
+
+
+ } + @designExample("Account suggestions", "Ranked by mask, transaction overlap, institution, and saved profiles.") { +
+ + + +
+ } + @designExample("Classification badges", "Per-row verdict in the preview table.") { +
+ New + Maybe dup + Duplicate + Error +
+ } + @designExample("Preview table", "Server-paginated; checkbox toggles include, pencil edits inline.") { +
+ + + + + + + + + + + + + + + + +
DateAmountDescriptionStatus
2026-07-0330.00DINNERNew
2026-07-0110.00COFFEE + Duplicate +
already imported
+
+
+ } + @designExample("Success", "Result screen after apply.") { +
+
+ @components.LucideIcon("check", "w-7 h-7 text-success") +
+
+
Import complete
+
1 added · 2 skipped
+
+
+ } +
+} diff --git a/internal/templates/components/pages/design_types.go b/internal/templates/components/pages/design_types.go index 5d221b87f..03fe517b7 100644 --- a/internal/templates/components/pages/design_types.go +++ b/internal/templates/components/pages/design_types.go @@ -414,6 +414,13 @@ func DesignSections() []DesignSection { }, // ── Patterns ──────────────────────────────────────────────── + { + Slug: "csv-import", + Title: "CSV import flow", + Description: "Drop-anywhere CSV import (components.ImportModal + $store.csvImport + the csvImport Alpine factory): the full-window drop overlay, ranked account suggestions (mask / overlap / institution / saved profile), per-row classification badges (new / maybe-dup / duplicate / error), the server-paginated preview table, and the success screen.", + Group: "patterns", + Render: func() templ.Component { return SectionCSVImport() }, + }, { Slug: "multi-select-toolbar", Title: "Multi-select toolbar", diff --git a/internal/templates/layout/base.html b/internal/templates/layout/base.html index 8d3d9ed6f..4e28a649b 100644 --- a/internal/templates/layout/base.html +++ b/internal/templates/layout/base.html @@ -212,6 +212,89 @@ }); + +