Skip to content

Commit 3077016

Browse files
hbarthelsclaude
andcommitted
Support a load_errors sink in the (relations ...) CSV loading form
Add an optional trailing `(load_errors :name)` clause to the generalized `(relations ...)` loading form, naming the relation that collects rows which fail to load: (relations (keys (column "id" INT)) (relation :wide (column "label" STRING) (column "ratio" FLOAT)) (load_errors :my_load_errors)) The load-errors relation has a loader-determined schema that differs from the declared keys and per-relation value columns, so it is called out as its own clause rather than being another target relation. Backed by a presence-tracked `optional RelationId load_errors` on `TargetRelations`, so an absent sink is distinct from a default-valued one. The clause is accepted after both plain targets and CDC insert/delete deltas. Regenerates the protobuf bindings, parsers, and pretty printers for all three SDKs, and extends the Julia equality/hashing for `TargetRelations` to cover the new field. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1 parent 011fd8b commit 3077016

24 files changed

Lines changed: 15153 additions & 14717 deletions

‎meta/src/meta/grammar.y‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@
9494
%nonterm cdc_inserts Sequence[logic.TargetRelation]
9595
%nonterm cdc_deletes Sequence[logic.TargetRelation]
9696
%nonterm relation_body logic.TargetRelations
97+
%nonterm load_errors logic.RelationId
9798
%nonterm target_relations logic.TargetRelations
9899
%nonterm csv_config logic.CSVConfig
99100
%nonterm csv_data logic.CSVData
@@ -1168,12 +1169,16 @@ relation_body
11681169
$1: Sequence[logic.TargetRelation] = $$.cdc.inserts
11691170
$2: Sequence[logic.TargetRelation] = $$.cdc.deletes
11701171

1172+
load_errors
1173+
: "(" "load_errors" relation_id ")"
1174+
11711175
target_relations
1172-
: "(" "relations" relation_keys relation_body ")"
1173-
construct: $$ = construct_relations($3, $4)
1176+
: "(" "relations" relation_keys relation_body load_errors? ")"
1177+
construct: $$ = construct_relations($3, $4, $5)
11741178
deconstruct:
11751179
$3: Tuple[Sequence[logic.NamedColumn], Boolean] = deconstruct_relation_keys($$)
11761180
$4: logic.TargetRelations = $$
1181+
$5: Optional[logic.RelationId] = deconstruct_load_errors_optional($$)
11771182

11781183
csv_locator_paths
11791184
: "(" "paths" STRING* ")"
@@ -1570,13 +1575,26 @@ def deconstruct_relation_keys(
15701575
return builtin.tuple(msg.keys, msg.synthetic_key)
15711576

15721577

1578+
def deconstruct_load_errors_optional(
1579+
msg: logic.TargetRelations,
1580+
) -> Optional[logic.RelationId]:
1581+
if builtin.has_proto_field(msg, "load_errors"):
1582+
return builtin.some(builtin.unwrap_option(msg.load_errors))
1583+
return builtin.none()
1584+
1585+
15731586
def construct_relations(
15741587
keys: Tuple[Sequence[logic.NamedColumn], Boolean],
15751588
body: logic.TargetRelations,
1589+
load_errors_opt: Optional[logic.RelationId],
15761590
) -> logic.TargetRelations:
15771591
if builtin.has_proto_field(body, "plain"):
1578-
return logic.TargetRelations(keys=keys[0], synthetic_key=keys[1], plain=body.plain)
1579-
return logic.TargetRelations(keys=keys[0], synthetic_key=keys[1], cdc=body.cdc)
1592+
return logic.TargetRelations(
1593+
keys=keys[0], synthetic_key=keys[1], plain=body.plain, load_errors=load_errors_opt
1594+
)
1595+
return logic.TargetRelations(
1596+
keys=keys[0], synthetic_key=keys[1], cdc=body.cdc, load_errors=load_errors_opt
1597+
)
15801598

15811599

15821600
def construct_csv_data(

‎proto/relationalai/lqp/v1/logic.proto‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,10 @@ message TargetRelations {
323323
// If true, the shared key is synthesized by the loader (e.g. a generated row key)
324324
// instead of being drawn from CSV columns. Mutually exclusive with `keys`.
325325
bool synthetic_key = 4;
326+
// If present, rows that fail to load are collected into this relation. Its schema is
327+
// determined by the loader and differs from `keys` plus the per-relation value columns,
328+
// which is why it is called out separately instead of being a regular target relation.
329+
optional RelationId load_errors = 5;
326330
}
327331

328332
message CSVData {

‎sdks/go/src/lqp/v1/logic.pb.go‎

Lines changed: 332 additions & 315 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)