Remove columns from iceberg export (#249)

* Remove columns from iceberg export

* Update artifacts

* Remove export column from equality

* Add reserved field

* Scope buf.yaml

Author: gbrgr

buf.yaml

@@ -4,5 +4,8 @@ modules:
     breaking:
       use:
         - FILE
-      except:
-        - FIELD_NO_DELETE
+      ignore_only:
+        FIELD_NO_DELETE:
+          - proto/relationalai/lqp/v1/transactions.proto
+        MESSAGE_NO_DELETE:
+          - proto/relationalai/lqp/v1/transactions.proto

meta/src/meta/grammar.y

@@ -116,8 +116,6 @@
 %nonterm export_csv_config transactions.ExportCSVConfig
 %nonterm export_csv_path String
 %nonterm export_csv_source transactions.ExportCSVSource
-%nonterm export_iceberg_column transactions.ExportColumn
-%nonterm export_iceberg_columns Sequence[transactions.ExportColumn]
 %nonterm export_iceberg_config transactions.ExportIcebergConfig
 %nonterm export_iceberg_table_def logic.RelationId
 %nonterm iceberg_auth_properties Sequence[Tuple[String, String]]
@@ -1222,13 +1220,6 @@ iceberg_catalog_config
         $5: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.properties)
         $6: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.auth_properties)
 
-export_iceberg_column
-    : "(" "column" STRING boolean_value ")"
-      construct: $$ = transactions.ExportColumn(name=$3, nullable=$4)
-      deconstruct:
-        $3: String = $$.name
-        $4: Boolean = $$.nullable
-
 iceberg_to_snapshot
     : "(" "to_snapshot" STRING ")"
       construct: $$ = $3
@@ -1372,26 +1363,20 @@ export_iceberg_table_def
       construct: $$ = $3
       deconstruct: $3: logic.RelationId = $$
 
-export_iceberg_columns
-    : "(" "columns" export_iceberg_column* ")"
-      construct: $$ = $3
-      deconstruct: $3: Sequence[transactions.ExportColumn] = $$
-
 iceberg_table_properties
     : "(" "table_properties" iceberg_property_entry* ")"
       construct: $$ = $3
       deconstruct: $3: Sequence[Tuple[String, String]] = $$
 
 export_iceberg_config
-    : "(" "export_iceberg_config" iceberg_locator iceberg_catalog_config export_iceberg_table_def export_iceberg_columns iceberg_table_properties config_dict? ")"
-      construct: $$ = construct_export_iceberg_config_full($3, $4, $5, $6, $7, $8)
+    : "(" "export_iceberg_config" iceberg_locator iceberg_catalog_config export_iceberg_table_def iceberg_table_properties config_dict? ")"
+      construct: $$ = construct_export_iceberg_config_full($3, $4, $5, $6, $7)
       deconstruct:
         $3: logic.IcebergLocator = $$.locator
         $4: logic.IcebergCatalogConfig = $$.config
         $5: logic.RelationId = $$.table_def
-        $6: Sequence[transactions.ExportColumn] = $$.columns
-        $7: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.table_properties)
-        $8: Optional[Sequence[Tuple[String, logic.Value]]] = deconstruct_export_iceberg_config_optional($$)
+        $6: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.table_properties)
+        $7: Optional[Sequence[Tuple[String, logic.Value]]] = deconstruct_export_iceberg_config_optional($$)
 
 
 %%
@@ -1770,7 +1755,6 @@ def construct_export_iceberg_config_full(
     locator: logic.IcebergLocator,
     config: logic.IcebergCatalogConfig,
     table_def: logic.RelationId,
-    columns: Sequence[transactions.ExportColumn],
     table_property_pairs: Sequence[Tuple[String, String]],
     config_dict: Optional[Sequence[Tuple[String, logic.Value]]],
 ) -> transactions.ExportIcebergConfig:
@@ -1783,7 +1767,6 @@ def construct_export_iceberg_config_full(
         locator=locator,
         config=config,
         table_def=table_def,
-        columns=columns,
         prefix=builtin.some(prefix),
         target_file_size_bytes=builtin.some(target_file_size_bytes),
         compression=compression,

proto/relationalai/lqp/v1/transactions.proto

@@ -122,16 +122,11 @@ message ExportCSVSource {
 
 // Iceberg Export config
 
-message ExportColumn {
-  string name = 1;
-  bool nullable = 2;
-}
-
 message ExportIcebergConfig {
   IcebergLocator locator = 1;
   IcebergCatalogConfig config = 2;
   RelationId table_def = 3;              // Which definition to export as a table.
-  repeated ExportColumn columns = 4;
+  reserved 4;                            // was: repeated ExportColumn columns
   optional string prefix = 5;            // File name prefix for parquet files that are produced.
   optional int64 target_file_size_bytes = 6;
   string compression = 7;