diff --git a/crates/sail-function/src/scalar/csv/mod.rs b/crates/sail-function/src/scalar/csv/mod.rs index 08f5bc3100..2695b8de92 100644 --- a/crates/sail-function/src/scalar/csv/mod.rs +++ b/crates/sail-function/src/scalar/csv/mod.rs @@ -1,3 +1,4 @@ +mod options; mod schema_of_csv; pub mod spark_from_csv; pub mod spark_to_csv; diff --git a/crates/sail-function/src/scalar/csv/options.rs b/crates/sail-function/src/scalar/csv/options.rs new file mode 100644 index 0000000000..fc0773b6f8 --- /dev/null +++ b/crates/sail-function/src/scalar/csv/options.rs @@ -0,0 +1,330 @@ +use std::ops::Range; + +use datafusion::arrow::array::{Array, MapArray, StringArray}; +use datafusion::error::Result; +use datafusion_common::exec_err; +use sail_common::spec::{SAIL_MAP_KEY_FIELD_NAME, SAIL_MAP_VALUE_FIELD_NAME}; + +/// Which CSV expression function is reading the options. +/// +/// Spark validates almost every option identically in all three, but not quite: only the writer +/// rejects an empty `lineSep`, and only `from_csv` rejects the `DROPMALFORMED` parse mode. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum CsvFunction { + From, + To, + SchemaOf, +} + +impl CsvFunction { + fn name(self) -> &'static str { + match self { + Self::From => "from_csv", + Self::To => "to_csv", + Self::SchemaOf => "schema_of_csv", + } + } +} + +/// The boolean options that Spark's `CSVOptions` parses with `getBool`, which reports +/// `