diff --git a/.github/agents/spark-test-expert.agent.md b/.github/agents/spark-test-expert.agent.md index 516362a69c..7edfcc38a1 100644 --- a/.github/agents/spark-test-expert.agent.md +++ b/.github/agents/spark-test-expert.agent.md @@ -13,7 +13,7 @@ Run the tests inside the Hatch environments defined in `pyproject.toml`. The following test environments are currently available: - `test-spark.spark-3.5.7` -- `test-spark.spark-4.1.1` +- `test-spark.spark-4.2.0` - `test-ibis` The `test-spark.spark-*` environments support running PySpark unit tests and doctests. The `test-ibis` environment supports running Ibis unit tests with PySpark as the backend. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eefc20a5a8..138660ac5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -108,7 +108,7 @@ jobs: strategy: fail-fast: false matrix: - spark-version: ["3.5.7", "4.1.1"] + spark-version: ["3.5.7", "4.2.0"] with: spark_version: ${{ matrix.spark-version }} needs: @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - spark-version: ["3.5.7", "4.0.1", "4.1.1"] + spark-version: ["3.5.7", "4.0.1", "4.1.1", "4.2.0"] with: spark_version: ${{ matrix.spark-version }} needs: diff --git a/.github/workflows/catalog-tests.yml b/.github/workflows/catalog-tests.yml index d9ad85bb16..b1185e99c8 100644 --- a/.github/workflows/catalog-tests.yml +++ b/.github/workflows/catalog-tests.yml @@ -11,7 +11,7 @@ jobs: name: Test runs-on: ubuntu-latest env: - HATCH_ENV: "test.spark-4.1.1" + HATCH_ENV: "test.spark-4.2.0" # Coverage environment variables RUSTC_WORKSPACE_WRAPPER: ${{ github.workspace }}/.github/scripts/rustc-workspace-wrapper.sh LLVM_PROFILE_FILE: ${{ github.workspace }}/target/sail-catalog-it-%p-%m.profraw diff --git a/.github/workflows/gold-data-script-validation.yml b/.github/workflows/gold-data-script-validation.yml index da3507731d..a10471cc9d 100644 --- a/.github/workflows/gold-data-script-validation.yml +++ b/.github/workflows/gold-data-script-validation.yml @@ -26,7 +26,7 @@ jobs: with: repository: apache/spark path: opt/spark - ref: v4.1.1 + ref: v4.2.0 fetch-depth: 1 - uses: actions/setup-java@v5 diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 4d6f044188..aa4b5327ae 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -35,8 +35,8 @@ jobs: continue-on-error: true with: workflow_run_id: ${{ github.event.workflow_run.id }} - artifact_name: spark-4.1.1-test-report - report_name: spark-4.1.1-test + artifact_name: spark-4.2.0-test-report + report_name: spark-4.2.0-test - uses: ./.github/actions/pull-request-report if: github.event.workflow_run.name == 'Build' diff --git a/.github/workflows/spark-package-artifacts.yml b/.github/workflows/spark-package-artifacts.yml index cec6c9041b..0463a621ac 100644 --- a/.github/workflows/spark-package-artifacts.yml +++ b/.github/workflows/spark-package-artifacts.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - spark-version: ["3.5.7", "4.1.1"] + spark-version: ["3.5.7", "4.2.0"] steps: - uses: actions/checkout@v7 diff --git a/crates/sail-execution/proto/sail/plan/physical.proto b/crates/sail-execution/proto/sail/plan/physical.proto index d5b08003ed..6bd50fd9a9 100644 --- a/crates/sail-execution/proto/sail/plan/physical.proto +++ b/crates/sail-execution/proto/sail/plan/physical.proto @@ -470,6 +470,7 @@ message PySparkUdfConfig { bool python_udtf_pandas_conversion_enabled = 7; bool python_udf_pandas_int_to_decimal_coercion_enabled = 8; bool binary_as_bytes = 9; + bool python_udf_pandas_prefer_int_extension_dtype = 10; } message EquivalenceProperties { diff --git a/crates/sail-execution/src/proto/codec.rs b/crates/sail-execution/src/proto/codec.rs index c717cb66c7..034de71006 100644 --- a/crates/sail-execution/src/proto/codec.rs +++ b/crates/sail-execution/src/proto/codec.rs @@ -4203,6 +4203,8 @@ impl RemoteExecutionCodec { python_udf_pandas_int_to_decimal_coercion_enabled: config .python_udf_pandas_int_to_decimal_coercion_enabled, binary_as_bytes: config.binary_as_bytes, + python_udf_pandas_prefer_int_extension_dtype: config + .python_udf_pandas_prefer_int_extension_dtype, }; Ok(config) } @@ -4223,6 +4225,8 @@ impl RemoteExecutionCodec { python_udf_pandas_int_to_decimal_coercion_enabled: config .python_udf_pandas_int_to_decimal_coercion_enabled, binary_as_bytes: config.binary_as_bytes, + python_udf_pandas_prefer_int_extension_dtype: config + .python_udf_pandas_prefer_int_extension_dtype, }; Ok(config) } diff --git a/crates/sail-plan/src/resolver/query/udtf.rs b/crates/sail-plan/src/resolver/query/udtf.rs index acf6418b54..4ac83e8c81 100644 --- a/crates/sail-plan/src/resolver/query/udtf.rs +++ b/crates/sail-plan/src/resolver/query/udtf.rs @@ -181,6 +181,7 @@ impl PlanResolver<'_> { function.eval_type, arguments_len, &input_types, + passthrough_columns, kwargs, &return_type, &self.config.pyspark_udf_config, diff --git a/crates/sail-python-udf/src/cereal/mod.rs b/crates/sail-python-udf/src/cereal/mod.rs index 807bcc60fb..799ff72b54 100644 --- a/crates/sail-python-udf/src/cereal/mod.rs +++ b/crates/sail-python-udf/src/cereal/mod.rs @@ -1,5 +1,6 @@ use datafusion::arrow::datatypes::DataType; use pyo3::prelude::PyAnyMethods; +use pyo3::sync::PyOnceLock; use pyo3::types::{PyList, PyModule}; use pyo3::{PyResult, Python, intern}; use sail_common::spec; @@ -10,36 +11,34 @@ use crate::error::{PyUdfError, PyUdfResult}; pub mod pyspark_udf; pub mod pyspark_udtf; +#[derive(Clone, Copy, PartialEq, Eq)] enum PySparkVersion { V3, V4_0, V4_1, -} - -impl PySparkVersion { - fn is_v4(&self) -> bool { - matches!(self, PySparkVersion::V4_0 | PySparkVersion::V4_1) - } + V4_2, } fn get_pyspark_version() -> PyUdfResult { - use pyo3::prelude::PyAnyMethods; - use pyo3::types::PyModule; + static PYSPARK_VERSION: PyOnceLock = PyOnceLock::new(); Python::attach(|py| { - let module = PyModule::import(py, "pyspark")?; - let version: String = module.getattr("__version__")?.extract()?; - if version.starts_with("3.") { - Ok(PySparkVersion::V3) - } else if version.starts_with("4.0.") { - Ok(PySparkVersion::V4_0) - } else if version.starts_with("4.") { - Ok(PySparkVersion::V4_1) - } else { - Err(PyUdfError::invalid(format!( - "unsupported PySpark version: {version}" - ))) - } + PYSPARK_VERSION + .get_or_try_init(py, || { + let module = PyModule::import(py, "pyspark")?; + let version: String = module.getattr("__version__")?.extract()?; + let parts: Vec<_> = version.split('.').collect(); + match parts.as_slice() { + ["3", ..] => Ok(PySparkVersion::V3), + ["4", "0", ..] => Ok(PySparkVersion::V4_0), + ["4", "1", ..] => Ok(PySparkVersion::V4_1), + ["4", "2", ..] => Ok(PySparkVersion::V4_2), + _ => Err(PyUdfError::invalid(format!( + "unsupported PySpark version: {version}" + ))), + } + }) + .copied() }) } @@ -104,6 +103,16 @@ pub(crate) fn write_kwarg(data: &mut Vec, kwargs: &[Option], index: } } +pub(crate) fn write_conf(data: &mut Vec, conf: Vec<(String, String)>) { + data.extend((conf.len() as i32).to_be_bytes()); + for (key, value) in conf { + data.extend((key.len() as i32).to_be_bytes()); + data.extend(key.as_bytes()); + data.extend((value.len() as i32).to_be_bytes()); + data.extend(value.as_bytes()); + } +} + fn should_write_config(eval_type: spec::PySparkUdfType) -> bool { use spec::PySparkUdfType; diff --git a/crates/sail-python-udf/src/cereal/pyspark_udf.rs b/crates/sail-python-udf/src/cereal/pyspark_udf.rs index 3ad8d9bccd..caabdffbc0 100644 --- a/crates/sail-python-udf/src/cereal/pyspark_udf.rs +++ b/crates/sail-python-udf/src/cereal/pyspark_udf.rs @@ -7,7 +7,7 @@ use sail_common::spec; use crate::cereal::{ PySparkVersion, build_input_types_json, check_python_udf_version, get_pyspark_version, - should_write_config, supports_kwargs, write_kwarg, + should_write_config, supports_kwargs, write_conf, write_kwarg, }; use crate::config::PySparkUdfConfig; use crate::error::{PyUdfError, PyUdfResult}; @@ -29,9 +29,20 @@ impl PySparkUdfPayload { let serializer = PyModule::import(py, intern!(py, "pyspark.serializers"))? .getattr(intern!(py, "CPickleSerializer"))? .call0()?; - let tuple = PyModule::import(py, intern!(py, "pyspark.worker"))? - .getattr(intern!(py, "read_udfs"))? - .call1((serializer, infile, eval_type))?; + let worker = PyModule::import(py, intern!(py, "pyspark.worker"))?; + let read_udfs = worker.getattr(intern!(py, "read_udfs"))?; + let tuple = match get_pyspark_version()? { + PySparkVersion::V4_2 => { + let runner_conf = worker + .getattr(intern!(py, "RunnerConf"))? + .call1((&infile,))?; + let eval_conf = worker.getattr(intern!(py, "EvalConf"))?.call1((&infile,))?; + read_udfs.call1((serializer, infile, eval_type, runner_conf, eval_conf))? + } + PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => { + read_udfs.call1((serializer, infile, eval_type))? + } + }; tuple .get_item(0)? .into_pyobject(py) @@ -54,29 +65,41 @@ impl PySparkUdfPayload { data.extend(i32::from(eval_type).to_be_bytes()); - if should_write_config(eval_type) { - let config = config.to_key_value_pairs(); - data.extend((config.len() as i32).to_be_bytes()); // number of configuration options - for (key, value) in config { - data.extend((key.len() as i32).to_be_bytes()); // length of the key - data.extend(key.as_bytes()); - data.extend((value.len() as i32).to_be_bytes()); // length of the value - data.extend(value.as_bytes()); + match pyspark_version { + PySparkVersion::V4_2 => { + // Spark 4.2 reads both maps before dispatching to read_udfs. + write_conf(&mut data, config.to_key_value_pairs()); + let mut eval_conf = vec![]; + if eval_type == spec::PySparkUdfType::ArrowBatched { + eval_conf.push(( + "input_type".to_string(), + build_input_types_json(input_types)?, + )) + } + write_conf(&mut data, eval_conf); } - } + PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => { + if should_write_config(eval_type) { + write_conf(&mut data, config.to_key_value_pairs()); + } - // PySpark 4.1+ reads input types for ArrowBatched UDFs. - // PySpark 4.0.x does not read input types and would misparse the stream. - if matches!(pyspark_version, PySparkVersion::V4_1) - && matches!(eval_type, spec::PySparkUdfType::ArrowBatched) - { - let schema_json = build_input_types_json(input_types)?; - data.extend((schema_json.len() as i32).to_be_bytes()); - data.extend(schema_json.as_bytes()); + // PySpark 4.1 reads input types for ArrowBatched UDFs. + // PySpark 4.0.x does not read input types and would misparse the stream. + if pyspark_version == PySparkVersion::V4_1 + && eval_type == spec::PySparkUdfType::ArrowBatched + { + let schema_json = build_input_types_json(input_types)?; + data.extend((schema_json.len() as i32).to_be_bytes()); + data.extend(schema_json.as_bytes()); + } + } } - if pyspark_version.is_v4() { - data.extend(0u8.to_be_bytes()); // profiling is not enabled + match pyspark_version { + PySparkVersion::V4_0 | PySparkVersion::V4_1 => { + data.extend(0u8.to_be_bytes()); // profiling is not enabled + } + PySparkVersion::V3 | PySparkVersion::V4_2 => {} } data.extend(1i32.to_be_bytes()); // number of UDFs @@ -87,7 +110,11 @@ impl PySparkUdfPayload { .map_err(|e| PyUdfError::invalid(format!("num args: {e}")))?; data.extend(num_arg_offsets.to_be_bytes()); // number of argument offsets - let allow_kwargs = pyspark_version.is_v4() && supports_kwargs(eval_type); + let allow_kwargs = match pyspark_version { + PySparkVersion::V4_2 => true, + PySparkVersion::V4_0 | PySparkVersion::V4_1 => supports_kwargs(eval_type), + PySparkVersion::V3 => false, + }; for (i, offset) in arg_offsets.iter().enumerate() { let offset: i32 = (*offset) @@ -103,6 +130,14 @@ impl PySparkUdfPayload { data.extend((command.len() as i32).to_be_bytes()); // length of the function data.extend_from_slice(command); + match pyspark_version { + PySparkVersion::V4_2 => { + // Spark 4.2 always sends this field, even with profiling disabled. + data.extend(0i64.to_be_bytes()); + } + PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => {} + } + Ok(data) } } diff --git a/crates/sail-python-udf/src/cereal/pyspark_udtf.rs b/crates/sail-python-udf/src/cereal/pyspark_udtf.rs index 1dd293763b..0a4a0a151a 100644 --- a/crates/sail-python-udf/src/cereal/pyspark_udtf.rs +++ b/crates/sail-python-udf/src/cereal/pyspark_udtf.rs @@ -9,7 +9,7 @@ use sail_pyarrow::{FromPyArrow, ToPyArrow}; use crate::cereal::{ PySparkVersion, build_input_types_json, check_python_udf_version, get_pyspark_version, - should_write_config, supports_kwargs, write_kwarg, + should_write_config, supports_kwargs, write_conf, write_kwarg, }; use crate::config::PySparkUdfConfig; use crate::error::{PyUdfError, PyUdfResult}; @@ -31,9 +31,20 @@ impl PySparkUdtfPayload { let serializer = PyModule::import(py, intern!(py, "pyspark.serializers"))? .getattr(intern!(py, "CPickleSerializer"))? .call0()?; - let tuple = PyModule::import(py, intern!(py, "pyspark.worker"))? - .getattr(intern!(py, "read_udtf"))? - .call1((serializer, infile, eval_type))?; + let worker = PyModule::import(py, intern!(py, "pyspark.worker"))?; + let read_udtf = worker.getattr(intern!(py, "read_udtf"))?; + let tuple = match get_pyspark_version()? { + PySparkVersion::V4_2 => { + let runner_conf = worker + .getattr(intern!(py, "RunnerConf"))? + .call1((&infile,))?; + let eval_conf = worker.getattr(intern!(py, "EvalConf"))?.call1((&infile,))?; + read_udtf.call1((serializer, infile, eval_type, runner_conf, eval_conf))? + } + PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => { + read_udtf.call1((serializer, infile, eval_type))? + } + }; tuple .get_item(0)? .into_pyobject(py) @@ -126,12 +137,14 @@ impl PySparkUdtfPayload { }) } + #[expect(clippy::too_many_arguments)] pub fn build( python_version: &str, command: &[u8], eval_type: spec::PySparkUdfType, num_args: usize, input_types: &[DataType], + passthrough_columns: usize, kwargs: &[Option], return_type: &DataType, config: &PySparkUdfConfig, @@ -142,39 +155,67 @@ impl PySparkUdtfPayload { data.extend(i32::from(eval_type).to_be_bytes()); // Add eval_type for extraction in visit_bytes - if should_write_config(eval_type) { - let config = config.to_key_value_pairs(); - data.extend((config.len() as i32).to_be_bytes()); // number of configuration options - for (key, value) in config { - data.extend((key.len() as i32).to_be_bytes()); // length of the key - data.extend(key.as_bytes()); - data.extend((value.len() as i32).to_be_bytes()); // length of the value - data.extend(value.as_bytes()); + match pyspark_version { + PySparkVersion::V4_2 => { + write_conf(&mut data, config.to_key_value_pairs()); + let mut eval_conf = vec![]; + if eval_type == spec::PySparkUdfType::ArrowTable { + let argument_types = input_types + .get(passthrough_columns..) + .ok_or_else(|| { + PyUdfError::invalid(format!( + "passthrough columns ({passthrough_columns}) exceed input columns ({})", + input_types.len() + )) + })?; + eval_conf.push(( + "input_type".to_string(), + build_input_types_json(argument_types)?, + )) + } + write_conf(&mut data, eval_conf); } - } + PySparkVersion::V3 | PySparkVersion::V4_0 | PySparkVersion::V4_1 => { + if should_write_config(eval_type) { + write_conf(&mut data, config.to_key_value_pairs()); + } - // PySpark 4.1+ reads input types for ArrowTable UDTFs. - // PySpark 4.0.x does not read input types and would misparse the stream. - if matches!(pyspark_version, PySparkVersion::V4_1) - && matches!(eval_type, spec::PySparkUdfType::ArrowTable) - { - let schema_json = build_input_types_json(input_types)?; - data.extend((schema_json.len() as i32).to_be_bytes()); - data.extend(schema_json.as_bytes()); - } + // PySpark 4.1+ reads input types for ArrowTable UDTFs. + // PySpark 4.0.x does not read input types and would misparse the stream. + if pyspark_version == PySparkVersion::V4_1 + && eval_type == spec::PySparkUdfType::ArrowTable + { + let argument_types = input_types + .get(passthrough_columns..) + .ok_or_else(|| { + PyUdfError::invalid(format!( + "passthrough columns ({passthrough_columns}) exceed input columns ({})", + input_types.len() + )) + })?; + let schema_json = build_input_types_json(argument_types)?; + data.extend((schema_json.len() as i32).to_be_bytes()); + data.extend(schema_json.as_bytes()); + } - // PySpark 4.1+ reads table argument offsets for ArrowUdtf before the arg offsets. - // PySpark 4.0.x does not use the ArrowUdtf eval type. - if matches!(pyspark_version, PySparkVersion::V4_1) - && matches!(eval_type, spec::PySparkUdfType::ArrowUdtf) - { - data.extend(0i32.to_be_bytes()); // num_table_arg_offsets = 0 + // PySpark 4.1+ reads table argument offsets for ArrowUdtf before the arg offsets. + // PySpark 4.0.x does not use the ArrowUdtf eval type. + if pyspark_version == PySparkVersion::V4_1 + && eval_type == spec::PySparkUdfType::ArrowUdtf + { + data.extend(0i32.to_be_bytes()); // num_table_arg_offsets = 0 + } + } } let num_args: i32 = num_args .try_into() .map_err(|e| PyUdfError::invalid(format!("num_args: {e}")))?; - let allow_kwargs = pyspark_version.is_v4() && supports_kwargs(eval_type); + let allow_kwargs = match pyspark_version { + PySparkVersion::V4_2 => true, + PySparkVersion::V4_0 | PySparkVersion::V4_1 => supports_kwargs(eval_type), + PySparkVersion::V3 => false, + }; data.extend(num_args.to_be_bytes()); // number of arguments for index in 0..num_args { data.extend(index.to_be_bytes()); // argument offset @@ -183,9 +224,12 @@ impl PySparkUdtfPayload { } } - if pyspark_version.is_v4() { - data.extend(0i32.to_be_bytes()); // number of partition child indexes - data.extend(0u8.to_be_bytes()); // pickled analyze result is not present + match pyspark_version { + PySparkVersion::V4_0 | PySparkVersion::V4_1 | PySparkVersion::V4_2 => { + data.extend(0i32.to_be_bytes()); // number of partition child indexes + data.extend(0u8.to_be_bytes()); // pickled analyze result is not present + } + PySparkVersion::V3 => {} } data.extend((command.len() as i32).to_be_bytes()); // length of the function @@ -203,9 +247,12 @@ impl PySparkUdtfPayload { data.extend((type_string.len() as u32).to_be_bytes()); data.extend(type_string.as_bytes()); - if pyspark_version.is_v4() { - // TODO: support UDTF name - data.extend(0u32.to_be_bytes()); // length of UDTF name + match pyspark_version { + PySparkVersion::V4_0 | PySparkVersion::V4_1 | PySparkVersion::V4_2 => { + // TODO: support UDTF name + data.extend(0u32.to_be_bytes()); // length of UDTF name + } + PySparkVersion::V3 => {} } Ok(data) diff --git a/crates/sail-python-udf/src/config.rs b/crates/sail-python-udf/src/config.rs index fa2b557ae1..f03e1687e4 100644 --- a/crates/sail-python-udf/src/config.rs +++ b/crates/sail-python-udf/src/config.rs @@ -20,6 +20,8 @@ pub struct PySparkUdfConfig { #[pyo3(get)] pub python_udf_pandas_int_to_decimal_coercion_enabled: bool, #[pyo3(get)] + pub python_udf_pandas_prefer_int_extension_dtype: bool, + #[pyo3(get)] pub binary_as_bytes: bool, } @@ -34,6 +36,7 @@ impl Default for PySparkUdfConfig { python_udf_pandas_conversion_enabled: false, python_udtf_pandas_conversion_enabled: false, python_udf_pandas_int_to_decimal_coercion_enabled: false, + python_udf_pandas_prefer_int_extension_dtype: false, binary_as_bytes: true, } } @@ -82,6 +85,11 @@ impl PySparkUdfConfig { self.python_udf_pandas_int_to_decimal_coercion_enabled .to_string(), )); + out.push(( + "spark.sql.execution.pythonUDF.pandas.preferIntExtensionDtype".to_string(), + self.python_udf_pandas_prefer_int_extension_dtype + .to_string(), + )); out.push(( "spark.sql.execution.pyspark.binaryAsBytes".to_string(), self.binary_as_bytes.to_string(), diff --git a/crates/sail-python-udf/src/python/spark.py b/crates/sail-python-udf/src/python/spark.py index 2efce2f36d..f11c31f041 100644 --- a/crates/sail-python-udf/src/python/spark.py +++ b/crates/sail-python-udf/src/python/spark.py @@ -13,7 +13,7 @@ import pyspark from pyspark.sql.pandas.serializers import ArrowStreamPandasUDFSerializer, ArrowStreamPandasUDTFSerializer from pyspark.sql.pandas.types import from_arrow_type -from pyspark.sql.types import Row +from pyspark.sql.types import DataType, Row, StructField, StructType _PYARROW_HAS_VIEW_TYPES = all(hasattr(pa, x) for x in ("list_view", "large_list_view", "string_view", "binary_view")) @@ -67,6 +67,15 @@ def batched(iterable, n): _LocalDataToArrowConversion = None +PYSPARK_VERSION: tuple[int, ...] = tuple(int(part) for part in pyspark.__version__.split(".")) + + +def _pandas_serializer_kwargs(config) -> dict[str, bool]: + if PYSPARK_VERSION >= (4, 2): + return {"prefer_int_ext_dtype": config.python_udf_pandas_prefer_int_extension_dtype} + return {} + + class Converter: """ A converter that converts between PySpark data and Arrow data. @@ -477,24 +486,94 @@ def _field_values(self, data: Any) -> Sequence[Any]: return self._spark_data_type.toInternal(data) -if pyspark.__version__.startswith(("3.", "4.0.")): +if (3,) <= PYSPARK_VERSION < (4, 1): def _arrow_column_to_pandas(column: pa.Array, serializer: ArrowStreamPandasUDFSerializer): return serializer.arrow_to_pandas(column) -else: +elif (4, 1) <= PYSPARK_VERSION < (4, 2): def _arrow_column_to_pandas(column: pa.Array, serializer: ArrowStreamPandasUDFSerializer): return serializer.arrow_to_pandas(column, 0) +else: + + def _arrow_column_to_pandas(column: pa.Array, serializer: ArrowStreamPandasUDFSerializer): + from pyspark.sql.conversion import ArrowBatchTransformer + + schema = StructType([StructField("_0", from_arrow_type(column.type), True)]) + batch = pa.RecordBatch.from_arrays([column], ["_0"]) + return ArrowBatchTransformer.to_pandas( + batch, + timezone=serializer._timezone, # noqa: SLF001 + schema=schema, + struct_in_pandas=serializer._struct_in_pandas, # noqa: SLF001 + ndarray_as_list=serializer._ndarray_as_list, # noqa: SLF001 + prefer_int_ext_dtype=serializer._prefer_int_ext_dtype, # noqa: SLF001 + df_for_struct=serializer._df_for_struct, # noqa: SLF001 + )[0] + + +if (3,) <= PYSPARK_VERSION < (4, 2): + + def _pandas_to_arrow_array(data, data_type: pa.DataType, serializer: ArrowStreamPandasUDFSerializer) -> pa.Array: + if ( + serializer._struct_in_pandas == "dict" # noqa: SLF001 + and pa.types.is_struct(data_type) + and not _is_variant_struct_type(data_type) + ): + return serializer._create_struct_array(data, data_type) # noqa: SLF001 + return serializer._create_array(data, data_type, arrow_cast=serializer._arrow_cast) # noqa: SLF001 +else: + + def _pandas_to_arrow_array(data, data_type: pa.DataType, serializer: ArrowStreamPandasUDFSerializer) -> pa.Array: + from pyspark.sql.conversion import PandasToArrowConversion + + # Spark 4.2's worker wrappers return the Spark return type, whereas + # prior versions return its PyArrow counterpart. + spark_type = data_type if isinstance(data_type, DataType) else from_arrow_type(data_type) + schema = StructType([StructField("_0", spark_type, True)]) + return PandasToArrowConversion.convert( + [data], + schema, + timezone=serializer._timezone, # noqa: SLF001 + safecheck=serializer._safecheck, # noqa: SLF001 + arrow_cast=serializer._arrow_cast, # noqa: SLF001 + prefers_large_types=serializer._prefers_large_types, # noqa: SLF001 + assign_cols_by_name=serializer._assign_cols_by_name, # noqa: SLF001 + int_to_decimal_coercion_enabled=serializer._int_to_decimal_coercion_enabled, # noqa: SLF001 + ignore_unexpected_complex_type_values=serializer._ignore_unexpected_complex_type_values, # noqa: SLF001 + is_legacy=serializer._is_legacy, # noqa: SLF001 + ).column(0) + + +def _record_batch( + args: Sequence[pa.Array], *, names: Sequence[str] | None = None, num_rows: int | None = None +) -> pa.RecordBatch: + if args: + return pa.RecordBatch.from_arrays(args, names if names is not None else [f"_{i}" for i in range(len(args))]) + # Older supported PyArrow releases do not expose a row-count argument on + # RecordBatch.from_arrays. Select away a temporary column instead. + return pa.RecordBatch.from_arrays([pa.nulls(num_rows or 0)], ["_"]).select([]) + + +def _output_column(output: Iterator[pa.RecordBatch]) -> pa.Array: + arrays = [] + for batch in output: + if isinstance(batch, pa.Table): + arrays.extend(part.column(0) for part in batch.to_batches()) + else: + arrays.append(batch.column(0)) + if not arrays: + return pa.array([]) + return arrays[0] if len(arrays) == 1 else pa.concat_arrays(arrays) -def _pandas_to_arrow_array(data, data_type: pa.DataType, serializer: ArrowStreamPandasUDFSerializer) -> pa.Array: - if ( - serializer._struct_in_pandas == "dict" # noqa: SLF001 - and pa.types.is_struct(data_type) - and not _is_variant_struct_type(data_type) - ): - return serializer._create_struct_array(data, data_type) # noqa: SLF001 - return serializer._create_array(data, data_type, arrow_cast=serializer._arrow_cast) # noqa: SLF001 +def _wrap_struct(batch: pa.RecordBatch) -> pa.RecordBatch: + return pa.RecordBatch.from_arrays([batch.to_struct_array()], ["_0"]) + + +def _flatten_struct(batch: pa.RecordBatch) -> pa.RecordBatch: + struct = batch.column(0) + return pa.RecordBatch.from_arrays(struct.flatten(), schema=pa.schema(struct.type)) def _arrow_columns_to_python(args: list[pa.Array], *, binary_as_bytes: bool = True) -> tuple: @@ -603,7 +682,7 @@ def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array: class PySparkArrowBatchUdf: def __init__(self, udf: Callable[..., Any], config): self._udf = udf - self._use_legacy = config.python_udf_pandas_conversion_enabled or pyspark.__version__.startswith(("3.", "4.0.")) + self._use_legacy = config.python_udf_pandas_conversion_enabled or (3,) <= PYSPARK_VERSION < (4, 1) if self._use_legacy: self._serializer = ArrowStreamPandasUDFSerializer( timezone=config.session_timezone, @@ -613,6 +692,7 @@ def __init__(self, udf: Callable[..., Any], config): struct_in_pandas="row", ndarray_as_list=True, arrow_cast=True, + **_pandas_serializer_kwargs(config), ) else: self._binary_as_bytes = config.binary_as_bytes @@ -620,6 +700,8 @@ def __init__(self, udf: Callable[..., Any], config): self._safecheck = config.arrow_convert_safely def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array: + if PYSPARK_VERSION >= (4, 2): + return _output_column(self._udf(None, iter([_record_batch(args, num_rows=num_rows)]))) if self._use_legacy: return self._call_legacy(args, num_rows) return self._call_arrow(args, num_rows) @@ -675,6 +757,7 @@ def __init__( struct_in_pandas="dict", ndarray_as_list=False, arrow_cast=False, + **_pandas_serializer_kwargs(config), ) def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array: @@ -698,6 +781,7 @@ def __init__( struct_in_pandas="dict", ndarray_as_list=False, arrow_cast=False, + **_pandas_serializer_kwargs(config), ) def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array: @@ -720,7 +804,9 @@ def __init__( ): self._udf = udf - def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array: + def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array: + if PYSPARK_VERSION >= (4, 2): + return _output_column(self._udf(None, iter([_record_batch(args, num_rows=num_rows)]))) inputs = tuple(args) [(output, output_type)] = list(self._udf(None, (inputs,))) if isinstance(output, pa.ChunkedArray): @@ -747,7 +833,9 @@ def __init__( ): self._udf = udf - def __call__(self, args: list[pa.Array], _num_rows: int) -> pa.Array: + def __call__(self, args: list[pa.Array], num_rows: int) -> pa.Array: + if PYSPARK_VERSION >= (4, 2): + return _output_column(self._udf(None, iter([_record_batch(args, num_rows=num_rows)]))) inputs = tuple(args) [(output, output_type)] = list(self._udf(None, [inputs])) if isinstance(output, pa.ChunkedArray): @@ -777,9 +865,14 @@ def __init__( struct_in_pandas="dict", ndarray_as_list=False, arrow_cast=False, + **_pandas_serializer_kwargs(config), ) def __call__(self, args: list[pa.Array]) -> pa.Array: + if PYSPARK_VERSION >= (4, 2): + inputs = _named_arrays_to_pandas(args, self._input_names, self._serializer) + [(output, output_type)] = list(self._udf(None, ([inputs],))) + return _pandas_to_arrow_array(output, output_type, self._serializer) inputs = _named_arrays_to_pandas(args, self._input_names, self._serializer) [(output, output_type)] = list(self._udf(None, (inputs,))) return _pandas_to_arrow_array(output, output_type, self._serializer) @@ -800,6 +893,9 @@ def __init__( self._udf = udf def __call__(self, args: list[pa.Array]) -> pa.Array: + if PYSPARK_VERSION >= (4, 2): + batch = _record_batch(args) + return _output_column(self._udf(None, iter([iter([batch])]))) [(output, output_type)] = list(self._udf(None, (args,))) if isinstance(output, pa.ChunkedArray): output = output.combine_chunks() @@ -836,9 +932,24 @@ def __init__( struct_in_pandas="dict", ndarray_as_list=False, arrow_cast=False, + **_pandas_serializer_kwargs(config), ) def __call__(self, args: list[pa.Array]) -> pa.Array: + if PYSPARK_VERSION >= (4, 2): + batch = pa.RecordBatch.from_arrays(args, self._input_names) + if self.is_pandas: + if self.is_iter: + result = self._udf(None, iter([iter([batch])])) + arrays = [ + _pandas_to_arrow_array(output, output_type, self._serializer) + for group_outputs in result + for outputs in group_outputs + for output, output_type in outputs + ] + return pa.array([]) if not arrays else pa.concat_arrays(arrays) + return _output_column(self._udf(None, iter([iter([batch])]))) + return _output_column(self._udf(None, iter([iter([_wrap_struct(batch)])]))) m = self._max_records_per_batch if self.is_pandas: inputs = _named_arrays_to_pandas(args, self._input_names, self._serializer) @@ -898,6 +1009,7 @@ def __init__( struct_in_pandas="dict", ndarray_as_list=False, arrow_cast=False, + **_pandas_serializer_kwargs(config), ) def __call__(self, left: list[pa.Array], right: list[pa.Array]) -> pa.Array: @@ -932,6 +1044,7 @@ def __init__( struct_in_pandas="dict", ndarray_as_list=False, arrow_cast=False, + **_pandas_serializer_kwargs(config), ) def __call__(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]: @@ -951,6 +1064,9 @@ def __init__(self, udf: Callable[..., Iterator[pa.RecordBatch]]): self._udf = udf def __call__(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]: + if PYSPARK_VERSION >= (4, 2): + output = self._udf(None, (_wrap_struct(batch) for batch in args)) + return (_flatten_struct(batch) for batch in output) output = self._udf(None, ((x,) for x in args)) return (x for x, _ in output) @@ -1025,15 +1141,13 @@ def __init__( self._passthrough_columns = passthrough_columns self._output_schema = output_schema self._output_type = pa.struct([output_schema.field(i) for i in range(len(output_schema.names))]) - self._use_legacy = config.python_udtf_pandas_conversion_enabled or pyspark.__version__.startswith( - ("3.", "4.0.") - ) + self._use_legacy = config.python_udtf_pandas_conversion_enabled or (3,) <= PYSPARK_VERSION < (4, 1) if self._use_legacy: - if pyspark.__version__.startswith(("3.", "4.0.")): + if (3,) <= PYSPARK_VERSION < (4, 1): self._serializer = ArrowStreamPandasUDTFSerializer( timezone=config.session_timezone, safecheck=config.arrow_convert_safely ) - else: + elif (4, 1) <= PYSPARK_VERSION < (4, 2): spark_input_types = [from_arrow_type(t) for t in input_types] self._serializer = ArrowStreamPandasUDTFSerializer( timezone=config.session_timezone, @@ -1041,6 +1155,20 @@ def __init__( input_types=spark_input_types, int_to_decimal_coercion_enabled=config.python_udf_pandas_int_to_decimal_coercion_enabled, ) + else: + input_type = StructType( + [ + StructField(name, from_arrow_type(data_type), True) + for name, data_type in zip(input_names, input_types, strict=True) + ] + ) + self._serializer = ArrowStreamPandasUDTFSerializer( + timezone=config.session_timezone, + safecheck=config.arrow_convert_safely, + input_type=input_type, + prefer_int_ext_dtype=config.python_udf_pandas_prefer_int_extension_dtype, + int_to_decimal_coercion_enabled=config.python_udf_pandas_int_to_decimal_coercion_enabled, + ) def __call__(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]: if self._use_legacy: @@ -1052,10 +1180,21 @@ def _call_arrow(self, args: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch if first is None: return args = itertools.chain([first], args) - # Tee: one branch for the mapper (full batches), one for passthrough extraction. - # The mapper expects full input batches — it handles Python conversion and column - # selection internally (via converters + args_kwargs_offsets). + # Tee: one branch for the mapper, one for passthrough extraction. batches_for_passthrough, batches_for_mapper = itertools.tee(args) + if PYSPARK_VERSION >= (4, 1): + # Spark 4.1+ interprets UDTF argument offsets relative to the input + # supplied to its worker. Sail's execution batches also include + # lateral-join passthrough columns, so remove those before invoking + # the worker and retain the original stream for result expansion. + def strip_passthrough(batch: pa.RecordBatch) -> pa.RecordBatch: + return _record_batch( + batch.columns[self._passthrough_columns :], + names=batch.schema.names[self._passthrough_columns :], + num_rows=batch.num_rows, + ) + + batches_for_mapper = (strip_passthrough(batch) for batch in batches_for_mapper) outputs = self._udf(None, batches_for_mapper) # Extract passthrough values from raw arrow batches (no pandas) columns = (tuple(batch.column(i) for i in range(batch.num_columns)) for batch in batches_for_passthrough) @@ -1094,7 +1233,7 @@ def _iter_input_legacy(self, args: Iterator[pa.RecordBatch]) -> Iterator[tuple[p for batch in args: arrays = batch.to_struct_array().flatten() columns = tuple(_arrow_column_to_pandas(x, self._serializer) for x in arrays) - if len(columns) == 0 and not pyspark.__version__.startswith(("3.", "4.0.")): + if len(columns) == 0 and PYSPARK_VERSION >= (4, 1): # PySpark 4.1+ legacy mapper uses len(a[0]) for num_rows. # Preserve the row count for 0-column batches (e.g. 0-arg UDTFs). yield (pd.Series(range(batch.num_rows)),) @@ -1106,12 +1245,9 @@ def _iter_output_legacy(self, args: Iterator[pa.RecordBatch]) -> Iterator[pd.Dat if sum(1 for _ in args2) == 0: return (batches1, batches2) = itertools.tee(self._iter_input_legacy(args1)) - if pyspark.__version__.startswith(("3.", "4.0.")): - # PySpark 3.x/4.0 mapper receives sliced non-passthrough columns. - inputs = (x[self._passthrough_columns :] for x in batches2) - else: - # PySpark 4.1+ mapper receives full tuples and selects columns via args_kwargs_offsets. - inputs = batches2 + # PySpark 3.x/4.0 mapper receives sliced non-passthrough columns. + # PySpark 4.1+ receives full tuples and selects columns via args_kwargs_offsets. + inputs = (x[self._passthrough_columns :] for x in batches2) if (3,) <= PYSPARK_VERSION < (4, 1) else batches2 outputs = self._udf(None, inputs) last = None for passthrough, (out, *_) in itertools.zip_longest(self._iter_passthrough(batches1), outputs): diff --git a/crates/sail-spark-connect/data/spark_config.json b/crates/sail-spark-connect/data/spark_config.json index 292c372495..b500dc6c50 100644 --- a/crates/sail-spark-connect/data/spark_config.json +++ b/crates/sail-spark-connect/data/spark_config.json @@ -1,5 +1,5 @@ { - "sparkVersion": "4.1.1", + "sparkVersion": "4.2.0", "comment": "This is a generated file. Do not edit it directly.", "entries": [ { @@ -166,11 +166,21 @@ "doc": "DeduplicateRelations shouldn't remap expressions to new ExprIds if old ExprId still exists in output.", "defaultValue": "true" }, + { + "key": "spark.sql.analyzer.expandTagPassthroughDuplicates", + "doc": "When true, Expand tags pass-through child attributes that share a name with a grouping attribute using __is_duplicate metadata, so that name-based resolution against the Expand output does not produce AMBIGUOUS_REFERENCE errors.", + "defaultValue": "true" + }, { "key": "spark.sql.analyzer.failAmbiguousSelfJoin", "doc": "When true, fail the Dataset query if it contains ambiguous self-join.", "defaultValue": "true" }, + { + "key": "spark.sql.analyzer.lowerEmptyGroupingSetToGlobalAggregate.enabled", + "doc": "When true, a grand-total GROUP BY GROUPING SETS (()) (and the equivalent empty CUBE() / ROLLUP()) is lowered to a global aggregate during analysis, so it returns one row over empty input, matching an aggregation with no GROUP BY clause. When false, falls back to the legacy Expand-based lowering that returns no rows over empty input.", + "defaultValue": "true" + }, { "key": "spark.sql.analyzer.maxIterations", "doc": "The max number of iterations the analyzer runs.", @@ -251,6 +261,16 @@ "doc": "When true, applies the conf overrides for certain feature flags during the resolution of user-defined sql table valued functions, consistent with view resolution.", "defaultValue": "true" }, + { + "key": "spark.sql.analyzer.strictDataFrameColumnResolution", + "doc": "When true (default), enforce strict resolution of Spark Connect DataFrame columns (UnresolvedAttribute carrying a plan id tag) via plan-id-based resolution. When false, also try name-based resolution as a fallback for tagged attributes.", + "defaultValue": "true" + }, + { + "key": "spark.sql.analyzer.subqueryAliasAlwaysPropagateMetadataColumns", + "doc": "When true, SubqueryAlias always propagates metadata columns from its child. When false, SubqueryAlias only propagates metadata columns if the child is a LeafNode or another SubqueryAlias (legacy behavior).", + "defaultValue": "true" + }, { "key": "spark.sql.analyzer.unionIsResolvedWhenDuplicatesPerChildResolved", "doc": "When true, union should only be resolved once there are no duplicate attributes in each branch.", @@ -290,6 +310,11 @@ "comment": "This was an internal configuration. It is not needed anymore since Spark SQL always returns null when getting a map value with a non-existing key. See SPARK-40066 for more details." } }, + { + "key": "spark.sql.artifact.cacheStorageLevel", + "doc": "Storage level for cached blocks in artifact manager. Valid values are any StorageLevel name (e.g., MEMORY_AND_DISK_SER, DISK_ONLY, MEMORY_ONLY, etc.).", + "defaultValue": "MEMORY_AND_DISK_SER" + }, { "key": "spark.sql.artifact.copyFromLocalToFs.allowDestLocal", "doc": "\nAllow `spark.copyFromLocalToFs` destination to be local file system\n path on spark driver node when\n`spark.sql.artifact.copyFromLocalToFs.allowDestLocal` is true.\nThis will allow user to overwrite arbitrary file on spark\ndriver node we should only enable it for testing purpose.\n" @@ -599,6 +624,31 @@ "doc": "When true, the whole stage (of multiple operators) will be compiled into single java method.", "defaultValue": "true" }, + { + "key": "spark.sql.codegen.wholeStage.union.enabled", + "doc": "When both this conf and `spark.sql.codegen.wholeStage` are true, UnionExec participates in whole-stage codegen on its non-partitioning-aware path: the parent and all children fuse into a single WholeStageCodegenExec stage.", + "defaultValue": "false" + }, + { + "key": "spark.sql.codegen.wholeStage.union.maxChildren", + "doc": "Maximum number of UnionExec children eligible for whole-stage codegen fusion. Each child is emitted as its own helper method, so this conf bounds class-level costs of the fused stage (total bytecode size, constant pool growth, JIT compilation time) rather than the JVM per-method bytecode limit. Unions with more children fall back to per-child codegen stages. Only effective when `spark.sql.codegen.wholeStage.union.enabled` is true.", + "defaultValue": "64" + }, + { + "key": "spark.sql.collation.allowInMapKeys", + "doc": "Allow for non-UTF8_BINARY collated strings inside of map's keys", + "defaultValue": "false" + }, + { + "key": "spark.sql.collation.objectLevel.enabled", + "doc": "Object level collations feature is under development and its use should be done under this feature flag. The feature allows setting default collation for all underlying columns within that object, except the ones that were previously created.", + "defaultValue": "true" + }, + { + "key": "spark.sql.collation.schemaLevel.enabled", + "doc": "Schema level collations feature is under development and its use should be done under this feature flag. The feature allows setting default collation for all underlying objects within that schema, except the ones that were previously created.An object with an explicitly set collation will not inherit the collation from the schema.", + "defaultValue": "false" + }, { "key": "spark.sql.columnNameOfCorruptRecord", "doc": "The name of internal column for storing raw/un-parsed JSON and CSV records that fail to parse.", @@ -724,6 +774,11 @@ "doc": "When true, and DEFAULT columns are enabled, allow INSERT INTO commands with user-specified lists of fewer columns than the target table to behave as if they had specified DEFAULT for all remaining columns instead, in order.", "defaultValue": "true" }, + { + "key": "spark.sql.defaultPath", + "doc": "Default SQL PATH used when no SET PATH has been issued in the session; this is also the value to which `SET PATH = DEFAULT_PATH` expands. Accepts the full SET PATH grammar; an inner DEFAULT_PATH token resolves to the spark-builtin default ordering. The PATH keyword is not allowed in this conf value. When empty, the spark-builtin default ordering controlled by `spark.sql.functionResolution.sessionOrder` applies. Validated for syntax at set time; redundant entries are tolerated (lookup uses first-match resolution). The interactive SET PATH form still rejects static duplicates as a typo guard.", + "defaultValue": "" + }, { "key": "spark.sql.defaultSizeInBytes", "doc": "The default table size used in query planning. By default, it is set to Long.MaxValue which is larger than `spark.sql.autoBroadcastJoinThreshold` to be more conservative. That is to say by default the optimizer will not choose to broadcast a table unless it knows for sure its size is small enough.", @@ -735,6 +790,11 @@ "defaultValue": "true", "isStatic": true }, + { + "key": "spark.sql.dropTableOnView.enabled", + "doc": "When true, DROP TABLE command will work on VIEW as well.", + "defaultValue": "true" + }, { "key": "spark.sql.enforceTypeCoercionBeforeUnionDeduplication.enabled", "doc": "When set to true, we enforce type coercion to run before deduplication of UNION children outputs. Otherwise, order is relative to rule ordering.", @@ -742,7 +802,7 @@ }, { "key": "spark.sql.error.messageFormat", - "doc": "When PRETTY, the error message consists of textual representation of error class, message and query context. The MINIMAL and STANDARD formats are pretty JSON formats where STANDARD includes an additional JSON field `message`. This configuration property influences on error messages of Thrift Server and SQL CLI while running queries.", + "doc": "When PRETTY, the error message consists of textual representation of error class, message and query context. Stack traces are only shown for internal errors (SQLSTATE XX***). When DEBUG, the output is the same as PRETTY but stack traces are always included. The MINIMAL and STANDARD formats are pretty JSON formats where STANDARD includes an additional JSON field `message`. This configuration property influences on error messages of Thrift Server and SQL CLI while running queries.", "defaultValue": "PRETTY" }, { @@ -769,7 +829,7 @@ { "key": "spark.sql.execution.arrow.enabled", "doc": "(Deprecated since Spark 3.0, please set 'spark.sql.execution.arrow.pyspark.enabled'.)", - "defaultValue": "false", + "defaultValue": "true", "deprecated": { "version": "3.0", "comment": "Use 'spark.sql.execution.arrow.pyspark.enabled' instead of it." @@ -829,6 +889,11 @@ "doc": "When true, validate the schema of Arrow batches returned by mapInArrow, mapInPandas and DataSource against the expected schema to ensure that they are compatible.", "defaultValue": "false" }, + { + "key": "spark.sql.execution.arrow.pythonUDF.columnarInput.enabled", + "doc": "When true, Arrow-based Python UDFs (pandas UDFs) can accept columnar input directly from upstream operators that produce Arrow-backed ColumnarBatch (e.g., DataSource V2 connectors), bypassing the ColumnarToRow and ArrowWriter conversion. This optimization reduces data transfer overhead between the JVM and Python worker processes.", + "defaultValue": "true" + }, { "key": "spark.sql.execution.arrow.sparkr.enabled", "doc": "When true, make use of Apache Arrow for columnar data transfers in SparkR. This optimization applies to: 1. createDataFrame when its input is an R DataFrame 2. collect 3. dapply 4. gapply The following data types are unsupported: FloatType, BinaryType, ArrayType, StructType and MapType.", @@ -957,7 +1022,7 @@ { "key": "spark.sql.execution.pythonUDF.arrow.enabled", "doc": "Enable Arrow optimization in regular Python UDFs. This optimization can only be enabled when the given function takes at least one argument.", - "defaultValue": "false" + "defaultValue": "true" }, { "key": "spark.sql.execution.pythonUDF.arrow.legacy.fallbackOnUDT", @@ -969,10 +1034,15 @@ "doc": "When true, convert int to Decimal python objects before converting Pandas.Series to Arrow array during serialization.Disabled by default, impacts performance.", "defaultValue": "false" }, + { + "key": "spark.sql.execution.pythonUDF.pandas.preferIntExtensionDtype", + "doc": "When true, convert integers to Pandas ExtensionDtype (e.g. pandas.Int64Dtype) for Pandas UDF execution. Otherwise, depends on the behavior of pyarrow.Array.to_pandas on each input arrow batch.", + "defaultValue": "false" + }, { "key": "spark.sql.execution.pythonUDTF.arrow.enabled", "doc": "Enable Arrow optimization for Python UDTFs.", - "defaultValue": "false" + "defaultValue": "true" }, { "key": "spark.sql.execution.rangeExchange.sampleSizePerPartition", @@ -1039,6 +1109,11 @@ "defaultValue": "true", "isStatic": true }, + { + "key": "spark.sql.fileSource.insert.enforceNotNull", + "doc": "When true, Spark enforces NOT NULL constraints when inserting data into file-based data source tables (e.g., Parquet, ORC, JSON), consistent with the behavior for other data sources and V2 catalog tables. When false (default), null values are silently accepted into NOT NULL columns.", + "defaultValue": "false" + }, { "key": "spark.sql.files.ignoreCorruptFiles", "doc": "Whether to ignore corrupt files. If true, the Spark jobs will continue to run when encountering corrupted files and the contents that have been read will still be returned. This configuration is effective only when using file-based sources such as Parquet, JSON and ORC.", @@ -1107,10 +1182,20 @@ "doc": "When this option is set to false and all inputs are binary, `elt` returns an output as binary. Otherwise, it returns as a string.", "defaultValue": "false" }, + { + "key": "spark.sql.function.protobufExtensions.enabled", + "doc": "When true, the from_protobuf and to_protobuf operators will support proto2 extensions when a binary file descriptor set is provided. This property will have no effect for the overloads taking a Java class name instead of a file descriptor set.", + "defaultValue": "false" + }, + { + "key": "spark.sql.functionResolution.sessionOrder", + "doc": "Order of the session (temporary) function namespace when resolving unqualified function names. Valid values: 'first', 'second', 'last'. 'first': session -> builtin -> persistent (SQL temp creation blocked if conflicts with builtin). 'second': builtin -> session -> persistent (default). 'last': builtin -> persistent (current schema) -> session.", + "defaultValue": "second" + }, { "key": "spark.sql.geospatial.enabled", "doc": "When true, enables geospatial types (GEOGRAPHY/GEOMETRY) and ST functions.", - "defaultValue": "false" + "defaultValue": "true" }, { "key": "spark.sql.globalTempDatabase", @@ -1141,11 +1226,7 @@ { "key": "spark.sql.hive.convertCTAS", "doc": "When true, a table created by a Hive CTAS statement (no USING clause) without specifying any storage property will be converted to a data source table, using the data source set by spark.sql.sources.default.", - "defaultValue": "false", - "deprecated": { - "version": "3.1", - "comment": "Set 'spark.sql.legacy.createHiveTableByDefault' to false instead." - } + "defaultValue": "false" }, { "key": "spark.sql.hive.dropPartitionByName.enabled", @@ -1246,6 +1327,31 @@ "doc": "When true, enable in-memory table scan accumulators.", "defaultValue": "false" }, + { + "key": "spark.sql.insertIntoReplaceOn.enabled", + "doc": "Enable the SQL syntax INSERT INTO ... REPLACE ON (...). The command atomically inserts new rows into a table after deleting all existing rows that match the new rows according to the specified matching condition. The inserted rows are specified by a VALUES expression or the result of a query.", + "defaultValue": "true" + }, + { + "key": "spark.sql.insertIntoReplaceOnByName.enabled", + "doc": "Enable the SQL syntax INSERT INTO ... BY NAME REPLACE ON. Allows using the BY NAME clause with INSERT INTO REPLACE ON.", + "defaultValue": "true" + }, + { + "key": "spark.sql.insertIntoReplaceUsing.enabled", + "doc": "Enable the SQL syntax INSERT INTO ... REPLACE USING (...). The command atomically inserts new rows into a table after deleting all existing rows that match the new rows according to the key columns specified in the statement. The inserted rows are specified by a VALUES expression or the result of a query.", + "defaultValue": "true" + }, + { + "key": "spark.sql.insertIntoReplaceUsingByName.enabled", + "doc": "Enable the SQL syntax INSERT INTO ... BY NAME REPLACE USING (...). Allows using the BY NAME clause with INSERT INTO REPLACE USING.", + "defaultValue": "true" + }, + { + "key": "spark.sql.insertNestedTypeCoercion.enabled", + "doc": "If enabled, allow INSERT INTO WITH SCHEMA EVOLUTION to fill missing nested struct fields with null when the source has fewer nested fields than the target table. Also relaxes by-position column-count enforcement so trailing missing top-level columns are filled with their default value (or null). This is experimental and the semantics may change.", + "defaultValue": "false" + }, { "key": "spark.sql.join.preferSortMergeJoin", "doc": "When true, prefer sort merge join over shuffled hash join. Sort merge join consumes less memory than shuffled hash join and it works efficiently when both join tables are large. On the other hand, shuffled hash join can improve performance (e.g., of full outer joins) when one of join tables is much smaller.", @@ -1359,6 +1465,11 @@ "doc": "When true, temp view creation Dataset APIs will allow the view creation even if the view name is multiple name parts. The extra name parts will be dropped during the view creation", "defaultValue": "false" }, + { + "key": "spark.sql.legacy.allowUdfParameterToShadowParameterlessFunction", + "doc": "When true (legacy behavior), a SQL UDF parameter alias shadows a parameterless built-in function (current_user, current_date, current_time, current_timestamp, user, session_user, grouping__id) of the same name. When false (the default), the parameterless built-in function takes precedence, matching the documented name resolution rules.", + "defaultValue": "false" + }, { "key": "spark.sql.legacy.allowUntypedScalaUDF", "doc": "When set to true, user is allowed to use org.apache.spark.sql.functions. udf(f: AnyRef, dataType: DataType). Otherwise, an exception will be thrown at runtime.", @@ -1426,6 +1537,11 @@ "doc": "When set to true, encode/decode functions replace unmappable characters with mojibake instead of reporting coding errors.", "defaultValue": "false" }, + { + "key": "spark.sql.legacy.collationAwareHashFunctions", + "doc": "Enables collation aware hashing (legacy behavior) for collated strings in Murmur3Hash and XxHash64 user-facing expressions.", + "defaultValue": "false" + }, { "key": "spark.sql.legacy.compareDateTimestampInTimestamp", "doc": "(Removed)", @@ -1454,6 +1570,11 @@ "key": "spark.sql.legacy.csv.enableDateTimeParsingFallback", "doc": "When true, enable legacy date/time parsing fallback in CSV" }, + { + "key": "spark.sql.legacy.cteDuplicateAttributeNames", + "doc": "When true, CTE references sometimes (e.g., self-joins) may incorrectly unify columnnames that differ only in casing (e.g., COLNAME and colname become the same). This isthe legacy behavior. When false, column name casing is preserved correctly.", + "defaultValue": "false" + }, { "key": "spark.sql.legacy.ctePrecedencePolicy", "doc": "When LEGACY, outer CTE definitions takes precedence over inner definitions. If set to EXCEPTION, AnalysisException is thrown while name conflict is detected in nested CTE. The default is CORRECTED, inner CTE definitions take precedence. This config will be removed in future versions and CORRECTED will be the only behavior.", @@ -1733,6 +1854,11 @@ "doc": "If true, the old bogus percentile_disc calculation is used. The old calculation incorrectly mapped the requested percentile to the sorted range of values in some cases and so returned incorrect results. Also, the new implementation is faster as it doesn't contain the interpolation logic that the old percentile_cont based one did.", "defaultValue": "false" }, + { + "key": "spark.sql.legacy.persistentCatalogFirst", + "doc": "When true (legacy), partially qualified function names like 'builtin.func' or 'session.func' resolve to the persistent catalog first. When false (default), the system catalog is prioritized (system.builtin.func or system.session.func).", + "defaultValue": "false" + }, { "key": "spark.sql.legacy.postgres.datetimeMapping.enabled", "doc": "When true, TimestampType maps to TIMESTAMP WITHOUT TIME ZONE in PostgreSQL for writing; otherwise, TIMESTAMP WITH TIME ZONE. When true, TIMESTAMP WITH TIME ZONE can be converted to TimestampNTZType when JDBC read option preferTimestampNTZ is true; otherwise, converted to TimestampType regardless of preferTimestampNTZ.", @@ -1868,6 +1994,11 @@ "doc": "Minimal increase rate in number of partitions between attempts when executing a take on a query. Higher values lead to more partitions read. Lower values might lead to longer execution times as more jobs will be run", "defaultValue": "4" }, + { + "key": "spark.sql.listagg.allowDistinctCastWithOrder.enabled", + "doc": "When true, LISTAGG(DISTINCT expr) WITHIN GROUP (ORDER BY expr) is allowed on non-string expr when the implicit cast to string preserves equality (e.g., integer, decimal, date). When false, the function argument and ORDER BY expression must have the exact same type, which requires explicit casts.", + "defaultValue": "true" + }, { "key": "spark.sql.mapKeyDedupPolicy", "doc": "The policy to deduplicate map keys in builtin function: CreateMap, MapFromArrays, MapFromEntries, StringToMap, MapConcat and TransformKeys. When EXCEPTION, the query fails if duplicated map keys are detected. When LAST_WIN, the map key that is inserted at last takes precedence.", @@ -1944,6 +2075,11 @@ "doc": "Whether to avoid collapsing projections that would duplicate expensive expressions in UDFs.", "defaultValue": "true" }, + { + "key": "spark.sql.optimizer.avoidDoubleFilterEval", + "doc": "When true avoid pushing expensive (UDF, etc.) filters down if it could result indouble evaluation. This was the behaviour prior to 3.X.", + "defaultValue": "true" + }, { "key": "spark.sql.optimizer.canChangeCachedPlanOutputPartitioning", "doc": "Whether to forcibly enable some optimization rules that can change the output partitioning of a cached query when executing it for caching. If it is set to true, queries may need an extra shuffle to read the cached data. This configuration is disabled by default. The optimization rule enabled by this configuration is spark.sql.adaptive.applyFinalStageShuffleOptimizations.", @@ -2078,11 +2214,31 @@ "doc": "Configures the max set size in InSet for which Spark will generate code with switch statements. This is applicable only to bytes, shorts, ints, dates.", "defaultValue": "400" }, + { + "key": "spark.sql.optimizer.mapLookupHashThreshold", + "doc": "The minimum number of map entries to attempt hash-based lookup in `element_at` and the `[]` operator. Only applies to foldable map expressions (constants / literals), where the hash index can be built once and reused across all rows. Non-foldable maps always use linear scan to avoid per-row hash-table rebuild overhead. Below this threshold, linear scan is used. For key types that do not support hashing (e.g. arrays, structs, binary), linear scan is always used regardless of map size.", + "defaultValue": "1000" + }, { "key": "spark.sql.optimizer.maxIterations", "doc": "The max number of iterations the optimizer runs.", "defaultValue": "100" }, + { + "key": "spark.sql.optimizer.mergeSubplans.filterPropagation.enabled", + "doc": "When set to true, subquery plans that differ only in their filter conditions can be merged by propagating filters up to enclosing non-grouping aggregates.", + "defaultValue": "true" + }, + { + "key": "spark.sql.optimizer.mergeSubplans.filterPropagation.symmetricFilterPropagation.enabled", + "doc": "When set to true, two non-grouping aggregate subplans that both have filter conditions (but with different predicates) can be merged into a single scan using FILTER (WHERE ...) clauses on each aggregate expression. Merging two filtered scans broadens the combined filter to OR(f1, f2), which may reduce IO pruning (e.g. partition or file skipping) compared to the individual filters. Disabled by default; enable once the behaviour has been validated in your workload, particularly on heavily partitioned or file-pruned tables. Has no effect when spark.sql.optimizer.mergeSubplans.filterPropagation.enabled is false.", + "defaultValue": "false" + }, + { + "key": "spark.sql.optimizer.mergeSubplans.filterPropagation.throughJoin.enabled", + "doc": "When set to true, filter attributes can propagate through Join nodes during subplan merging, allowing subplans that differ only in their filter conditions and share a common join to be merged into a single scan. A filter attribute is only propagated through a join when it originates from the non-nullable (preserved) side: the left side of LeftOuter/LeftSemi/LeftAnti, the right side of RightOuter, or either side of Inner/Cross. FullOuter joins are never eligible. Has no effect when spark.sql.optimizer.mergeSubplans.filterPropagation.enabled is false.", + "defaultValue": "false" + }, { "key": "spark.sql.optimizer.metadataOnly", "doc": "When true, enable the metadata-only query optimization that use the table's metadata to produce the partition columns instead of table scans. It applies when all the columns scanned are partition columns and the query has an aggregate operator that satisfies distinct semantics. By default the optimization is disabled, and deprecated as of Spark 3.0 since it may return incorrect results when the files are empty, see also SPARK-26709. It will be removed in the future releases. If you must use, use 'SparkSessionExtensions' instead to inject it as a custom rule.", @@ -2102,6 +2258,11 @@ "doc": "Prune nested fields from a logical relation's output which are unnecessary in satisfying a query. This optimization allows columnar file format readers to avoid reading unnecessary nested column data. Currently Parquet and ORC are the data sources that implement this optimization.", "defaultValue": "true" }, + { + "key": "spark.sql.optimizer.optimizeExpandRatio", + "doc": "Threshold for inserting a pre-aggregation step before the Expand operator produced by RewriteDistinctAggregates. Queries with multiple COUNT(DISTINCT) on different columns are rewritten using an Expand that duplicates each row N times (one per distinct group). When N >= this threshold, a de-duplication aggregate on (grouping keys + all distinct columns) is inserted before the Expand to reduce the amplified data volume. Only applies to pure distinct aggregates without non-distinct aggregates or FILTER clauses. Set to -1 to disable.", + "defaultValue": "-1" + }, { "key": "spark.sql.optimizer.optimizeOneRowRelationSubquery", "doc": "When true, the optimizer will inline subqueries with OneRowRelation as leaf nodes.", @@ -2174,6 +2335,11 @@ "doc": "Handle correlation over nested data extract expressions by pulling out the expression into the outer plan. This enables correlation on map attributes for example.", "defaultValue": "true" }, + { + "key": "spark.sql.optimizer.pushDownJoinThroughUnion.enabled", + "doc": "When true, pushes down Join through Union when the right side is small enough to broadcast. This can improve performance by allowing each Union branch to directly perform a broadcast join, avoiding materializing the entire Union result.", + "defaultValue": "false" + }, { "key": "spark.sql.optimizer.replaceExceptWithFilter", "doc": "When true, the apply function of the rule verifies whether the right node of the except operation is of type Filter or Project followed by Filter. If yes, the rule further verifies 1) Excluding the filter operations from the right (as well as the left node, if any) on the top, whether both the nodes evaluates to a same result. 2) The left and right nodes don't contain any SubqueryExpressions. 3) The output column names of the left node are distinct. If all the conditions are met, the rule will replace the except operation with a Filter by flipping the filter condition(s) of the right node.", @@ -2216,7 +2382,7 @@ }, { "key": "spark.sql.optimizer.runtime.rowLevelOperationGroupFilter.enabled", - "doc": "Enables runtime group filtering for group-based row-level operations. Data sources that replace groups of data (e.g. files, partitions) may prune entire groups using provided data source filters when planning a row-level operation scan. However, such filtering is limited as not all expressions can be converted into data source filters and some expressions can only be evaluated by Spark (e.g. subqueries). Since rewriting groups is expensive, Spark can execute a query at runtime to find what records match the condition of the row-level operation. The information about matching records will be passed back to the row-level operation scan, allowing data sources to discard groups that don't have to be rewritten.", + "doc": "Enables runtime filtering for group-based and delta-based row-level operations. Data sources may prune entire file groups at runtime when planning a row-level operation scan. Planning-time filter pushdown is limited as not all expressions can be converted into data source filters and some expressions can only be evaluated by Spark (e.g. subqueries). Since rewriting groups or scanning unnecessary files is expensive, Spark can execute a lightweight query at runtime to find what records match the condition of the row-level operation. The information about matching records will be passed back to the row-level operation scan, allowing data sources to skip files that don't have to be processed.", "defaultValue": "true" }, { @@ -2460,6 +2626,11 @@ "doc": "For IN predicate, Parquet filter will push-down a set of OR clauses if its number of values not exceeds this threshold. Otherwise, Parquet filter will push-down a value greater than or equal to its minimum value and less than or equal to its maximum value. By setting this value to 0 this feature can be disabled. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled.", "defaultValue": "10" }, + { + "key": "spark.sql.parquet.reader.respectUnknownTypeAnnotation.enabled", + "doc": "When enabled, respects the UNKNOWN type annotation in Parquet files during schema inference and infers NullType. When disabled, ignores the UNKNOWN annotation and uses the physical type instead.", + "defaultValue": "false" + }, { "key": "spark.sql.parquet.recordLevelFilter.enabled", "doc": "If true, enables Parquet's native record-level filtering using the pushed down filters. This configuration only has an effect when 'spark.sql.parquet.filterPushdown' is enabled and the vectorized reader is not used. You can ensure the vectorized reader is not used by setting 'spark.sql.parquet.enableVectorizedReader' to false.", @@ -2514,6 +2685,16 @@ "doc": "When true, quoted Identifiers (using backticks) in SELECT statement are interpreted as regular expressions.", "defaultValue": "false" }, + { + "key": "spark.sql.parser.singleCharacterPipeOperator.enabled", + "doc": "When true, the single character pipe token '|' can be used as an alternative to '|>' for SQL pipe operators. When false, only '|>' is recognized as a pipe operator, and '|' is only used for bitwise OR operations. This provides syntax compatibility with other languages like Splunk SPL and Kusto that use '|' for pipe operations.", + "defaultValue": "true" + }, + { + "key": "spark.sql.path.enabled", + "doc": "When true, enables the SQL Standard PATH feature: SET PATH, path-based routine resolution, and CURRENT_PATH(). When false, SET PATH is rejected and resolution uses the default path only.", + "defaultValue": "false" + }, { "key": "spark.sql.pipelines.event.queue.capacity", "doc": "Capacity of the event queue used in pipelined execution. When the queue is full, non-terminal FlowProgressEvents will be dropped.", @@ -2586,6 +2767,10 @@ "doc": "When set to true, we prioritize ordinal resolution in Sort over other expressions. Otherwise, no order is enforced.", "defaultValue": "true" }, + { + "key": "spark.sql.pyspark.dataSource.profiler", + "doc": "Configure the Python Data Source profiler by enabling or disabling it with the option to choose between \"perf\" and \"memory\" types, or unsetting the config disables the profiler. This is disabled by default." + }, { "key": "spark.sql.pyspark.inferNestedDictAsStruct.enabled", "doc": "PySpark's SparkSession.createDataFrame infers the nested dict as a map by default. When it set to true, it infers the nested dict as a struct.", @@ -2611,6 +2796,11 @@ "doc": "The visual limit on plots. If set to 1000 for top-n-based plots (pie, bar, barh), the first 1000 data points will be used for plotting. For sampled-based plots (scatter, area, line), 1000 data points will be randomly sampled.", "defaultValue": "1000" }, + { + "key": "spark.sql.pyspark.toJSON.returnDataFrame", + "doc": "When true, DataFrame.toJSON in PySpark Classic returns a Dataframe instead of RDD.", + "defaultValue": "false" + }, { "key": "spark.sql.pyspark.udf.profiler", "doc": "Configure the Python/Pandas UDF profiler by enabling or disabling it with the option to choose between \"perf\" and \"memory\" types, or unsetting the config disables the profiler. This is disabled by default." @@ -2698,7 +2888,12 @@ }, { "key": "spark.sql.scripting.continueHandlerEnabled", - "doc": "EXPERIMENTAL FEATURE/WORK IN PROGRESS: SQL Scripting CONTINUE HANDLER feature is under development and still not working as intended. This feature switch is intended to be used internally for development and testing, not by end users. YOU ARE ADVISED AGAINST USING THIS FEATURE AS ITS NOT FINISHED.", + "doc": "SQL Scripting CONTINUE HANDLER feature enables CONTINUE exception handlers which are essential for cursor iteration with NOT FOUND conditions.", + "defaultValue": "false" + }, + { + "key": "spark.sql.scripting.cursorEnabled", + "doc": "SQL Scripting CURSOR feature enables declarative cursors with DECLARE CURSOR, OPEN, FETCH, and CLOSE statements.", "defaultValue": "false" }, { @@ -2739,7 +2934,7 @@ { "key": "spark.sql.session.localRelationSizeLimit", "doc": "Limit on how large ChunkedCachedLocalRelation.data can be in bytes.If the limit is exceeded, an exception is thrown.", - "defaultValue": "3GB" + "defaultValue": "3221225472" }, { "key": "spark.sql.session.timeZone", @@ -2764,12 +2959,17 @@ { "key": "spark.sql.shuffle.orderIndependentChecksum.enableFullRetryOnMismatch", "doc": "Whether to retry all tasks of a consumer stage when we detect checksum mismatches with its producer stages.", + "defaultValue": "true" + }, + { + "key": "spark.sql.shuffle.orderIndependentChecksum.enableQueryLevelRollbackOnMismatch", + "doc": "Whether to rollback all the consumer stages from the same query executor when we detect checksum mismatches with its producer stages, including cancel running shuffle map stages and resubmit, clean up all the shuffle data written for available shuffle map stages and abort the running result stages.", "defaultValue": "false" }, { "key": "spark.sql.shuffle.orderIndependentChecksum.enabled", "doc": "Whether to calculate order independent checksum for the shuffle data or not. If enabled, Spark will calculate a checksum that is independent of the input row order for each mapper and returns the checksums from executors to driver. This is different from the checksum computed when spark.shuffle.checksum.enabled is enabled which is sensitive to shuffle data ordering to detect file corruption. While this checksum will be the same even if the shuffle row order changes and it is used to detect whether different task attempts of the same partition produce different output data or not (same set of keyValue pairs). In case the output data has changed across retries, Spark will need to retry all tasks of the consumer stages to avoid correctness issues.", - "defaultValue": "false" + "defaultValue": "true" }, { "key": "spark.sql.shuffle.partitions", @@ -2928,7 +3128,7 @@ }, { "key": "spark.sql.sources.v2.bucketing.partiallyClusteredDistribution.enabled", - "doc": "During a storage-partitioned join, whether to allow input partitions to be partially clustered, when both sides of the join are of KeyGroupedPartitioning. At planning time, Spark will pick the side with less data size based on table statistics, group and replicate them to match the other side. This is an optimization on skew join and can help to reduce data skewness when certain partitions are assigned large amount of data. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled", + "doc": "During a storage-partitioned join, whether to allow input partitions to be partially clustered, when both sides of the join are of KeyedPartitioning. At planning time, Spark will pick the side with less data size based on table statistics, group and replicate them to match the other side. This is an optimization on skew join and can help to reduce data skewness when certain partitions are assigned large amount of data. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled", "defaultValue": "false" }, { @@ -2936,14 +3136,29 @@ "doc": "Whether to filter partitions when running storage-partition join. When enabled, partitions without matches on the other side can be omitted for scanning, if allowed by the join type. This config requires both spark.sql.sources.v2.bucketing.enabled and spark.sql.sources.v2.bucketing.pushPartValues.enabled to be enabled.", "defaultValue": "false" }, + { + "key": "spark.sql.sources.v2.bucketing.partitionKeyOrdering.enabled", + "doc": "When enabled, Spark derives output ordering from the partition key expressions of a V2 data source that reports a KeyedPartitioning but does not report explicit ordering via SupportsReportOrdering. Within a single partition all rows share the same key value, so the data is trivially sorted by those expressions. Requires spark.sql.sources.v2.bucketing.enabled to be enabled.", + "defaultValue": "false" + }, + { + "key": "spark.sql.sources.v2.bucketing.preserveKeyOrderingOnCoalesce.enabled", + "doc": "When enabled, Spark preserves sort orders over partition key expressions when GroupPartitionsExec coalesces multiple input partitions into one output partition. Because all merged partitions share the same partition key value, sort orders over those key expressions remain valid after the merge. This applies to both key-derived ordering (from SupportsReportOrdering) and ordering derived from spark.sql.sources.v2.bucketing.partitionKeyOrdering.enabled. Requires spark.sql.sources.v2.bucketing.enabled to be enabled.", + "defaultValue": "false" + }, + { + "key": "spark.sql.sources.v2.bucketing.preserveOrderingOnCoalesce.enabled", + "doc": "When turned on, GroupPartitionsExec will use sorted merge to preserve full ordering (as opposed to the key-derived ordering preserved by spark.sql.sources.v2.bucketing.preserveKeyOrderingOnCoalesce.enabled) when coalescing multiple partitions with the same key. This allows eliminating downstream sorts when data is both partitioned and sorted. When this config is enabled, the effect of spark.sql.sources.v2.bucketing.preserveKeyOrderingOnCoalesce.enabled is fully subsumed: full ordering implies key-derived ordering. However, sorted merge uses more resources (priority queue, comparison overhead) than simple concatenation, especially when coalescing many partitions. When turned off, only key-derived ordering is preserved during coalescing. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled.", + "defaultValue": "false" + }, { "key": "spark.sql.sources.v2.bucketing.pushPartValues.enabled", - "doc": "Whether to pushdown common partition values when spark.sql.sources.v2.bucketing.enabled is enabled. When turned on, if both sides of a join are of KeyGroupedPartitioning and if they share compatible partition keys, even if they don't have the exact same partition values, Spark will calculate a superset of partition values and pushdown that info to scan nodes, which will use empty partitions for the missing partition values on either side. This could help to eliminate unnecessary shuffles", + "doc": "Whether to pushdown common partition values when spark.sql.sources.v2.bucketing.enabled is enabled. When turned on, if both sides of a join are of KeyedPartitioning and if they share compatible partition keys, even if they don't have the exact same partition values, Spark will calculate a superset of partition values and pushdown that info to group partition nodes, which will use empty partitions for the missing partition values on either side. This could help to eliminate unnecessary shuffles", "defaultValue": "true" }, { "key": "spark.sql.sources.v2.bucketing.shuffle.enabled", - "doc": "During a storage-partitioned join, whether to allow to shuffle only one side. When only one side is KeyGroupedPartitioning, if the conditions are met, spark will only shuffle the other side. This optimization will reduce the amount of data that needs to be shuffle. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled", + "doc": "During a storage-partitioned join, whether to allow to shuffle only one side. When only one side is KeyedPartitioning, if the conditions are met, spark will only shuffle the other side. This optimization will reduce the amount of data that needs to be shuffle. This config requires spark.sql.sources.v2.bucketing.enabled to be enabled", "defaultValue": "false" }, { @@ -3021,6 +3236,11 @@ "doc": "When true, purging the offset log and commit log of old entries will be done asynchronously.", "defaultValue": "true" }, + { + "key": "spark.sql.streaming.checkUnfinishedRepartitionOnRestart", + "doc": "When true, checks if the latest batch is for an unfinished state repartitioning operation, on query restart. If so, it will fail the query and require the repartitioning to be complete before restarting the query.", + "defaultValue": "true" + }, { "key": "spark.sql.streaming.checkpoint.escapedPathCheck.enabled", "doc": "Whether to detect a streaming query may pick up an incorrect checkpoint path due to SPARK-26824.", @@ -3028,7 +3248,7 @@ }, { "key": "spark.sql.streaming.checkpoint.fileChecksum.enabled", - "doc": "When true, checksum would be generated and verified for checkpoint files. This is used to detect file corruption.", + "doc": "When true, checksum would be generated and verified for checkpoint files. This is used to detect file corruption. This is only enabled when STATE_STORE_CHECKPOINT_FORMAT_VERSION >= 2", "defaultValue": "true" }, { @@ -3041,6 +3261,11 @@ "doc": "When true, Spark will validate if renamed checkpoint file exists.", "defaultValue": "false" }, + { + "key": "spark.sql.streaming.checkpoint.verifyMetadataExists.enabled", + "doc": "When true, validates that the checkpoint metadata file exists when offset or commit logs contain data. This prevents generating a new query ID when checkpoint data already exists, which would cause data duplication in exactly-once sinks.", + "defaultValue": "true" + }, { "key": "spark.sql.streaming.checkpointLocation", "doc": "The default location for storing checkpoint data for streaming queries." @@ -3139,9 +3364,14 @@ "key": "spark.sql.streaming.internal.stateStore.partitions", "doc": "WARN: This config is used internally and is not intended to be user-facing. This config can be removed without support of compatibility in any time. DO NOT USE THIS CONFIG DIRECTLY AND USE THE CONFIG `spark.sql.shuffle.partitions`. The default number of partitions to use when shuffling data for stateful operations. If not specified, this config picks up the value of `spark.sql.shuffle.partitions`. Note: For structured streaming, this configuration cannot be changed between query restarts from the same checkpoint location." }, + { + "key": "spark.sql.streaming.join.stateFormatV4.enabled", + "doc": "When true, enables state format version 4 for stream-stream joins. This config will be removed once V4 is complete.", + "defaultValue": "false" + }, { "key": "spark.sql.streaming.join.stateFormatVersion", - "doc": "State format version used by streaming join operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running. Version 3 uses a single state store with virtual column families instead of four stores and is only supported with RocksDB.", + "doc": "State format version used by streaming join operations in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running. Version 3 uses a single state store with virtual column families instead of four stores and is only supported with RocksDB. NOTE: version 1 is DEPRECATED and should not be explicitly set by users. Version 4 is under development and only available for testing.", "defaultValue": "2" }, { @@ -3189,6 +3419,11 @@ "doc": "The number of progress updates to retain for a streaming query", "defaultValue": "100" }, + { + "key": "spark.sql.streaming.offsetLog.formatVersion", + "doc": "The version of the offset log format. The default value is 1.", + "defaultValue": "1" + }, { "key": "spark.sql.streaming.optimizeOneRowPlan.enabled", "doc": "When true, enable OptimizeOneRowPlan rule for the case where the child is a streaming Dataset. This is a fallback flag to revert the 'incorrect' behavior, hence this configuration must not be used without understanding in depth. Use this only to quickly recover failure in existing query!", @@ -3199,6 +3434,16 @@ "doc": "How long to delay polling new data when no data is available", "defaultValue": "10ms" }, + { + "key": "spark.sql.streaming.queryEvolution.enableSinkEvolution", + "doc": "When true, streaming sinks can be named using the name() API on DataStreamWriter. This enables sink evolution capability where sinks can be changed while maintaining a historical record of all sinks used in the checkpoint.", + "defaultValue": "false" + }, + { + "key": "spark.sql.streaming.queryEvolution.enableSourceEvolution", + "doc": "When true, enforces that all streaming sources must be explicitly named via the .name() API. This enables streaming source evolution, allowing sources to be added, removed, or reordered without losing state.", + "defaultValue": "false" + }, { "key": "spark.sql.streaming.ratioExtraSpaceAllowedInCheckpoint", "doc": "The ratio of extra space allowed for batch deletion of files when maintenance isinvoked. When value > 0, it optimizes the cost of discovering and deleting old checkpoint versions. The minimum number of stale versions we retain in checkpoint location for batch deletion is calculated by minBatchesToRetain * ratioExtraSpaceAllowedInCheckpoint.", @@ -3229,6 +3474,21 @@ "doc": "State format version used by streaming session window in a streaming query. State between versions are tend to be incompatible, so state format version shouldn't be modified after running.", "defaultValue": "1" }, + { + "key": "spark.sql.streaming.stateStore.autoSnapshotRepair.enabled", + "doc": "When true, enables automatic repair of state store snapshot, when a bad snapshot is detected while loading the state store, to prevent the query from failing. Typically, queries will fail when they are unable to load a snapshot, but this helps recover by skipping the bad snapshot and uses the change files.NOTE: For RocksDB state store, changelog checkpointing must be enabled", + "defaultValue": "true" + }, + { + "key": "spark.sql.streaming.stateStore.autoSnapshotRepair.maxChangeFileReplay", + "doc": "When autoSnapshotRepair is enabled, this specifies the maximum number of change files allowed to be replayed to rebuild state due to bad snapshots.", + "defaultValue": "500" + }, + { + "key": "spark.sql.streaming.stateStore.autoSnapshotRepair.numFailuresBeforeActivating", + "doc": "When autoSnapshotRepair is enabled, it will wait for the specified number of snapshot load failures, before it attempts to repair.", + "defaultValue": "1" + }, { "key": "spark.sql.streaming.stateStore.checkpointFormatVersion", "doc": "The version of the approach of doing state store checkpoint", @@ -3254,11 +3514,26 @@ "doc": "The encoding format used for stateful operators to store information in the state store", "defaultValue": "unsaferow" }, + { + "key": "spark.sql.streaming.stateStore.fileChecksumThreadPoolSize", + "doc": "Number of threads used to read/write files and their corresponding checksum files concurrently. Set to 0 to disable the thread pool and run operations sequentially on the calling thread. WARNING: Reducing below the default value of 4 may have performance impact.", + "defaultValue": "4" + }, + { + "key": "spark.sql.streaming.stateStore.forceSnapshotUploadOnLag", + "doc": "When enabled, state stores with lagging snapshot uploads will automatically trigger a snapshot on the next commit. Requires spark.sql.streaming.stateStore.coordinatorReportSnapshotUploadLag to be true.", + "defaultValue": "true" + }, { "key": "spark.sql.streaming.stateStore.formatValidation.enabled", "doc": "When true, check if the data from state store is valid or not when running streaming queries. This can happen if the state store format has been changed. Note, the feature is only effective in the build-in HDFS state store provider now.", "defaultValue": "true" }, + { + "key": "spark.sql.streaming.stateStore.maintenanceForceShutdownTimeout", + "doc": "Timeout in seconds to wait for tasks to respond to cancellation after force shutdown is initiated. This applies after the graceful shutdown timeout has been exceeded.", + "defaultValue": "60000ms" + }, { "key": "spark.sql.streaming.stateStore.maintenanceInterval", "doc": "The interval in milliseconds between triggering maintenance tasks in StateStore. The maintenance task executes background maintenance task in all the loaded store providers if they are still the active instances according to the coordinator. If not, inactive instances of store providers will be closed.", @@ -3324,6 +3599,21 @@ "doc": "Set the RocksDB format version. This will be stored in the checkpoint when starting a streaming query. The checkpoint will use this RocksDB format version in the entire lifetime of the query.", "defaultValue": "5" }, + { + "key": "spark.sql.streaming.stateStore.rocksdb.mergeOperatorVersion", + "doc": "Set the RocksDB merge operator version. This will be stored in the checkpoint when starting a streaming query. The checkpoint will use this merge operator version for the entire lifetime of the query.", + "defaultValue": "2" + }, + { + "key": "spark.sql.streaming.stateStore.rowChecksum.enabled", + "doc": "When true, checksum would be generated and verified for each state store row. This is used to detect row level corruption. Note: This configuration cannot be changed between query restarts from the same checkpoint location.", + "defaultValue": "false" + }, + { + "key": "spark.sql.streaming.stateStore.rowChecksum.readVerificationRatio", + "doc": "When specified, Spark will do row checksum verification for every specified number of rows read from state store. The check is to ensure the row read from state store is not corrupt. Default is 0, which means no verification during read but we will still do verification when loading from checkpoint location.Example, if you set to 1, it will do the check for every row read from the state store.If set to 10, it will do the check for every 10th row read from the state store.", + "defaultValue": "0" + }, { "key": "spark.sql.streaming.stateStore.skipNullsForStreamStreamJoins.enabled", "doc": "When true, this config will skip null values in hash based stream-stream joins. The number of skipped null values will be shown as custom metric of stream join operator. If the streaming query was started with Spark 3.5 or above, please exercise caution before enabling this config since it may hide potential data loss/corruption issues.", @@ -3418,6 +3708,11 @@ "doc": "When true, the logical plan for streaming query will be checked for unsupported operations.", "defaultValue": "true" }, + { + "key": "spark.sql.streaming.validateEventTimeWatermarkColumn", + "doc": "When true, check that eventTime in withWatermark is a top-level column.", + "defaultValue": "true" + }, { "key": "spark.sql.streaming.verifyCheckpointDirectoryEmptyOnStart", "doc": "When true, verifies that the checkpoint directory (offsets, state, commits) is empty when first starting a streaming query. This prevents prevents sharing checkpoint directories between different queries.", @@ -3433,6 +3728,11 @@ "doc": "When true, common subexpressions will be eliminated.", "defaultValue": "true" }, + { + "key": "spark.sql.subexpressionElimination.filterExec.enabled", + "doc": "When true (and subexpression elimination is enabled), FilterExec whole-stage codegen eliminates common subexpressions shared across its predicates. When false, FilterExec falls back to the predicate codegen that loads input columns lazily and short-circuits, avoiding eager materialization of all predicate-referenced columns on every row. Only affects FilterExec; subexpression elimination elsewhere is unaffected.", + "defaultValue": "true" + }, { "key": "spark.sql.subexpressionElimination.skipForShortcutExpr", "doc": "When true, shortcut eliminate subexpression with `AND`, `OR`. The subexpression may not need to eval even if it appears more than once. e.g., `if(or(a, and(b, b)))`, the expression `b` would be skipped if `a` is true.", @@ -3463,6 +3763,11 @@ "key": "spark.sql.thriftserver.scheduler.pool", "doc": "Set a Fair Scheduler pool for a JDBC client session." }, + { + "key": "spark.sql.thriftserver.shuffleDependency.fileCleanup.enabled", + "doc": "When enabled, shuffle files will be cleaned up at the end of Thrift server SQL executions.", + "defaultValue": "false" + }, { "key": "spark.sql.thriftserver.ui.retainedSessions", "doc": "The number of SQL client sessions kept in the JDBC/ODBC web UI history.", @@ -3508,6 +3813,21 @@ "doc": "When true, allows multiple table arguments for table-valued functions, receiving the cartesian product of all the rows of these tables.", "defaultValue": "false" }, + { + "key": "spark.sql.types.framework.enabled", + "doc": "When true, use the Types Framework for supported types (currently TimeType). The framework centralizes type-specific operations in Ops classes instead of scattered pattern matching. When false, use legacy scattered implementation.", + "defaultValue": "false" + }, + { + "key": "spark.sql.udt.allowCreatingUDTFromString", + "doc": "When true, Spark loads and instantiates the UserDefinedType class named in a schema string (for example the schema stored in Parquet/ORC file metadata) while inferring or parsing a schema. Because the class name is taken from the data being read, a crafted file can make Spark load an arbitrary class from the classpath. Set this to false to block loading UDT classes by name, optionally allowing specific classes via 'spark.sql.udt.allowedDynamicUDTClasses'.", + "defaultValue": "true" + }, + { + "key": "spark.sql.udt.allowedDynamicUDTClasses", + "doc": "When 'spark.sql.udt.allowCreatingUDTFromString' is false, UserDefinedType classes listed here (by fully qualified class name) may still be loaded and instantiated from a schema string. Has no effect when UDT loading is enabled.", + "defaultValue": "" + }, { "key": "spark.sql.ui.explainMode", "doc": "Configures the query explain mode used in the Spark SQL UI. The value can be 'simple', 'extended', 'codegen', 'cost', or 'formatted'. The default value is 'formatted'.", @@ -3569,6 +3889,11 @@ "doc": "Maximum number of shredded fields to create when inferring a schema for Variant", "defaultValue": "300" }, + { + "key": "spark.sql.variant.validateUnicodeInJsonParsing", + "doc": "When true, parsing variant from JSON rejects strings that contain unpaired UTF-16 surrogate code units (such as a lone high surrogate like \\uD835), which are invalid per RFC 8259 section 7. When false, restores the legacy permissive behavior in which the unpaired surrogate is silently replaced by the Unicode replacement character during UTF-8 encoding, causing data corruption that diverges from strict JSON parsers.", + "defaultValue": "true" + }, { "key": "spark.sql.variant.writeShredding.enabled", "doc": "When true, the Parquet writer is allowed to write shredded variant. ", @@ -3585,6 +3910,26 @@ "defaultValue": null, "isStatic": true }, + { + "key": "spark.sql.window.segmentTree.blockSize", + "doc": "Block size, in rows, for the block-chunked segment tree used by moving window frames. Each leaf of the tree aggregates this many consecutive rows. Smaller values reduce per-partition memory and speed up tree build for small partitions, but make the tree deeper and increase query cost for wide frames. Larger values amortize build cost and shrink the tree but increase the per-block prefix/suffix scan cost within a block. The default is tuned for partitions on the order of tens of thousands to millions of rows.", + "defaultValue": "65536" + }, + { + "key": "spark.sql.window.segmentTree.enabled", + "doc": "Use block-chunked segment tree for moving aggregate window frames whose functions are all DeclarativeAggregate without FILTER/DISTINCT.", + "defaultValue": "false" + }, + { + "key": "spark.sql.window.segmentTree.fanout", + "doc": "Fanout of internal nodes for the block-chunked segment tree.", + "defaultValue": "16" + }, + { + "key": "spark.sql.window.segmentTree.minPartitionRows", + "doc": "Minimum partition row count to activate the segment-tree moving frame. Partitions smaller than this fall back to the default sliding implementation.", + "defaultValue": "64" + }, { "key": "spark.sql.windowExec.buffer.in.memory.threshold", "doc": "Threshold for number of rows guaranteed to be held in memory by the window operator", @@ -3599,6 +3944,11 @@ "key": "spark.sql.windowExec.buffer.spill.threshold", "doc": "Threshold for number of rows to be spilled by window operator", "defaultValue": "2147483647" + }, + { + "key": "spark.sql.xml.variant.respectInferSchema", + "doc": "Kill switch for the SPARK-56554 fix. When true (default), the XML to Variant parser honors the 'inferSchema' option: if 'inferSchema' is false, primitive leaf values (text and attributes) are preserved as strings inside the Variant instead of being inferred as boolean, long, or decimal. Set this conf to false to restore the pre-SPARK-56554 behavior of always inferring types regardless of the 'inferSchema' option.", + "defaultValue": "true" } ] } diff --git a/crates/sail-spark-connect/proto/spark/connect/base.proto b/crates/sail-spark-connect/proto/spark/connect/base.proto index a97d2d25f4..c7247129f1 100644 --- a/crates/sail-spark-connect/proto/spark/connect/base.proto +++ b/crates/sail-spark-connect/proto/spark/connect/base.proto @@ -491,6 +491,12 @@ message ExecutePlanResponse { repeated Expression.Literal values = 2; repeated string keys = 3; int64 plan_id = 4; + // (Optional) The index of the root error in errors. + // The field will not be set if there are no errors. + optional int32 root_error_idx = 5; + // A list of errors that occurred while collecting the observed metrics. + // If the length is 0, it means no errors occurred. + repeated FetchErrorDetailsResponse.Error errors = 6; } message ResultComplete { @@ -1213,11 +1219,94 @@ message CloneSessionResponse { // Session id of the new cloned session. string new_session_id = 3; - + // Server-side session ID of the new cloned session. string new_server_side_session_id = 4; } +// Next ID: 6 +message GetStatusRequest { + // (Required) + // + // The session_id specifies a Spark session for a user identified by user_context.user_id. + // The id should be an UUID string of the format `00112233-4455-6677-8899-aabbccddeeff` + string session_id = 1; + + // (Required) + // + // user_context.user_id and session_id both identify a unique remote spark session on the + // server side. + UserContext user_context = 2; + + // (Optional) + // + // Provides optional information about the client sending the request. This field + // can be used for language or version specific information and is only intended for + // logging purposes and will not be interpreted by the server. + optional string client_type = 3; + + // (Optional) + // + // Server-side generated idempotency key from the previous responses (if any). Server + // can use this to validate that the server side session has not changed. + optional string client_observed_server_side_session_id = 4; + + // (Optional) + // + // Get status of operations in the session. + optional OperationStatusRequest operation_status = 5; + + // Extension point for custom status request types. + repeated google.protobuf.Any extensions = 999; + + message OperationStatusRequest { + // Get status of operations with these operation_ids. + // If unset or empty, returns status of all operations in the session. + repeated string operation_ids = 1; + + // Extension point for custom operation-level status requests. + repeated google.protobuf.Any extensions = 999; + } +} + +// Next ID: 4 +message GetStatusResponse { + // Session id of the session for which the status was requested. + string session_id = 1; + + // Server-side generated idempotency key that the client can use to assert that the server side + // session has not changed. + string server_side_session_id = 2; + + // Status information about requested operations. + repeated OperationStatus operation_statuses = 3; + + // Extension point for custom status response types. + repeated google.protobuf.Any extensions = 999; + + // Status information for a single operation. + message OperationStatus { + // The operation_id of the operation. + string operation_id = 1; + + // The current status of the operation. + OperationState state = 2; + + // Extension point for custom operation-level status fields. + repeated google.protobuf.Any extensions = 999; + + enum OperationState { + OPERATION_STATE_UNSPECIFIED = 0; + OPERATION_STATE_UNKNOWN = 1; + OPERATION_STATE_RUNNING = 2; + OPERATION_STATE_TERMINATING = 3; + OPERATION_STATE_SUCCEEDED = 4; + OPERATION_STATE_FAILED = 5; + OPERATION_STATE_CANCELLED = 6; + } + } +} + // Main interface for the SparkConnect service. service SparkConnectService { @@ -1272,4 +1361,7 @@ service SparkConnectService { // The request can optionally specify a custom session ID for the cloned session (must be // a valid UUID). If not provided, a new UUID will be generated automatically. rpc CloneSession(CloneSessionRequest) returns (CloneSessionResponse) {} + + // Get status information of different types. + rpc GetStatus(GetStatusRequest) returns (GetStatusResponse) {} } diff --git a/crates/sail-spark-connect/proto/spark/connect/catalog.proto b/crates/sail-spark-connect/proto/spark/connect/catalog.proto index 5b1b90b008..b5341d16eb 100644 --- a/crates/sail-spark-connect/proto/spark/connect/catalog.proto +++ b/crates/sail-spark-connect/proto/spark/connect/catalog.proto @@ -55,6 +55,16 @@ message Catalog { CurrentCatalog current_catalog = 24; SetCurrentCatalog set_current_catalog = 25; ListCatalogs list_catalogs = 26; + DropTable drop_table = 27; + DropView drop_view = 28; + CreateDatabase create_database = 29; + DropDatabase drop_database = 30; + ListPartitions list_partitions = 31; + ListViews list_views = 32; + GetTableProperties get_table_properties = 33; + GetCreateTableString get_create_table_string = 34; + TruncateTable truncate_table = 35; + AnalyzeTable analyze_table = 36; } } @@ -241,3 +251,74 @@ message ListCatalogs { // (Optional) The pattern that the catalog name needs to match optional string pattern = 1; } + +// See `spark.catalog.dropTable` +message DropTable { + // (Required) + string table_name = 1; + bool if_exists = 2; + bool purge = 3; +} + +// See `spark.catalog.dropView` +message DropView { + // (Required) + string view_name = 1; + bool if_exists = 2; +} + +// See `spark.catalog.createDatabase` +message CreateDatabase { + // (Required) + string db_name = 1; + bool if_not_exists = 2; + map properties = 3; +} + +// See `spark.catalog.dropDatabase` +message DropDatabase { + // (Required) + string db_name = 1; + bool if_exists = 2; + bool cascade = 3; +} + +// See `spark.catalog.listPartitions` +message ListPartitions { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.listViews` +message ListViews { + // (Optional) + optional string db_name = 1; + // (Optional) The pattern that the view name needs to match + optional string pattern = 2; +} + +// See `spark.catalog.getTableProperties` +message GetTableProperties { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.getCreateTableString` +message GetCreateTableString { + // (Required) + string table_name = 1; + bool as_serde = 2; +} + +// See `spark.catalog.truncateTable` +message TruncateTable { + // (Required) + string table_name = 1; +} + +// See `spark.catalog.analyzeTable` +message AnalyzeTable { + // (Required) + string table_name = 1; + bool no_scan = 2; +} diff --git a/crates/sail-spark-connect/proto/spark/connect/commands.proto b/crates/sail-spark-connect/proto/spark/connect/commands.proto index 861af17e26..dcf5aff236 100644 --- a/crates/sail-spark-connect/proto/spark/connect/commands.proto +++ b/crates/sail-spark-connect/proto/spark/connect/commands.proto @@ -144,6 +144,9 @@ message WriteOperation { // (Optional) Columns used for clustering the table. repeated string clustering_columns = 10; + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 11; + message SaveTable { // (Required) The table name. string table_name = 1; @@ -211,6 +214,9 @@ message WriteOperationV2 { // (Optional) Columns used for clustering the table. repeated string clustering_columns = 9; + + // (Optional) Whether schema evolution is enabled for the write. + bool with_schema_evolution = 10; } // Starts write stream operation as streaming query. Query ID and Run ID of the streaming @@ -232,6 +238,7 @@ message WriteStreamOperationStart { bool available_now = 6; bool once = 7; string continuous_checkpoint_interval = 8; + string real_time_batch_duration = 100; } string output_mode = 9; diff --git a/crates/sail-spark-connect/proto/spark/connect/pipelines.proto b/crates/sail-spark-connect/proto/spark/connect/pipelines.proto index 0874c2d10e..7632ec95b9 100644 --- a/crates/sail-spark-connect/proto/spark/connect/pipelines.proto +++ b/crates/sail-spark-connect/proto/spark/connect/pipelines.proto @@ -22,6 +22,7 @@ package spark.connect; import "google/protobuf/any.proto"; import "google/protobuf/timestamp.proto"; import "spark/connect/common.proto"; +import "spark/connect/expressions.proto"; import "spark/connect/relations.proto"; import "spark/connect/types.proto"; @@ -40,6 +41,8 @@ message PipelineCommand { DefineSqlGraphElements define_sql_graph_elements = 6; GetQueryFunctionExecutionSignalStream get_query_function_execution_signal_stream = 7; DefineFlowQueryFunctionResult define_flow_query_function_result = 8; + ExecuteOutputFlows execute_output_flows = 9; + // Reserved field for protocol extensions. // Used to support forward-compatibility by carrying additional command types // that are not yet defined in this version of the proto. During planning, the @@ -142,6 +145,7 @@ message PipelineCommand { oneof details { WriteRelationFlowDetails relation_flow_details = 7; + AutoCdcFlowDetails auto_cdc_flow_details = 10; google.protobuf.Any extension = 999; } @@ -152,6 +156,46 @@ message PipelineCommand { optional spark.connect.Relation relation = 1; } + // Details for Auto CDC flows. + message AutoCdcFlowDetails { + // The name of the CDC source to stream from. + optional string source = 1; + + // Column(s) that uniquely identify a row in source and target data. + repeated Expression keys = 2; + + // Expression to order the source data. + optional Expression sequence_by = 3; + + // Delete condition for the merged operation. + optional Expression apply_as_deletes = 6; + + // Truncate condition for the merged operation. + optional Expression apply_as_truncates = 7; + + // Columns included in the output table. + repeated Expression column_list = 8; + + // Columns excluded from the output table. + repeated Expression except_column_list = 9; + + // SCD Type for target table. + SCDType stored_as_scd_type = 10; + + // Subset of columns to ignore null in updates. + repeated Expression ignore_null_updates_column_list = 14; + + // Subset of columns excluded from ignoring null in updates. + repeated Expression ignore_null_updates_except_column_list = 15; + + } + + // SCD Type for Auto CDC target tables. + enum SCDType { + SCD_TYPE_UNSPECIFIED = 0; + SCD_TYPE_1 = 1; + } + // If true, define the flow as a one-time flow, such as for backfill. // Set to true changes the flow in two ways: // - The flow is run one time by default. If the pipeline is ran with a full refresh, @@ -165,6 +209,25 @@ message PipelineCommand { } } + // Request to execute all flows for a single output (dataset or sink) remotely. + message ExecuteOutputFlows { + + // The output (table or materialized view or sink) definition. + optional DefineOutput define_output = 1; + + // The flows to execute for this table. + repeated DefineFlow define_flows = 2; + + // Whether to perform a full refresh instead of an incremental update. + optional bool full_refresh = 3; + + // Storage location for pipeline checkpoints and metadata. + optional string storage = 4; + + // Reserved field for protocol extensions. + repeated google.protobuf.Any extension = 999; + } + // Resolves all datasets and flows and start a pipeline update. Should be called after all // graph elements are registered. message StartRun { @@ -213,8 +276,13 @@ message PipelineCommand { // Request from the client to update the flow function evaluation result // for a previously un-analyzed flow. message DefineFlowQueryFunctionResult { - // The fully qualified name of the flow being updated. - optional string flow_name = 1; + // (Deprecated) The fully qualified name of the flow being updated. + // + // This field is deprecated since Spark 4.2+. Use flow_identifier field instead. + optional string flow_name = 1 [deprecated = true]; + + // The fully qualified identifier of the flow being updated. + optional ResolvedIdentifier flow_identifier = 4; // The ID of the graph this flow belongs to. optional string dataflow_graph_id = 2; @@ -290,7 +358,13 @@ message SourceCodeLocation { // A signal from the server to the client to execute the query function for one or more flows, and // to register their results with the server. message PipelineQueryFunctionExecutionSignal { - repeated string flow_names = 1; + // (Deprecated) The name of flows that are ready to be re-evaluated. + // + // This field is deprecated since Spark 4.2+. Use flow_identifiers field instead. + repeated string flow_names = 1 [deprecated = true]; + + // The identifier of flows that are ready to be re-evaluated + repeated ResolvedIdentifier flow_identifiers = 2; } // Metadata providing context about the pipeline during Spark Connect query analysis. @@ -299,8 +373,12 @@ message PipelineAnalysisContext { optional string dataflow_graph_id = 1; // The path of the top-level pipeline file determined at runtime during pipeline initialization. optional string definition_path = 2; - // The name of the Flow involved in this analysis - optional string flow_name = 3; + // (Deprecated) The name of the Flow involved in this analysis + // + // This field is deprecated since Spark 4.2+. Use flow_identifier field instead. + optional string flow_name = 3 [deprecated = true]; + // The identifier of the Flow involved in this analysis + optional ResolvedIdentifier flow_identifier = 4; // Reserved field for protocol extensions. repeated google.protobuf.Any extension = 999; diff --git a/crates/sail-spark-connect/proto/spark/connect/relations.proto b/crates/sail-spark-connect/proto/spark/connect/relations.proto index 1583785e69..95cc9281d8 100644 --- a/crates/sail-spark-connect/proto/spark/connect/relations.proto +++ b/crates/sail-spark-connect/proto/spark/connect/relations.proto @@ -81,6 +81,8 @@ message Relation { UnresolvedTableValuedFunction unresolved_table_valued_function = 43; LateralJoin lateral_join = 44; ChunkedCachedLocalRelation chunked_cached_local_relation = 45; + RelationChanges relation_changes = 46; + NearestByJoin nearest_by_join = 47; // NA functions NAFill fill_na = 90; @@ -256,9 +258,33 @@ message Read { // // This is only supported by the JDBC data source. repeated string predicates = 5; + + // (Optional) A user-provided name for the streaming source. + // This name is used in checkpoint metadata and enables stable checkpoint locations + // for source evolution. + optional string source_name = 6; } } +// Reads Change Data Capture (CDC) changes for a named table. +// +// This corresponds to the `DataFrameReader.changes()` or `DataStreamReader.changes()` API. +// CDC-specific options (startingVersion, endingVersion, startingTimestamp, endingTimestamp, +// deduplicationMode, computeUpdates, etc.) are passed in the options map. +message RelationChanges { + // (Required) Unparsed identifier for the table. + string unparsed_identifier = 1; + + // Options for the CDC query. The map key is case insensitive. + // Supported keys include: startingVersion, endingVersion, startingTimestamp, + // endingTimestamp, deduplicationMode, computeUpdates, startingBoundInclusive, + // endingBoundInclusive. + map options = 2; + + // (Optional) Indicates if this is a streaming CDC read. + bool is_streaming = 3; +} + // Projection of a bag of expressions for a given input relation. // // The input relation must be specified. @@ -1184,12 +1210,13 @@ message Parse { // (Optional) DataType representing the schema. If not set, Spark will infer the schema. optional DataType schema = 3; - // Options for the csv/json parser. The map key is case insensitive. + // Options for the csv/json/xml parser. The map key is case insensitive. map options = 4; enum ParseFormat { PARSE_FORMAT_UNSPECIFIED = 0; PARSE_FORMAT_CSV = 1; PARSE_FORMAT_JSON = 2; + PARSE_FORMAT_XML = 3; } } @@ -1250,3 +1277,33 @@ message LateralJoin { // (Required) The join type. Join.JoinType join_type = 4; } + +// Relation of type [[NearestByJoin]]. +// +// For each row on the left side, returns up to `num_results` rows from the right side ranked +// by `ranking_expression`. +message NearestByJoin { + // (Required) Left (query) input relation. + Relation left = 1; + + // (Required) Right (base) input relation. + Relation right = 2; + + // (Required) Scalar expression used to rank candidate rows on the right side. + Expression ranking_expression = 3; + + // (Required) Maximum number of matches per left row. Must be between 1 and 100000. + int32 num_results = 4; + + // The following three fields use `string` (not typed enums) for parity with `AsOfJoin`, + // which models analogous fields the same way. Validation happens server-side at planning time. + + // (Required) The join type. Must be one of: "inner", "leftouter". + string join_type = 5; + + // (Required) Search algorithm contract. Must be one of: "approx", "exact". + string mode = 6; + + // (Required) Ranking direction. Must be one of: "distance", "similarity". + string direction = 7; +} diff --git a/crates/sail-spark-connect/src/config.rs b/crates/sail-spark-connect/src/config.rs index 71d1bc2b4d..056daa69f7 100644 --- a/crates/sail-spark-connect/src/config.rs +++ b/crates/sail-spark-connect/src/config.rs @@ -11,6 +11,8 @@ use crate::spark::config::{ SPARK_SQL_EXECUTION_PANDAS_CONVERT_TO_ARROW_ARRAY_SAFELY, SPARK_SQL_EXECUTION_PYSPARK_BINARY_AS_BYTES, SPARK_SQL_EXECUTION_PYTHON_UDF_PANDAS_INT_TO_DECIMAL_COERCION_ENABLED, + SPARK_SQL_EXECUTION_PYTHON_UDF_PANDAS_PREFER_INT_EXTENSION_DTYPE, + SPARK_SQL_EXECUTION_PYTHON_UDTF_ARROW_ENABLED, SPARK_SQL_LEGACY_EXECUTION_PANDAS_GROUPED_MAP_ASSIGN_COLUMNS_BY_NAME, SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDF_PANDAS_CONVERSION_ENABLED, SPARK_SQL_LEGACY_EXECUTION_PYTHON_UDTF_PANDAS_CONVERSION_ENABLED, SPARK_SQL_PIVOT_MAX_VALUES, @@ -194,6 +196,11 @@ fn versioned_default_value(key: &str) -> Option<&'static str> { SPARK_SQL_ANSI_ENABLED if get_pyspark_major_version().is_some_and(|x| x < 4) => { Some("false") } + SPARK_SQL_EXECUTION_PYTHON_UDTF_ARROW_ENABLED + if get_pyspark_major_minor_version().is_some_and(|x| x < (4, 2)) => + { + Some("false") + } _ => None, } } @@ -208,6 +215,16 @@ fn get_pyspark_major_version() -> Option { }) } +fn get_pyspark_major_minor_version() -> Option<(u64, u64)> { + static PYSPARK_MAJOR_MINOR_VERSION: OnceLock> = OnceLock::new(); + + *PYSPARK_MAJOR_MINOR_VERSION.get_or_init(|| { + let version = get_pyspark_version().ok()?; + let mut parts = version.split('.'); + Some((parts.next()?.parse().ok()?, parts.next()?.parse().ok()?)) + }) +} + pub(crate) fn get_pyspark_version() -> SparkResult { use pyo3::Python; use pyo3::prelude::PyAnyMethods; @@ -377,6 +394,14 @@ impl TryFrom<&SparkRuntimeConfig> for PySparkUdfConfig { output.python_udf_pandas_int_to_decimal_coercion_enabled = value; } + if let Some(value) = config + .get(SPARK_SQL_EXECUTION_PYTHON_UDF_PANDAS_PREFER_INT_EXTENSION_DTYPE)? + .map(|x| x.to_lowercase().parse::()) + .transpose()? + { + output.python_udf_pandas_prefer_int_extension_dtype = value; + } + if let Some(value) = config .get(SPARK_SQL_EXECUTION_PYSPARK_BINARY_AS_BYTES)? .map(|x| x.to_lowercase().parse::()) diff --git a/crates/sail-spark-connect/src/proto/plan.rs b/crates/sail-spark-connect/src/proto/plan.rs index 96e1f27700..f7d41776c8 100644 --- a/crates/sail-spark-connect/src/proto/plan.rs +++ b/crates/sail-spark-connect/src/proto/plan.rs @@ -206,7 +206,11 @@ impl TryFrom for RelationNode { options, paths, predicates, + source_name, } = x; + if source_name.is_some() { + return Err(SparkError::unsupported("streaming source name")); + } let schema = schema .and_then(|s| { if s.is_empty() { @@ -905,6 +909,7 @@ impl TryFrom for RelationNode { ParseFormat::Unspecified => spec::ParseFormat::Unspecified, ParseFormat::Csv => spec::ParseFormat::Csv, ParseFormat::Json => spec::ParseFormat::Json, + ParseFormat::Xml => return Err(SparkError::unsupported("XML parse relation")), }; let schema: Option = schema.map(|x| x.try_into()).transpose()?; Ok(RelationNode::Query(spec::QueryNode::Parse(spec::Parse { @@ -1328,6 +1333,8 @@ impl TryFrom for RelationNode { })) } RelType::Catalog(catalog) => Ok(RelationNode::Command(catalog.try_into()?)), + RelType::RelationChanges(_) => Err(SparkError::unsupported("relation changes")), + RelType::NearestByJoin(_) => Err(SparkError::unsupported("nearest-by join")), RelType::MlRelation(_) => Err(SparkError::unsupported("ML relation")), RelType::Extension(_) => Err(SparkError::unsupported("extension relation")), RelType::Unknown(_) => Err(SparkError::unsupported("unknown relation")), @@ -1605,6 +1612,18 @@ impl TryFrom for spec::CommandNode { let sc::ListCatalogs { pattern } = x; Ok(spec::CommandNode::ListCatalogs { pattern }) } + CatType::DropTable(_) => Err(SparkError::unsupported("drop table")), + CatType::DropView(_) => Err(SparkError::unsupported("drop view")), + CatType::CreateDatabase(_) => Err(SparkError::unsupported("create database")), + CatType::DropDatabase(_) => Err(SparkError::unsupported("drop database")), + CatType::ListPartitions(_) => Err(SparkError::unsupported("list partitions")), + CatType::ListViews(_) => Err(SparkError::unsupported("list views")), + CatType::GetTableProperties(_) => Err(SparkError::unsupported("get table properties")), + CatType::GetCreateTableString(_) => { + Err(SparkError::unsupported("get create table string")) + } + CatType::TruncateTable(_) => Err(SparkError::unsupported("truncate table")), + CatType::AnalyzeTable(_) => Err(SparkError::unsupported("analyze table")), } } } @@ -1648,7 +1667,11 @@ impl TryFrom for spec::Write { options, clustering_columns, save_type, + with_schema_evolution, } = write; + if with_schema_evolution { + return Err(SparkError::unsupported("write schema evolution")); + } let input = input.required("input")?.try_into()?; let mode = match SaveMode::try_from(mode).required("save mode")? { SaveMode::Unspecified => None, @@ -1747,7 +1770,11 @@ impl TryFrom for spec::WriteTo { mode, overwrite_condition, clustering_columns, + with_schema_evolution, } = write; + if with_schema_evolution { + return Err(SparkError::unsupported("write schema evolution")); + } let input = input.required("input")?.try_into()?; let table = from_ast_object_name(parse_object_name(table_name.as_str())?)?; let partitioning_columns = partitioning_columns diff --git a/crates/sail-spark-connect/src/server.rs b/crates/sail-spark-connect/src/server.rs index 020b15cdc7..6d23177934 100644 --- a/crates/sail-spark-connect/src/server.rs +++ b/crates/sail-spark-connect/src/server.rs @@ -18,9 +18,9 @@ use crate::spark::connect::{ AddArtifactsRequest, AddArtifactsResponse, AnalyzePlanRequest, AnalyzePlanResponse, ArtifactStatusesRequest, ArtifactStatusesResponse, CloneSessionRequest, CloneSessionResponse, Command, ConfigRequest, ConfigResponse, ExecutePlanRequest, FetchErrorDetailsRequest, - FetchErrorDetailsResponse, InterruptRequest, InterruptResponse, Plan, ReattachExecuteRequest, - ReleaseExecuteRequest, ReleaseExecuteResponse, ReleaseSessionRequest, ReleaseSessionResponse, - config_request, plan, + FetchErrorDetailsResponse, GetStatusRequest, GetStatusResponse, InterruptRequest, + InterruptResponse, Plan, ReattachExecuteRequest, ReleaseExecuteRequest, ReleaseExecuteResponse, + ReleaseSessionRequest, ReleaseSessionResponse, config_request, plan, }; #[derive(Debug)] @@ -484,4 +484,12 @@ impl SparkConnectService for SparkConnectServer { debug!("{request:?}"); Err(Status::unimplemented("clone session")) } + + async fn get_status( + &self, + request: Request, + ) -> Result, Status> { + debug!("{:?}", request.into_inner()); + Err(Status::unimplemented("get status")) + } } diff --git a/crates/sail-spark-connect/tests/gold_data/data_type.json b/crates/sail-spark-connect/tests/gold_data/data_type.json index 0f154a8775..d2fbc42a46 100644 --- a/crates/sail-spark-connect/tests/gold_data/data_type.json +++ b/crates/sail-spark-connect/tests/gold_data/data_type.json @@ -365,6 +365,17 @@ "failure": "error in JSON serde: expected value at line 1 column 1" } }, + { + "input": "TIMESTAMP WITH LOCAL TIME ZONE", + "output": { + "success": { + "timestamp": { + "timeUnit": "microsecond", + "timestampType": "withLocalTimeZone" + } + } + } + }, { "input": "TIMESTAMP WITHOUT TIME ZONE", "output": { @@ -794,6 +805,17 @@ "failure": "error in JSON serde: expected ident at line 1 column 2" } }, + { + "input": "timestamp with local time zone", + "output": { + "success": { + "timestamp": { + "timeUnit": "microsecond", + "timestampType": "withLocalTimeZone" + } + } + } + }, { "input": "timestamp without time zone", "output": { diff --git a/crates/sail-spark-connect/tests/gold_data/function/agg.json b/crates/sail-spark-connect/tests/gold_data/function/agg.json index 822f49cabc..c12afd17a2 100644 --- a/crates/sail-spark-connect/tests/gold_data/function/agg.json +++ b/crates/sail-spark-connect/tests/gold_data/function/agg.json @@ -1130,6 +1130,24 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT dimension_col, measure(measure_col)", + "result": [ + "FROM test_metric_view", + "GROUP BY dimension_col;", + "dim_1, 100", + "dim_2, 200" + ], + "schema": { + "type": "struct", + "fields": [] + } + }, + "output": { + "failure": "analysis error: attribute ObjectName([Identifier(\"dimension_col\")]) is missing from the schema: cannot resolve attribute" + } + }, { "input": { "query": "SELECT every(col) FROM VALUES (NULL), (true), (true) AS tab(col);", @@ -1436,6 +1454,72 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT kll_sketch_get_n_bigint(kll_merge_agg_bigint(sketch)) FROM (SELECT kll_sketch_agg_bigint(col) as sketch FROM VALUES (1), (2), (3) tab(col) UNION ALL SELECT kll_sketch_agg_bigint(col) as sketch FROM VALUES (4), (5), (6) tab(col)) t;", + "result": [ + "6" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "kll_sketch_get_n_bigint(kll_merge_agg_bigint(sketch))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_bigint" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_n_double(kll_merge_agg_double(sketch)) FROM (SELECT kll_sketch_agg_double(col) as sketch FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)) tab(col) UNION ALL SELECT kll_sketch_agg_double(col) as sketch FROM VALUES (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)), (CAST(6.0 AS DOUBLE)) tab(col)) t;", + "result": [ + "6" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "kll_sketch_get_n_double(kll_merge_agg_double(sketch))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_n_float(kll_merge_agg_float(sketch)) FROM (SELECT kll_sketch_agg_float(col) as sketch FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)) tab(col) UNION ALL SELECT kll_sketch_agg_float(col) as sketch FROM VALUES (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)), (CAST(6.0 AS FLOAT)) tab(col)) t;", + "result": [ + "6" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "kll_sketch_get_n_float(kll_merge_agg_float(sketch))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_float" + } + }, { "input": { "query": "SELECT kurtosis(col) FROM VALUES (-10), (-20), (100), (1000) AS tab(col);", @@ -1810,6 +1894,32 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT max_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);", + "result": [ + "[\"b\",\"c\"]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "max_by(x, y, 2)", + "nullable": true, + "type": { + "type": "array", + "elementType": "string", + "containsNull": true + }, + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, { "input": { "query": "SELECT mean(col) FROM VALUES (1), (2), (3) AS tab(col);", @@ -1942,6 +2052,32 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT min_by(x, y, 2) FROM VALUES ('a', 10), ('b', 50), ('c', 20) AS tab(x, y);", + "result": [ + "[\"a\",\"c\"]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "min_by(x, y, 2)", + "nullable": true, + "type": { + "type": "array", + "elementType": "string", + "containsNull": true + }, + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, { "input": { "query": "SELECT mode() WITHIN GROUP (ORDER BY col DESC) FROM VALUES (0), (10), (10), (20), (20) AS tab(col);", @@ -4240,6 +4376,138 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D), (3, 15.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D), (4, 12.0D) tab(key, summary));", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary, 12, 'sum')) FROM VALUES (1, 5.0D), (1, 1.0D), (2, 2.0D), (2, 3.0D), (3, 2.2D) tab(key, summary);", + "result": [ + "3.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_union_agg_double(sketch)) FROM (SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (1, 5.0D), (2, 10.0D) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_double(key, summary) as sketch FROM VALUES (2, 3.0D), (3, 7.0D) tab(key, summary));", + "result": [ + "3.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_union_agg_double(sketch, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 2), (3, 3), (4, 4) tab(key, summary));", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary, 12, 'sum')) FROM VALUES (1, 5), (1, 1), (2, 2), (2, 3), (3, 2) tab(key, summary);", + "result": [ + "3.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch)) FROM (SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (1, 5), (2, 10) tab(key, summary) UNION ALL SELECT tuple_sketch_agg_integer(key, summary) as sketch FROM VALUES (2, 3), (3, 7) tab(key, summary));", + "result": [ + "3.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, { "input": { "query": "SELECT var_pop(col) FROM VALUES (1), (2), (3) AS tab(col);", diff --git a/crates/sail-spark-connect/tests/gold_data/function/avro.json b/crates/sail-spark-connect/tests/gold_data/function/avro.json new file mode 100644 index 0000000000..fe899b757d --- /dev/null +++ b/crates/sail-spark-connect/tests/gold_data/function/avro.json @@ -0,0 +1,64 @@ +{ + "tests": [ + { + "input": { + "query": "SELECT from_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('u', NAMED_STRUCT('member0', member0, 'member1', member1)) AS s FROM VALUES (1, NULL), (NULL, 'a') tab(member0, member1));", + "result": [ + "[false]" + ], + "schema": { + "type": "struct", + "fields": [] + } + }, + "output": { + "failure": "not implemented: function: from_avro" + } + }, + { + "input": { + "query": "SELECT schema_of_avro('{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{\"name\": \"u\", \"type\": [\"int\", \"string\"]}]}', map());", + "result": [ + "STRUCT NOT NULL>" + ], + "schema": { + "type": "struct", + "fields": [] + } + }, + "output": { + "failure": "not implemented: function: schema_of_avro" + } + }, + { + "input": { + "query": "SELECT to_avro(s) IS NULL FROM (SELECT NULL AS s);", + "result": [ + "[true]" + ], + "schema": { + "type": "struct", + "fields": [] + } + }, + "output": { + "failure": "not implemented: function: to_avro" + } + }, + { + "input": { + "query": "SELECT to_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}') IS NULL FROM (SELECT NULL AS s);", + "result": [ + "[true]" + ], + "schema": { + "type": "struct", + "fields": [] + } + }, + "output": { + "failure": "not implemented: function: to_avro" + } + } + ] +} diff --git a/crates/sail-spark-connect/tests/gold_data/function/collection.json b/crates/sail-spark-connect/tests/gold_data/function/collection.json index 2940e2767b..547ea93d2b 100644 --- a/crates/sail-spark-connect/tests/gold_data/function/collection.json +++ b/crates/sail-spark-connect/tests/gold_data/function/collection.json @@ -136,6 +136,28 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT hex(reverse(x'CAFE'));", + "result": [ + "FECA" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "hex(reverse(X'CAFE'))", + "nullable": false, + "type": "string", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "error in DataFusion: Execution error: array_reverse does not support type 'Binary'." + } + }, { "input": { "query": "SELECT reverse('Spark SQL');", diff --git a/crates/sail-spark-connect/tests/gold_data/function/datetime.json b/crates/sail-spark-connect/tests/gold_data/function/datetime.json index 4a7e346cfa..9fc017e077 100644 --- a/crates/sail-spark-connect/tests/gold_data/function/datetime.json +++ b/crates/sail-spark-connect/tests/gold_data/function/datetime.json @@ -2573,6 +2573,72 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT time_bucket(INTERVAL '1' HOUR, TIMESTAMP '2024-01-01 11:27:00');", + "result": [ + "2024-01-01 11:00:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_bucket(INTERVAL '01' HOUR, TIMESTAMP '2024-01-01 11:27:00', TIMESTAMP '1970-01-01 00:00:00')", + "nullable": false, + "type": "timestamp", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_bucket" + } + }, + { + "input": { + "query": "SELECT time_bucket(INTERVAL '1' MONTH, TIMESTAMP '2024-07-20 14:30:00', TIMESTAMP '2024-06-15 09:00:00');", + "result": [ + "2024-07-15 09:00:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_bucket(INTERVAL '1' MONTH, TIMESTAMP '2024-07-20 14:30:00', TIMESTAMP '2024-06-15 09:00:00')", + "nullable": false, + "type": "timestamp", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_bucket" + } + }, + { + "input": { + "query": "SELECT time_bucket(INTERVAL '15' MINUTE, TIMESTAMP '2024-01-01 11:27:00', TIMESTAMP '1970-01-01 00:00:00');", + "result": [ + "2024-01-01 11:15:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_bucket(INTERVAL '15' MINUTE, TIMESTAMP '2024-01-01 11:27:00', TIMESTAMP '1970-01-01 00:00:00')", + "nullable": false, + "type": "timestamp", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_bucket" + } + }, { "input": { "query": "SELECT time_diff('HOUR', TIME'20:30:29', TIME'12:00:00');", @@ -2639,6 +2705,534 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT time_from_micros(0);", + "result": [ + "00:00:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_micros(0)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_micros" + } + }, + { + "input": { + "query": "SELECT time_from_micros(52200000000);", + "result": [ + "14:30:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_micros(52200000000)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_micros" + } + }, + { + "input": { + "query": "SELECT time_from_micros(52200500000);", + "result": [ + "14:30:00.5" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_micros(52200500000)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_micros" + } + }, + { + "input": { + "query": "SELECT time_from_micros(86399999999);", + "result": [ + "23:59:59.999999" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_micros(86399999999)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_micros" + } + }, + { + "input": { + "query": "SELECT time_from_millis(0);", + "result": [ + "00:00:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_millis(0)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_millis" + } + }, + { + "input": { + "query": "SELECT time_from_millis(52200000);", + "result": [ + "14:30:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_millis(52200000)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_millis" + } + }, + { + "input": { + "query": "SELECT time_from_millis(52200500);", + "result": [ + "14:30:00.5" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_millis(52200500)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_millis" + } + }, + { + "input": { + "query": "SELECT time_from_millis(86399999);", + "result": [ + "23:59:59.999" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_millis(86399999)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_millis" + } + }, + { + "input": { + "query": "SELECT time_from_seconds(0);", + "result": [ + "00:00:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_seconds(0)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_seconds" + } + }, + { + "input": { + "query": "SELECT time_from_seconds(52200);", + "result": [ + "14:30:00" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_seconds(52200)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_seconds" + } + }, + { + "input": { + "query": "SELECT time_from_seconds(52200.5);", + "result": [ + "14:30:00.5" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_seconds(52200.5)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_seconds" + } + }, + { + "input": { + "query": "SELECT time_from_seconds(86399.999999);", + "result": [ + "23:59:59.999999" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_from_seconds(86399.999999)", + "nullable": true, + "type": "time(6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_from_seconds" + } + }, + { + "input": { + "query": "SELECT time_to_micros(TIME'00:00:00');", + "result": [ + "0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_micros(TIME '00:00:00')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_micros" + } + }, + { + "input": { + "query": "SELECT time_to_micros(TIME'14:30:00');", + "result": [ + "52200000000" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_micros(TIME '14:30:00')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_micros" + } + }, + { + "input": { + "query": "SELECT time_to_micros(TIME'14:30:00.5');", + "result": [ + "52200500000" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_micros(TIME '14:30:00.5')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_micros" + } + }, + { + "input": { + "query": "SELECT time_to_micros(TIME'23:59:59.999999');", + "result": [ + "86399999999" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_micros(TIME '23:59:59.999999')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_micros" + } + }, + { + "input": { + "query": "SELECT time_to_millis(TIME'00:00:00');", + "result": [ + "0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_millis(TIME '00:00:00')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_millis" + } + }, + { + "input": { + "query": "SELECT time_to_millis(TIME'14:30:00');", + "result": [ + "52200000" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_millis(TIME '14:30:00')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_millis" + } + }, + { + "input": { + "query": "SELECT time_to_millis(TIME'14:30:00.5');", + "result": [ + "52200500" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_millis(TIME '14:30:00.5')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_millis" + } + }, + { + "input": { + "query": "SELECT time_to_millis(TIME'23:59:59.999');", + "result": [ + "86399999" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_millis(TIME '23:59:59.999')", + "nullable": true, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_millis" + } + }, + { + "input": { + "query": "SELECT time_to_seconds(TIME'00:00:00');", + "result": [ + "0.000000" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_seconds(TIME '00:00:00')", + "nullable": true, + "type": "decimal(14, 6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_seconds" + } + }, + { + "input": { + "query": "SELECT time_to_seconds(TIME'14:30:00');", + "result": [ + "52200.000000" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_seconds(TIME '14:30:00')", + "nullable": true, + "type": "decimal(14, 6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_seconds" + } + }, + { + "input": { + "query": "SELECT time_to_seconds(TIME'14:30:00.5');", + "result": [ + "52200.500000" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_seconds(TIME '14:30:00.5')", + "nullable": true, + "type": "decimal(14, 6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_seconds" + } + }, + { + "input": { + "query": "SELECT time_to_seconds(TIME'23:59:59.999999');", + "result": [ + "86399.999999" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "time_to_seconds(TIME '23:59:59.999999')", + "nullable": true, + "type": "decimal(14, 6)", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: time_to_seconds" + } + }, { "input": { "query": "SELECT time_trunc('HOUR', TIME'09:32:05.359');", diff --git a/crates/sail-spark-connect/tests/gold_data/function/json.json b/crates/sail-spark-connect/tests/gold_data/function/json.json index 26bf3e1a36..156dce61bd 100644 --- a/crates/sail-spark-connect/tests/gold_data/function/json.json +++ b/crates/sail-spark-connect/tests/gold_data/function/json.json @@ -542,6 +542,28 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT to_json(named_struct('b', 1, 'a', 2), map('sortKeys', 'true'));", + "result": [ + "{\"a\":2,\"b\":1}" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "to_json(named_struct(b, 1, a, 2))", + "nullable": true, + "type": "string", + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, { "input": { "query": "SELECT to_json(named_struct('time', to_timestamp('2015-08-26', 'yyyy-MM-dd')), map('timestampFormat', 'dd/MM/yyyy'));", diff --git a/crates/sail-spark-connect/tests/gold_data/function/misc.json b/crates/sail-spark-connect/tests/gold_data/function/misc.json index ac6fe1f9ae..88bd1898d7 100644 --- a/crates/sail-spark-connect/tests/gold_data/function/misc.json +++ b/crates/sail-spark-connect/tests/gold_data/function/misc.json @@ -1,137 +1,5 @@ { "tests": [ - { - "input": { - "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(length(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_bigint" - } - }, - { - "input": { - "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(length(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_bigint" - } - }, - { - "input": { - "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(length(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_double" - } - }, - { - "input": { - "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(length(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_double" - } - }, - { - "input": { - "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(length(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_float" - } - }, - { - "input": { - "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(length(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_float" - } - }, { "input": { "query": "SELECT aes_decrypt(unbase64('2NYmDCjgXTbbxGA3/SnJEfFC/JQ7olk2VQWReIAAFKo='), '1234567890abcdef', 'CBC');", @@ -264,90 +132,6 @@ "success": "ok" } }, - { - "input": { - "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr)) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr);", - "result": [ - "[{\"item\":0,\"count\":2},{\"item\":4,\"count\":2},{\"item\":1,\"count\":2},{\"item\":2,\"count\":1},{\"item\":3,\"count\":1}]" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 5)", - "nullable": false, - "type": { - "type": "array", - "elementType": { - "type": "struct", - "fields": [ - { - "name": "item", - "nullable": true, - "type": "integer", - "metadata": {} - }, - { - "name": "count", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - }, - "containsNull": false - }, - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: approx_top_k_accumulate" - } - }, - { - "input": { - "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr), 2) FROM VALUES 'a', 'b', 'c', 'c', 'c', 'c', 'd', 'd' tab(expr);", - "result": [ - "[{\"item\":\"c\",\"count\":4},{\"item\":\"d\",\"count\":2}]" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 2)", - "nullable": false, - "type": { - "type": "array", - "elementType": { - "type": "struct", - "fields": [ - { - "name": "item", - "nullable": true, - "type": "string", - "metadata": {} - }, - { - "name": "count", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - }, - "containsNull": false - }, - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: approx_top_k_accumulate" - } - }, { "input": { "query": "SELECT assert_true(0 < 1);", @@ -658,15 +442,15 @@ }, { "input": { - "query": "SELECT current_schema();", + "query": "SELECT current_path();", "result": [ - "default" + "system.builtin,system.session,spark_catalog.default" ], "schema": { "type": "struct", "fields": [ { - "name": "current_schema()", + "name": "current_path()", "nullable": false, "type": "string", "metadata": {} @@ -675,20 +459,20 @@ } }, "output": { - "success": "ok" + "failure": "not supported: unknown function: current_path" } }, { "input": { - "query": "SELECT current_user();", + "query": "SELECT current_schema();", "result": [ - "mockingjay" + "default" ], "schema": { "type": "struct", "fields": [ { - "name": "current_user()", + "name": "current_schema()", "nullable": false, "type": "string", "metadata": {} @@ -702,32 +486,24 @@ }, { "input": { - "query": "SELECT from_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('u', NAMED_STRUCT('member0', member0, 'member1', member1)) AS s FROM VALUES (1, NULL), (NULL, 'a') tab(member0, member1));", - "result": [ - "[false]" - ], - "schema": { - "type": "struct", - "fields": [] - } - }, - "output": { - "failure": "not implemented: function: from_avro" - } - }, - { - "input": { - "query": "SELECT from_protobuf(s, 'Person', '/path/to/descriptor.desc', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('name', name, 'id', id) AS s FROM VALUES ('John Doe', 1), (NULL, 2) tab(name, id));", + "query": "SELECT current_user();", "result": [ - "[false]" + "mockingjay" ], "schema": { "type": "struct", - "fields": [] + "fields": [ + { + "name": "current_user()", + "nullable": false, + "type": "string", + "metadata": {} + } + ] } }, "output": { - "failure": "not implemented: function: from_protobuf" + "success": "ok" } }, { @@ -774,50 +550,6 @@ "success": "ok" } }, - { - "input": { - "query": "SELECT hll_sketch_estimate(hll_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);", - "result": [ - "3" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "hll_sketch_estimate(hll_sketch_agg(col, 12))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "success": "ok" - } - }, - { - "input": { - "query": "SELECT hll_sketch_estimate(hll_union(hll_sketch_agg(col1), hll_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);", - "result": [ - "6" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "hll_sketch_estimate(hll_union(hll_sketch_agg(col1, 12), hll_sketch_agg(col2, 12), false))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "success": "ok" - } - }, { "input": { "query": "SELECT input_file_block_length();", @@ -926,204 +658,6 @@ "failure": "not implemented: function: java_method" } }, - { - "input": { - "query": "SELECT kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3), (4), (5) tab(col);", - "result": [ - "5" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_bigint" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_n_double(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", - "result": [ - "5" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "kll_sketch_get_n_double(kll_sketch_agg_double(col))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_double" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_n_float(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", - "result": [ - "5" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "kll_sketch_get_n_float(kll_sketch_agg_float(col))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_float" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1 FROM VALUES (1), (2), (3), (4), (5) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_bigint" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_double" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_float" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3 FROM VALUES (1), (2), (3), (4), (5) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_bigint" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_double" - } - }, - { - "input": { - "query": "SELECT kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", - "result": [ - "true" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "(kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3)", - "nullable": false, - "type": "boolean", - "metadata": {} - } - ] - } - }, - "output": { - "failure": "not supported: unknown function: kll_sketch_agg_float" - } - }, { "input": { "query": "SELECT monotonically_increasing_id();", @@ -1212,21 +746,6 @@ "failure": "not implemented: function: reflect" } }, - { - "input": { - "query": "SELECT schema_of_avro('{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{\"name\": \"u\", \"type\": [\"int\", \"string\"]}]}', map());", - "result": [ - "STRUCT NOT NULL>" - ], - "schema": { - "type": "struct", - "fields": [] - } - }, - "output": { - "failure": "not implemented: function: schema_of_avro" - } - }, { "input": { "query": "SELECT session_user();", @@ -1271,139 +790,6 @@ "success": "ok" } }, - { - "input": { - "query": "SELECT theta_sketch_estimate(theta_difference(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);", - "result": [ - "2" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "theta_sketch_estimate(theta_difference(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "success": "ok" - } - }, - { - "input": { - "query": "SELECT theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);", - "result": [ - "2" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "success": "ok" - } - }, - { - "input": { - "query": "SELECT theta_sketch_estimate(theta_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);", - "result": [ - "3" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "theta_sketch_estimate(theta_sketch_agg(col, 12))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "success": "ok" - } - }, - { - "input": { - "query": "SELECT theta_sketch_estimate(theta_union(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);", - "result": [ - "6" - ], - "schema": { - "type": "struct", - "fields": [ - { - "name": "theta_sketch_estimate(theta_union(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12), 12))", - "nullable": false, - "type": "long", - "metadata": {} - } - ] - } - }, - "output": { - "success": "ok" - } - }, - { - "input": { - "query": "SELECT to_avro(s) IS NULL FROM (SELECT NULL AS s);", - "result": [ - "[true]" - ], - "schema": { - "type": "struct", - "fields": [] - } - }, - "output": { - "failure": "not implemented: function: to_avro" - } - }, - { - "input": { - "query": "SELECT to_avro(s, '{\"type\": \"record\", \"name\": \"struct\", \"fields\": [{ \"name\": \"u\", \"type\": [\"int\",\"string\"] }]}') IS NULL FROM (SELECT NULL AS s);", - "result": [ - "[true]" - ], - "schema": { - "type": "struct", - "fields": [] - } - }, - "output": { - "failure": "not implemented: function: to_avro" - } - }, - { - "input": { - "query": "SELECT to_protobuf(s, 'Person', '/path/to/descriptor.desc', map('emitDefaultValues', 'true')) IS NULL FROM (SELECT NULL AS s);", - "result": [ - "[true]" - ], - "schema": { - "type": "struct", - "fields": [] - } - }, - "output": { - "failure": "not implemented: function: to_protobuf" - } - }, { "input": { "query": "SELECT try_aes_decrypt(unhex('----------468D3084B5744BCA729FB7B2B7BCB8E4472847D02670489D95FA97DBBA7D3210'), '0000111122223333', 'GCM');", diff --git a/crates/sail-spark-connect/tests/gold_data/function/protobuf.json b/crates/sail-spark-connect/tests/gold_data/function/protobuf.json new file mode 100644 index 0000000000..323b306824 --- /dev/null +++ b/crates/sail-spark-connect/tests/gold_data/function/protobuf.json @@ -0,0 +1,34 @@ +{ + "tests": [ + { + "input": { + "query": "SELECT from_protobuf(s, 'Person', '/path/to/descriptor.desc', map()) IS NULL AS result FROM (SELECT NAMED_STRUCT('name', name, 'id', id) AS s FROM VALUES ('John Doe', 1), (NULL, 2) tab(name, id));", + "result": [ + "[false]" + ], + "schema": { + "type": "struct", + "fields": [] + } + }, + "output": { + "failure": "not implemented: function: from_protobuf" + } + }, + { + "input": { + "query": "SELECT to_protobuf(s, 'Person', '/path/to/descriptor.desc', map('emitDefaultValues', 'true')) IS NULL FROM (SELECT NULL AS s);", + "result": [ + "[true]" + ], + "schema": { + "type": "struct", + "fields": [] + } + }, + "output": { + "failure": "not implemented: function: to_protobuf" + } + } + ] +} diff --git a/crates/sail-spark-connect/tests/gold_data/function/sketch.json b/crates/sail-spark-connect/tests/gold_data/function/sketch.json new file mode 100644 index 0000000000..c29ff7f508 --- /dev/null +++ b/crates/sail-spark-connect/tests/gold_data/function/sketch.json @@ -0,0 +1,946 @@ +{ + "tests": [ + { + "input": { + "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(length(kll_sketch_to_string_bigint(kll_sketch_agg_bigint(col))) > 0)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_bigint" + } + }, + { + "input": { + "query": "SELECT LENGTH(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0 FROM VALUES (1), (2), (3), (4), (5) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(length(kll_sketch_to_string_bigint(kll_sketch_merge_bigint(kll_sketch_agg_bigint(col), kll_sketch_agg_bigint(col)))) > 0)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_bigint" + } + }, + { + "input": { + "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(length(kll_sketch_to_string_double(kll_sketch_agg_double(col))) > 0)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT LENGTH(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(length(kll_sketch_to_string_double(kll_sketch_merge_double(kll_sketch_agg_double(col), kll_sketch_agg_double(col)))) > 0)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(length(kll_sketch_to_string_float(kll_sketch_agg_float(col))) > 0)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_float" + } + }, + { + "input": { + "query": "SELECT LENGTH(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(length(kll_sketch_to_string_float(kll_sketch_merge_float(kll_sketch_agg_float(col), kll_sketch_agg_float(col)))) > 0)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_float" + } + }, + { + "input": { + "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr)) FROM VALUES (0), (0), (1), (1), (2), (3), (4), (4) AS tab(expr);", + "result": [ + "[{\"item\":0,\"count\":2},{\"item\":4,\"count\":2},{\"item\":1,\"count\":2},{\"item\":2,\"count\":1},{\"item\":3,\"count\":1}]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 5)", + "nullable": false, + "type": { + "type": "array", + "elementType": { + "type": "struct", + "fields": [ + { + "name": "item", + "nullable": true, + "type": "integer", + "metadata": {} + }, + { + "name": "count", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + }, + "containsNull": false + }, + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: approx_top_k_accumulate" + } + }, + { + "input": { + "query": "SELECT approx_top_k_estimate(approx_top_k_accumulate(expr), 2) FROM VALUES 'a', 'b', 'c', 'c', 'c', 'c', 'd', 'd' tab(expr);", + "result": [ + "[{\"item\":\"c\",\"count\":4},{\"item\":\"d\",\"count\":2}]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "approx_top_k_estimate(approx_top_k_accumulate(expr, 10000), 2)", + "nullable": false, + "type": { + "type": "array", + "elementType": { + "type": "struct", + "fields": [ + { + "name": "item", + "nullable": true, + "type": "string", + "metadata": {} + }, + { + "name": "count", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + }, + "containsNull": false + }, + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: approx_top_k_accumulate" + } + }, + { + "input": { + "query": "SELECT hll_sketch_estimate(hll_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);", + "result": [ + "3" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "hll_sketch_estimate(hll_sketch_agg(col, 12))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, + { + "input": { + "query": "SELECT hll_sketch_estimate(hll_union(hll_sketch_agg(col1), hll_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);", + "result": [ + "6" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "hll_sketch_estimate(hll_union(hll_sketch_agg(col1, 12), hll_sketch_agg(col2, 12), false))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col)) FROM VALUES (1), (2), (3), (4), (5) tab(col);", + "result": [ + "5" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "kll_sketch_get_n_bigint(kll_sketch_agg_bigint(col))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_bigint" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_n_double(kll_sketch_agg_double(col)) FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", + "result": [ + "5" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "kll_sketch_get_n_double(kll_sketch_agg_double(col))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_n_float(kll_sketch_agg_float(col)) FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", + "result": [ + "5" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "kll_sketch_get_n_float(kll_sketch_agg_float(col))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_float" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1 FROM VALUES (1), (2), (3), (4), (5) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(kll_sketch_get_quantile_bigint(kll_sketch_agg_bigint(col), 0.5) > 1)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_bigint" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(kll_sketch_get_quantile_double(kll_sketch_agg_double(col), 0.5) > 1)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(kll_sketch_get_quantile_float(kll_sketch_agg_float(col), 0.5) > 1)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_float" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3 FROM VALUES (1), (2), (3), (4), (5) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(kll_sketch_get_rank_bigint(kll_sketch_agg_bigint(col), 3) > 0.3)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_bigint" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS DOUBLE)), (CAST(2.0 AS DOUBLE)), (CAST(3.0 AS DOUBLE)), (CAST(4.0 AS DOUBLE)), (CAST(5.0 AS DOUBLE)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(kll_sketch_get_rank_double(kll_sketch_agg_double(col), 3.0) > 0.3)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3 FROM VALUES (CAST(1.0 AS FLOAT)), (CAST(2.0 AS FLOAT)), (CAST(3.0 AS FLOAT)), (CAST(4.0 AS FLOAT)), (CAST(5.0 AS FLOAT)) tab(col);", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "(kll_sketch_get_rank_float(kll_sketch_agg_float(col), 3.0) > 0.3)", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: kll_sketch_agg_float" + } + }, + { + "input": { + "query": "SELECT theta_sketch_estimate(theta_difference(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);", + "result": [ + "2" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "theta_sketch_estimate(theta_difference(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, + { + "input": { + "query": "SELECT theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (5, 4), (1, 4), (2, 5), (2, 5), (3, 1) tab(col1, col2);", + "result": [ + "2" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "theta_sketch_estimate(theta_intersection(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12)))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, + { + "input": { + "query": "SELECT theta_sketch_estimate(theta_sketch_agg(col)) FROM VALUES (1), (1), (2), (2), (3) tab(col);", + "result": [ + "3" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "theta_sketch_estimate(theta_sketch_agg(col, 12))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, + { + "input": { + "query": "SELECT theta_sketch_estimate(theta_union(theta_sketch_agg(col1), theta_sketch_agg(col2))) FROM VALUES (1, 4), (1, 4), (2, 5), (2, 5), (3, 6) tab(col1, col2);", + "result": [ + "6" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "theta_sketch_estimate(theta_union(theta_sketch_agg(col1, 12), theta_sketch_agg(col2, 12), 12))", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "success": "ok" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_difference_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (5, 5.0D, 4, 4.0D), (1, 1.0D, 4, 4.0D), (2, 2.0D, 5, 5.0D), (3, 3.0D, 1, 1.0D) tab(col1, val1, col2, val2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_difference_double(tuple_sketch_agg_double(col1, val1, 12, sum), tuple_sketch_agg_double(col2, val2, 12, sum)))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_difference_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (5, 5.0D, 4), (1, 1.0D, 4), (2, 2.0D, 5), (3, 3.0D, 1) tab(col1, val1, col2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_difference_theta_double(tuple_sketch_agg_double(col1, val1, 12, sum), theta_sketch_agg(col2, 12)))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_intersection_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (1, 1.0D, 1, 4.0D), (2, 2.0D, 2, 5.0D), (3, 3.0D, 4, 6.0D) tab(col1, val1, col2, val2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_intersection_double(tuple_sketch_agg_double(col1, val1, 12, sum), tuple_sketch_agg_double(col2, val2, 12, sum), sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_intersection_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1.0D, 1), (2, 2.0D, 2), (3, 3.0D, 4) tab(col1, val1, col2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_intersection_theta_double(tuple_sketch_agg_double(col1, val1, 12, sum), theta_sketch_agg(col2, 12), sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (1, 2.0D), (2, 3.0D) tab(key, summary);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_sketch_agg_double(key, summary, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_union_double(tuple_sketch_agg_double(col1, val1), tuple_sketch_agg_double(col2, val2))) FROM VALUES (1, 1.0D, 4, 4.0D), (2, 2.0D, 5, 5.0D), (3, 3.0D, 6, 6.0D) tab(col1, val1, col2, val2);", + "result": [ + "6.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_union_double(tuple_sketch_agg_double(col1, val1, 12, sum), tuple_sketch_agg_double(col2, val2, 12, sum), 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_double(tuple_union_theta_double(tuple_sketch_agg_double(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1.0D, 4), (2, 2.0D, 5), (3, 3.0D, 6) tab(col1, val1, col2);", + "result": [ + "6.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_double(tuple_union_theta_double(tuple_sketch_agg_double(col1, val1, 12, sum), theta_sketch_agg(col2, 12), 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_difference_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (5, 5, 4, 4), (1, 1, 4, 4), (2, 2, 5, 5), (3, 3, 1, 1) tab(col1, val1, col2, val2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_difference_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), tuple_sketch_agg_integer(col2, val2, 12, sum)))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_difference_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (5, 5, 4), (1, 1, 4), (2, 2, 5), (3, 3, 1) tab(col1, val1, col2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_difference_theta_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), theta_sketch_agg(col2, 12)))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_intersection_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (1, 1, 1, 4), (2, 2, 2, 5), (3, 3, 4, 6) tab(col1, val1, col2, val2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_intersection_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), tuple_sketch_agg_integer(col2, val2, 12, sum), sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_intersection_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1, 1), (2, 2, 2), (3, 3, 4) tab(col1, val1, col2);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_intersection_theta_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), theta_sketch_agg(col2, 12), sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (1, 2), (2, 3) tab(key, summary);", + "result": [ + "2.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, summary, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_union_integer(tuple_sketch_agg_integer(col1, val1), tuple_sketch_agg_integer(col2, val2))) FROM VALUES (1, 1, 4, 4), (2, 2, 5, 5), (3, 3, 6, 6) tab(col1, val1, col2, val2);", + "result": [ + "6.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_union_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), tuple_sketch_agg_integer(col2, val2, 12, sum), 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_estimate_integer(tuple_union_theta_integer(tuple_sketch_agg_integer(col1, val1), theta_sketch_agg(col2))) FROM VALUES (1, 1, 4), (2, 2, 5), (3, 3, 6) tab(col1, val1, col2);", + "result": [ + "6.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_estimate_integer(tuple_union_theta_integer(tuple_sketch_agg_integer(col1, val1, 12, sum), theta_sketch_agg(col2, 12), 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_summary_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (1, 2.0D), (2, 3.0D) tab(key, summary);", + "result": [ + "6.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_summary_double(tuple_sketch_agg_double(key, summary, 12, sum), sum)", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (1, 2), (2, 3) tab(key, summary);", + "result": [ + "6" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, summary, 12, sum), sum)", + "nullable": false, + "type": "long", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_theta_double(tuple_sketch_agg_double(key, summary)) FROM VALUES (1, 1.0D), (2, 2.0D), (3, 3.0D) tab(key, summary);", + "result": [ + "1.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_theta_double(tuple_sketch_agg_double(key, summary, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_double" + } + }, + { + "input": { + "query": "SELECT tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, summary)) FROM VALUES (1, 1), (2, 2), (3, 3) tab(key, summary);", + "result": [ + "1.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, summary, 12, sum))", + "nullable": false, + "type": "double", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: tuple_sketch_agg_integer" + } + } + ] +} diff --git a/crates/sail-spark-connect/tests/gold_data/function/st.json b/crates/sail-spark-connect/tests/gold_data/function/st.json index 843a5597bf..f18e519710 100644 --- a/crates/sail-spark-connect/tests/gold_data/function/st.json +++ b/crates/sail-spark-connect/tests/gold_data/function/st.json @@ -10,7 +10,7 @@ "type": "struct", "fields": [ { - "name": "hex(st_asbinary(st_geogfromwkb(X'0101000000000000000000F03F0000000000000040')))", + "name": "hex(st_asbinary(st_geogfromwkb(X'0101000000000000000000F03F0000000000000040'), NDR))", "nullable": false, "type": "string", "metadata": {} @@ -32,7 +32,7 @@ "type": "struct", "fields": [ { - "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040')))", + "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0), NDR))", "nullable": false, "type": "string", "metadata": {} @@ -44,6 +44,50 @@ "success": "ok" } }, + { + "input": { + "query": "SELECT hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'), 'XDR'));", + "result": [ + "00000000013FF00000000000004000000000000000" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0), XDR))", + "nullable": false, + "type": "string", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "error in DataFusion: Error during planning: one value expected: [ScalarFunction(ScalarFunction { func: ScalarUDF { inner: StGeomFromWKB { signature: Signature { type_signature: Exact([Binary]), volatility: Immutable, parameter_names: None } } }, args: [Literal(Binary(\"1,1,0,0,0,0,0,0,0,0,0,240,63,0,0,0,0,0,0,0,64\"), None)] }), Literal(Utf8(\"XDR\"), None)]" + } + }, + { + "input": { + "query": "SELECT hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326)));", + "result": [ + "0101000000000000000000F03F0000000000000040" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "hex(st_asbinary(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326), NDR))", + "nullable": false, + "type": "string", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "error in DataFusion: Error during planning: one value expected: [Literal(Binary(\"1,1,0,0,0,0,0,0,0,0,0,240,63,0,0,0,0,0,0,0,64\"), None), Literal(Int32(4326), None)]" + } + }, { "input": { "query": "SELECT st_srid(NULL);", @@ -98,7 +142,7 @@ "type": "struct", "fields": [ { - "name": "st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'))", + "name": "st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0))", "nullable": false, "type": "integer", "metadata": {} @@ -110,6 +154,28 @@ "failure": "not implemented: function: st_srid" } }, + { + "input": { + "query": "SELECT st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326));", + "result": [ + "4326" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "st_srid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 4326))", + "nullable": false, + "type": "integer", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "error in DataFusion: Error during planning: one value expected: [Literal(Binary(\"1,1,0,0,0,0,0,0,0,0,0,240,63,0,0,0,0,0,0,0,64\"), None), Literal(Int32(4326), None)]" + } + }, { "input": { "query": "SELECT st_srid(st_setsrid(ST_GeogFromWKB(X'0101000000000000000000F03F0000000000000040'), 4326));", @@ -142,7 +208,7 @@ "type": "struct", "fields": [ { - "name": "st_srid(st_setsrid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040'), 3857))", + "name": "st_srid(st_setsrid(st_geomfromwkb(X'0101000000000000000000F03F0000000000000040', 0), 3857))", "nullable": false, "type": "integer", "metadata": {} diff --git a/crates/sail-spark-connect/tests/gold_data/function/variant.json b/crates/sail-spark-connect/tests/gold_data/function/variant.json index e681b74570..54a9f12526 100644 --- a/crates/sail-spark-connect/tests/gold_data/function/variant.json +++ b/crates/sail-spark-connect/tests/gold_data/function/variant.json @@ -1,5 +1,49 @@ { "tests": [ + { + "input": { + "query": "SELECT is_valid_variant(parse_json('[{\"b\":true,\"a\":0}]'));", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "is_valid_variant(parse_json([{\"b\":true,\"a\":0}]))", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: is_valid_variant" + } + }, + { + "input": { + "query": "SELECT is_valid_variant(parse_json('null'));", + "result": [ + "true" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "is_valid_variant(parse_json(null))", + "nullable": false, + "type": "boolean", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: is_valid_variant" + } + }, { "input": { "query": "SELECT is_variant_null(parse_json('\"null\"'));", diff --git a/crates/sail-spark-connect/tests/gold_data/function/vector.json b/crates/sail-spark-connect/tests/gold_data/function/vector.json new file mode 100644 index 0000000000..e52d67c09b --- /dev/null +++ b/crates/sail-spark-connect/tests/gold_data/function/vector.json @@ -0,0 +1,266 @@ +{ + "tests": [ + { + "input": { + "query": "SELECT vector_avg(col) FROM VALUES (array(1.0F, 2.0F)), (array(3.0F, 4.0F)) AS tab(col);", + "result": [ + "[2.0,3.0]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_avg(col)", + "nullable": true, + "type": { + "type": "array", + "elementType": "float", + "containsNull": false + }, + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_avg" + } + }, + { + "input": { + "query": "SELECT vector_cosine_similarity(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));", + "result": [ + "0.9746319" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_cosine_similarity(array(1.0, 2.0, 3.0), array(4.0, 5.0, 6.0))", + "nullable": true, + "type": "float", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_cosine_similarity" + } + }, + { + "input": { + "query": "SELECT vector_inner_product(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));", + "result": [ + "32.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_inner_product(array(1.0, 2.0, 3.0), array(4.0, 5.0, 6.0))", + "nullable": true, + "type": "float", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_inner_product" + } + }, + { + "input": { + "query": "SELECT vector_l2_distance(array(1.0F, 2.0F, 3.0F), array(4.0F, 5.0F, 6.0F));", + "result": [ + "5.196152" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_l2_distance(array(1.0, 2.0, 3.0), array(4.0, 5.0, 6.0))", + "nullable": true, + "type": "float", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_l2_distance" + } + }, + { + "input": { + "query": "SELECT vector_norm(array(3.0F, 4.0F), 1.0F);", + "result": [ + "7.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_norm(array(3.0, 4.0), 1.0)", + "nullable": true, + "type": "float", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_norm" + } + }, + { + "input": { + "query": "SELECT vector_norm(array(3.0F, 4.0F), 2.0F);", + "result": [ + "5.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_norm(array(3.0, 4.0), 2.0)", + "nullable": true, + "type": "float", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_norm" + } + }, + { + "input": { + "query": "SELECT vector_norm(array(3.0F, 4.0F), float('inf'));", + "result": [ + "4.0" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_norm(array(3.0, 4.0), inf)", + "nullable": true, + "type": "float", + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_norm" + } + }, + { + "input": { + "query": "SELECT vector_normalize(array(3.0F, 4.0F), 1.0F);", + "result": [ + "[0.42857143,0.5714286]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_normalize(array(3.0, 4.0), 1.0)", + "nullable": true, + "type": { + "type": "array", + "elementType": "float", + "containsNull": true + }, + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_normalize" + } + }, + { + "input": { + "query": "SELECT vector_normalize(array(3.0F, 4.0F), 2.0F);", + "result": [ + "[0.6,0.8]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_normalize(array(3.0, 4.0), 2.0)", + "nullable": true, + "type": { + "type": "array", + "elementType": "float", + "containsNull": true + }, + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_normalize" + } + }, + { + "input": { + "query": "SELECT vector_normalize(array(3.0F, 4.0F), float('inf'));", + "result": [ + "[0.75,1.0]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_normalize(array(3.0, 4.0), inf)", + "nullable": true, + "type": { + "type": "array", + "elementType": "float", + "containsNull": true + }, + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_normalize" + } + }, + { + "input": { + "query": "SELECT vector_sum(col) FROM VALUES (array(1.0F, 2.0F)), (array(3.0F, 4.0F)) AS tab(col);", + "result": [ + "[4.0,6.0]" + ], + "schema": { + "type": "struct", + "fields": [ + { + "name": "vector_sum(col)", + "nullable": true, + "type": { + "type": "array", + "elementType": "float", + "containsNull": false + }, + "metadata": {} + } + ] + } + }, + "output": { + "failure": "not supported: unknown function: vector_sum" + } + } + ] +} diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json index f6f37db520..9e396f0a0d 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_describe.json @@ -67,76 +67,6 @@ } } } - }, - { - "input": "DESCRIBE TABLE EXTENDED a.b", - "output": { - "success": { - "command": { - "describeTable": { - "table": [ - "a", - "b" - ], - "extended": true, - "partition": [], - "column": null - }, - "planId": null - } - } - } - }, - { - "input": "DESCRIBE EXTENDED a.b", - "output": { - "success": { - "command": { - "describeTable": { - "table": [ - "a", - "b" - ], - "extended": true, - "partition": [], - "column": null - }, - "planId": null - } - } - } - }, - { - "input": "DESCRIBE QUERY SELECT 1", - "output": { - "success": { - "command": { - "describeQuery": { - "query": { - "project": { - "input": { - "empty": { - "produceOneRow": true - }, - "planId": null - }, - "expressions": [ - { - "literal": { - "int32": { - "value": 1 - } - } - } - ] - }, - "planId": null - } - }, - "planId": null - } - } - } } ] } diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json index e9b150110c..b1f435f7f0 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_insert_into.json @@ -811,6 +811,12 @@ } } }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl\nREPLACE WHERE a > 5\n(SELECT * FROM source) AS s", + "output": { + "failure": "invalid argument: found AS at 75:77 expected 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "INSERT INTO testcat.ns1.ns2.tbl (a, b) SELECT * FROM source", "output": { @@ -874,6 +880,74 @@ } } }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t\nREPLACE ON (t.a = s.b)\n(SELECT * FROM source) AS s", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t\nREPLACE USING (col1, col2)\n(SELECT * FROM source) AS s", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE ON t.col1 = col2 SELECT * FROM source", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE USING (col1, col2) SELECT * FROM source", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE WHERE a > 5 SELECT * FROM source", + "exception": "\n[INSERT_REPLACE_WHERE_TABLE_ALIAS_NOT_ALLOWED] Table alias is not allowed with INSERT INTO ... REPLACE WHERE because the WHERE condition is evaluated against the target table directly.\nUse INSERT INTO ... REPLACE ON if you need to reference the target table via an alias. SQLSTATE: 42000\n== SQL (line 1, position 1) ==\nINSERT INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE WHERE a > 5 SELECT * FROM source\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE ON t.a = s.a AND t.b = s.b (SELECT * FROM source) AS s", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE ON t.col1 = col2 SELECT * FROM source", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE USING (col1, col2) SELECT * FROM source", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE USING (id) SELECT * FROM source", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl AS t REPLACE WHERE a > 5 SELECT * FROM source", + "exception": "\n[INSERT_REPLACE_WHERE_TABLE_ALIAS_NOT_ALLOWED] Table alias is not allowed with INSERT INTO ... REPLACE WHERE because the WHERE condition is evaluated against the target table directly.\nUse INSERT INTO ... REPLACE ON if you need to reference the target table via an alias. SQLSTATE: 42000\n== SQL (line 1, position 1) ==\nINSERT INTO testcat.ns1.ns2.tbl AS t REPLACE WHERE a > 5 SELECT * FROM source\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found AS at 32:34 expected '.', 'REPLACE', 'PARTITION', 'IF', 'BY', '(', or query" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl BY NAME REPLACE WHERE a > 5 SELECT * FROM source", + "output": { + "failure": "invalid argument: found REPLACE at 40:47 expected query" + } + }, { "input": "INSERT INTO testcat.ns1.ns2.tbl BY NAME SELECT * FROM source", "output": { @@ -933,6 +1007,105 @@ } } }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl REPLACE ON col1 = col2 SELECT * FROM source", + "output": { + "failure": "invalid argument: found ON at 40:42 expected 'WHERE'" + } + }, + { + "input": "INSERT INTO testcat.ns1.ns2.tbl REPLACE WHERE a > 5 SELECT * FROM source", + "output": { + "success": { + "command": { + "insertInto": { + "input": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "source" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + }, + "table": [ + "testcat", + "ns1", + "ns2", + "tbl" + ], + "mode": { + "replace": { + "condition": { + "expr": { + "unresolvedFunction": { + "functionName": [ + ">" + ], + "arguments": [ + { + "unresolvedAttribute": { + "name": [ + "a" + ], + "planId": null, + "isMetadataColumn": false + } + }, + { + "literal": { + "int32": { + "value": 5 + } + } + } + ], + "namedArguments": [], + "isDistinct": false, + "isUserDefinedFunction": false, + "isInternal": null, + "ignoreNulls": null, + "filter": null, + "orderBy": null + } + }, + "source": "a > 5 " + } + } + }, + "partition": [], + "ifNotExists": false + }, + "planId": null + } + } + } + }, { "input": "INSERT INTO testcat.ns1.ns2.tbl SELECT * FROM source", "output": { diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json index b4520a1ac0..30c0f25992 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_merge_into.json @@ -2576,6 +2576,13 @@ } } }, + { + "input": "MERGE INTO t1 USING t2 ON t1.id = t2.id\nWHEN NOT MATCHED THEN INSERT (col1, col2) VALUES (1)", + "exception": "\n[MERGE_INSERT_VALUE_COUNT_MISMATCH] The number of provided values (1) must match the number of expected columns (2) in the INSERT clause of MERGE. SQLSTATE: 21S01\n== SQL (line 1, position 1) ==\nMERGE INTO t1 USING t2 ON t1.id = t2.id\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nWHEN NOT MATCHED THEN INSERT (col1, col2) VALUES (1)\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: INSERT action has 2 columns but 1 expressions" + } + }, { "input": "MERGE INTO testcat1.ns1.ns2.tbl AS target\nUSING testcat2.ns1.ns2.tbl AS source\nON target.col1 = source.col1", "exception": "\n[MERGE_WITHOUT_WHEN] There must be at least one WHEN clause in a MERGE statement. SQLSTATE: 42601\n== SQL (line 1, position 1) ==\nMERGE INTO testcat1.ns1.ns2.tbl AS target\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nUSING testcat2.ns1.ns2.tbl AS source\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nON target.col1 = source.col1\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", diff --git a/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json b/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json index 3f08fb22fc..4e7f30cc56 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/ddl_misc.json @@ -150,6 +150,66 @@ } } }, + { + "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl (a, b)SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl AS t BY NAME REPLACE ON t.col1 = col2 SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl AS t REPLACE ON t.col1 = col2 SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl BY NAME REPLACE USING (col1) SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl BY NAME SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl REPLACE USING (col1) SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION INTO testcat.ns1.ns2.tbl SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION OVERWRITE testcat.ns1.ns2.tbl (a, b)SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION OVERWRITE testcat.ns1.ns2.tbl BY NAME SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, + { + "input": "INSERT WITH SCHEMA EVOLUTION OVERWRITE testcat.ns1.ns2.tbl SELECT * FROM source", + "output": { + "failure": "invalid argument: found WITH at 7:11 expected 'OVERWRITE', or 'INTO'" + } + }, { "input": "REFRESH FUNCTION a.b.c", "output": { diff --git a/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json b/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json index c728ce5e6b..669ba5c013 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/error_misc.json @@ -44,7 +44,7 @@ }, { "input": "SET CATALOG test-test", - "exception": "\n[INVALID_IDENTIFIER] The unquoted identifier test-test is invalid and must be back quoted as: `test-test`.\nUnquoted identifiers can only contain ASCII letters ('a' - 'z', 'A' - 'Z'), digits ('0' - '9'), and underbar ('_').\nUnquoted identifiers must also not start with a digit.\nDifferent data sources and meta stores may impose additional restrictions on valid identifiers. SQLSTATE: 42602 (line 1, pos 16)\n\n== SQL ==\nSET CATALOG test-test\n----------------^^^\n", + "exception": "\n[INVALID_SQL_SYNTAX.UNSUPPORTED_SQL_STATEMENT] Invalid SQL syntax: Unsupported SQL statement: SET CATALOG test-test. SQLSTATE: 42000\n== SQL (line 1, position 1) ==\nSET CATALOG test-test\n^^^^^^^^^^^^^^^^^^^^^\n", "output": { "failure": "invalid argument: found - at 16:17 expected ';', or end of input" } diff --git a/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json b/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json index 19f6bded0a..ca80fdfd3d 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/plan_join.json @@ -126,6 +126,12 @@ } } }, + { + "input": "SELECT * FROM t1 CHANGES FROM VERSION 1 TO VERSION 5 AS a JOIN t2 CHANGES FROM VERSION 1 TO VERSION 5 AS b ON a.id = b.id", + "output": { + "failure": "invalid argument: found FROM at 25:29 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from a join b join c right join d", "output": { @@ -1476,6 +1482,13 @@ } } }, + { + "input": "select * from t cross join u approx nearest 1 by similarity t.a", + "exception": "\n[NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE] Invalid nearest-by join. Unsupported nearest-by join type CROSS. Supported types: 'INNER', 'LEFT OUTER'. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t cross join u approx nearest 1 by similarity t.a\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found nearest at 36:43 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from t full join u as uu on a = b", "output": { @@ -1649,6 +1662,13 @@ } } }, + { + "input": "select * from t full outer join u approx nearest 1 by similarity t.a", + "exception": "\n[NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE] Invalid nearest-by join. Unsupported nearest-by join type FULL OUTER. Supported types: 'INNER', 'LEFT OUTER'. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t full outer join u approx nearest 1 by similarity t.a\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found nearest at 41:48 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from t full outer join u as uu on a = b", "output": { @@ -2016,6 +2036,12 @@ } } }, + { + "input": "select * from t inner join u exact nearest 3 by distance t.a - u.a", + "output": { + "failure": "invalid argument: found nearest at 35:42 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from t inner join u using(a, b)", "output": { @@ -2173,6 +2199,13 @@ } } }, + { + "input": "select * from t join lateral (select * from u) uu approx nearest 1 by similarity 1", + "exception": "\n[UNSUPPORTED_FEATURE.LATERAL_JOIN_NEAREST_BY] The feature is not supported: LATERAL correlation with NEAREST BY clause. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t join lateral (select * from u) uu approx nearest 1 by similarity 1\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found approx at 50:56 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from t join lateral (select * from u) uu on true", "output": { @@ -2268,6 +2301,33 @@ } } }, + { + "input": "select * from t join u approx nearest 0 by similarity t.a", + "exception": "\n[NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE] Invalid nearest-by join. The number of results 0 must be between 1 and 100000. Update the literal in `APPROX NEAREST 0 BY ...` (or `EXACT NEAREST 0 BY ...`) to fall within that range. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t join u approx nearest 0 by similarity t.a\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "select * from t join u approx nearest 100001 by distance t.a", + "exception": "\n[NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE] Invalid nearest-by join. The number of results 100001 must be between 1 and 100000. Update the literal in `APPROX NEAREST 100001 BY ...` (or `EXACT NEAREST 100001 BY ...`) to fall within that range. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t join u approx nearest 100001 by distance t.a\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "select * from t join u approx nearest 5 by similarity t.a + u.a", + "output": { + "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "select * from t join u approx nearest 99999999999999999999 by distance t.a", + "exception": "\n[NEAREST_BY_JOIN.NUM_RESULTS_OUT_OF_RANGE] Invalid nearest-by join. The number of results 99999999999999999999 must be between 1 and 100000. Update the literal in `APPROX NEAREST 99999999999999999999 BY ...` (or `EXACT NEAREST 99999999999999999999 BY ...`) to fall within that range. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t join u approx nearest 99999999999999999999 by distance t.a\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found nearest at 30:37 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from t join u as uu on a = b", "output": { @@ -2965,6 +3025,12 @@ } } }, + { + "input": "select * from t left outer join u approx nearest by similarity t.a + u.a", + "output": { + "failure": "invalid argument: found nearest at 41:48 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from t left outer join u as uu on a = b", "output": { @@ -3484,6 +3550,13 @@ } } }, + { + "input": "select * from t right outer join u approx nearest 1 by similarity t.a", + "exception": "\n[NEAREST_BY_JOIN.UNSUPPORTED_JOIN_TYPE] Invalid nearest-by join. Unsupported nearest-by join type RIGHT OUTER. Supported types: 'INNER', 'LEFT OUTER'. SQLSTATE: 42604\n== SQL (line 1, position 17) ==\nselect * from t right outer join u approx nearest 1 by similarity t.a\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found nearest at 42:49 expected '(', 'ON', 'USING', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "select * from t right outer join u as uu on a = b", "output": { diff --git a/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json b/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json index 186bce4ba2..7e6d5b8f1e 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/plan_select.json @@ -115,6 +115,18 @@ "failure": "invalid argument: found INTO at 22:26 expected '.', '(', 'FOR', 'SYSTEM_VERSION', 'VERSION', 'SYSTEM_TIME', 'TIMESTAMP', 'TABLESAMPLE', 'PIVOT', 'UNPIVOT', 'AS', identifier, 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', or ')'" } }, + { + "input": "SELECT * FROM (SELECT * FROM t CHANGES FROM VERSION 1 TO VERSION 5) sub", + "output": { + "failure": "invalid argument: found FROM at 39:43 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', or ')'" + } + }, + { + "input": "SELECT * FROM (SELECT * FROM t TABLESAMPLE SYSTEM (50 PERCENT)) sub", + "output": { + "failure": "invalid argument: found SYSTEM at 43:49 expected '('" + } + }, { "input": "SELECT * FROM (SELECT a AS x, b AS y FROM t) t(col1, col2)", "output": { @@ -209,7 +221,7 @@ }, { "input": "SELECT * FROM S WHERE C1 IN (INSERT INTO T VALUES (2))", - "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'IN'. SQLSTATE: 42601 (line 1, pos 25)\n\n== SQL ==\nSELECT * FROM S WHERE C1 IN (INSERT INTO T VALUES (2))\n-------------------------^^^\n", + "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'INTO'. SQLSTATE: 42601 (line 1, pos 36)\n\n== SQL ==\nSELECT * FROM S WHERE C1 IN (INSERT INTO T VALUES (2))\n------------------------------------^^^\n", "output": { "failure": "invalid argument: found INTO at 36:40 expected '->', '.', '(', '[', '::', 'ESCAPE', 'IS', 'NOT', 'IN', '*', '/', '%', 'DIV', '+', '-', '||', '>>>', '>>', '<<', '&', '^', '|', '!=', '!>', '!<', '==', '=', '>=', '>', '<=>', '<=', '<>', '<', 'BETWEEN', 'LIKE', 'ILIKE', 'RLIKE', 'REGEXP', 'SIMILAR', 'AND', 'OR', ',', or ')'" } @@ -309,6 +321,121 @@ } } }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM SYSTEM_TIME '2026-01-01' TO SYSTEM_TIME '2026-02-01'", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM SYSTEM_VERSION 10 TO SYSTEM_VERSION 20", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01'", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01' EXCLUSIVE TO TIMESTAMP '2026-02-01' EXCLUSIVE", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01' INCLUSIVE TO TIMESTAMP '2026-02-01' INCLUSIVE", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM TIMESTAMP '2026-01-01' TO TIMESTAMP '2026-02-01'", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION '5765898212649545898' TO VERSION '8439568982649545102'", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 EXCLUSIVE TO VERSION 20", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 EXCLUSIVE TO VERSION 20 EXCLUSIVE", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 INCLUSIVE TO VERSION 20 INCLUSIVE", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 EXCLUSIVE", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (computeUpdates = 'true')", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'DROPCARRYOVERS')", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'invalid')", + "exception": "\n[INVALID_CDC_OPTION.INVALID_DEDUPLICATION_MODE] Invalid Change Data Capture (CDC) option. Invalid `deduplicationMode`: 'invalid'. Expected one of: none, dropCarryovers, netChanges. SQLSTATE: 42K03\n== SQL ==\nSELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'invalid')", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'netChanges')", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'none')", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 WITH (deduplicationMode = 'none', computeUpdates = 'true')", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM a.b.c CHANGES FROM VERSION 5", + "output": { + "failure": "invalid argument: found FROM at 28:32 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "SELECT * FROM a.b.c FOR SYSTEM_TIME AS OF '2019-01-29 00:37:58'", "output": { @@ -1557,6 +1684,85 @@ } } }, + { + "input": "SELECT * FROM bernoulli TABLESAMPLE BERNOULLI (10 PERCENT) AS x", + "output": { + "failure": "invalid argument: found BERNOULLI at 36:45 expected '('" + } + }, + { + "input": "SELECT * FROM bernoulli TABLESAMPLE(10 PERCENT) bernoulli", + "output": { + "success": { + "query": { + "project": { + "input": { + "tableAlias": { + "input": { + "read": { + "namedTable": { + "name": [ + "bernoulli" + ], + "temporal": null, + "sample": { + "method": { + "percent": { + "value": { + "literal": { + "int32": { + "value": 10 + } + } + } + } + }, + "seed": null + }, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "name": "bernoulli", + "columns": [] + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "SELECT * FROM my_table CHANGES FROM VERSION 1", + "output": { + "failure": "invalid argument: found FROM at 31:35 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM mydb.system TABLESAMPLE SYSTEM (10 PERCENT) AS x", + "output": { + "failure": "invalid argument: found SYSTEM at 38:44 expected '('" + } + }, { "input": "SELECT * FROM parquet_t0 TABLESAMPLE(300M) s", "exception": "\nTABLESAMPLE(byteLengthLiteral) is not supported.\n== SQL (line 1, position 26) ==\nSELECT * FROM parquet_t0 TABLESAMPLE(300M) s\n ^^^^^^^^^^^^^^^^^\n", @@ -1686,7 +1892,19 @@ } }, { - "input": "SELECT * FROM testData AS t(col1, col2)", + "input": "SELECT * FROM system TABLESAMPLE BERNOULLI (10 PERCENT) AS x", + "output": { + "failure": "invalid argument: found BERNOULLI at 33:42 expected '('" + } + }, + { + "input": "SELECT * FROM system TABLESAMPLE SYSTEM (10 PERCENT) AS x", + "output": { + "failure": "invalid argument: found SYSTEM at 33:39 expected '('" + } + }, + { + "input": "SELECT * FROM system TABLESAMPLE(10 PERCENT) AS x", "output": { "success": { "query": { @@ -1697,21 +1915,31 @@ "read": { "namedTable": { "name": [ - "testData" + "system" ], "temporal": null, - "sample": null, + "sample": { + "method": { + "percent": { + "value": { + "literal": { + "int32": { + "value": 10 + } + } + } + } + }, + "seed": null + }, "options": [] }, "isStreaming": false }, "planId": null }, - "name": "t", - "columns": [ - "col1", - "col2" - ] + "name": "x", + "columns": [] }, "planId": null }, @@ -1737,24 +1965,42 @@ } }, { - "input": "SELECT * FROM testcat.db.tab", + "input": "SELECT * FROM system TABLESAMPLE(10 PERCENT) system", "output": { "success": { "query": { "project": { "input": { - "read": { - "namedTable": { - "name": [ - "testcat", - "db", - "tab" - ], - "temporal": null, - "sample": null, - "options": [] + "tableAlias": { + "input": { + "read": { + "namedTable": { + "name": [ + "system" + ], + "temporal": null, + "sample": { + "method": { + "percent": { + "value": { + "literal": { + "int32": { + "value": 10 + } + } + } + } + }, + "seed": null + }, + "options": [] + }, + "isStreaming": false + }, + "planId": null }, - "isStreaming": false + "name": "system", + "columns": [] }, "planId": null }, @@ -1780,22 +2026,233 @@ } }, { - "input": "SELECT 42", + "input": "SELECT * FROM t CHANGES FROM TIMESTAMP '2026-01-01' TO TIMESTAMP (SELECT MAX(ts) FROM other_table)", + "exception": "\n[INVALID_CDC_OPTION.INVALID_TIMESTAMP_EXPR] Invalid Change Data Capture (CDC) option. The timestamp expression for CDC queries must not contain subqueries. SQLSTATE: 42K03\n== SQL ==\nSELECT * FROM t CHANGES FROM TIMESTAMP '2026-01-01' TO TIMESTAMP (SELECT MAX(ts) FROM other_table)", + "output": { + "failure": "invalid argument: found FROM at 24:28 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM t CHANGES FROM TIMESTAMP (SELECT MAX(ts) FROM other_table)", + "exception": "\n[INVALID_CDC_OPTION.INVALID_TIMESTAMP_EXPR] Invalid Change Data Capture (CDC) option. The timestamp expression for CDC queries must not contain subqueries. SQLSTATE: 42K03\n== SQL ==\nSELECT * FROM t CHANGES FROM TIMESTAMP (SELECT MAX(ts) FROM other_table)", + "output": { + "failure": "invalid argument: found FROM at 24:28 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM t CHANGES FROM TIMESTAMP col_name", + "exception": "\nInvalid time travel spec: timestamp expression cannot refer to any columns.\n== SQL (line 1, position 15) ==\nSELECT * FROM t CHANGES FROM TIMESTAMP col_name\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found FROM at 24:28 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, + { + "input": "SELECT * FROM t bernoulli", "output": { "success": { "query": { "project": { "input": { - "empty": { - "produceOneRow": true + "tableAlias": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "name": "bernoulli", + "columns": [] }, "planId": null }, "expressions": [ { - "literal": { - "int32": { - "value": 42 + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "SELECT * FROM t system", + "output": { + "success": { + "query": { + "project": { + "input": { + "tableAlias": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "name": "system", + "columns": [] + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "SELECT * FROM testData AS t(col1, col2)", + "output": { + "success": { + "query": { + "project": { + "input": { + "tableAlias": { + "input": { + "read": { + "namedTable": { + "name": [ + "testData" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "name": "t", + "columns": [ + "col1", + "col2" + ] + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "SELECT * FROM testcat.db.tab", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "testcat", + "db", + "tab" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "SELECT 42", + "output": { + "success": { + "query": { + "project": { + "input": { + "empty": { + "produceOneRow": true + }, + "planId": null + }, + "expressions": [ + { + "literal": { + "int32": { + "value": 42 } } } @@ -2427,6 +2884,43 @@ } } }, + { + "input": "SELECT bernoulli FROM t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "bernoulli" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } + }, { "input": "SELECT interval FROM VALUES ('abc') AS tbl(interval);", "output": { @@ -2473,6 +2967,49 @@ } } }, + { + "input": "SELECT system FROM t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "system" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "SELECT t.* FROM a.b.c CHANGES FROM VERSION 10 TO VERSION 20 AS t", + "output": { + "failure": "invalid argument: found FROM at 30:34 expected '(', 'NATURAL', 'INNER', 'CROSS', 'OUTER', 'SEMI', 'ANTI', 'LEFT', 'RIGHT', 'FULL', 'JOIN', ',', 'LATERAL', 'WHERE', 'GROUP', 'HAVING', 'INTERSECT', 'UNION', 'EXCEPT', 'MINUS', 'WINDOW', 'ORDER', 'SORT', 'CLUSTER', 'DISTRIBUTE', 'LIMIT', 'OFFSET', ';', or end of input" + } + }, { "input": "SELECT t.col1, t.col2 FROM (SELECT a AS x, b AS y FROM t) t(col1, col2)", "output": { @@ -3008,20 +3545,60 @@ } }, { - "input": "select * from default.range(2)", - "exception": "\n[INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME] Invalid SQL syntax: Table valued function cannot specify database name: `default`.`range`. SQLSTATE: 42000\n== SQL (line 1, position 15) ==\nselect * from default.range(2)\n ^^^^^^^^^^^^^^^^\n", + "input": "select * from approx", "output": { "success": { "query": { "project": { "input": { "read": { - "udtf": { + "namedTable": { "name": [ - "default", - "range" + "approx" ], - "arguments": [ + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "select * from default.range(2)", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "udtf": { + "name": [ + "default", + "range" + ], + "arguments": [ { "literal": { "int32": { @@ -3058,6 +3635,88 @@ } } }, + { + "input": "select * from distance", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "distance" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "select * from exact", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "exact" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, { "input": "select * from my_tvf(2, arg1 => 'value1', arg2 => true)", "output": { @@ -3604,6 +4263,47 @@ } } }, + { + "input": "select * from nearest", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "nearest" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, { "input": "select * from range(2)", "output": { @@ -3653,9 +4353,49 @@ } } }, + { + "input": "select * from similarity", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "similarity" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, { "input": "select * from spark_catalog.default.range(2)", - "exception": "\n[INVALID_SQL_SYNTAX.INVALID_TABLE_VALUED_FUNC_NAME] Invalid SQL syntax: Table valued function cannot specify database name: `spark_catalog`.`default`.`range`. SQLSTATE: 42000\n== SQL (line 1, position 15) ==\nselect * from spark_catalog.default.range(2)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", "output": { "success": { "query": { @@ -3835,6 +4575,24 @@ } } }, + { + "input": "select * from t TABLESAMPLE BERNOULLI (43 PERCENT) as x", + "output": { + "failure": "invalid argument: found BERNOULLI at 28:37 expected '('" + } + }, + { + "input": "select * from t TABLESAMPLE SYSTEM (43 PERCENT) as x", + "output": { + "failure": "invalid argument: found SYSTEM at 28:34 expected '('" + } + }, + { + "input": "select * from t TabLeSaMpLe SyStEm (43 PeRcEnT) as x", + "output": { + "failure": "invalid argument: found SyStEm at 28:34 expected '('" + } + }, { "input": "select * from t as tt , u", "output": { @@ -4187,6 +4945,98 @@ } } }, + { + "input": "select * from t tablesample bernoulli (43 percent) as x", + "output": { + "failure": "invalid argument: found bernoulli at 28:37 expected '('" + } + }, + { + "input": "select * from t tablesample bernoulli (43 percent) repeatable (123) as x", + "output": { + "failure": "invalid argument: found bernoulli at 28:37 expected '('" + } + }, + { + "input": "select * from t tablesample system (-10 percent) as x", + "exception": "\nSampling fraction (-0.1) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (-10 percent) as x\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (0 percent) as x", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (0.1 percent) as x", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (100 percent) as x", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (100 rows)", + "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support ROWS sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (100 rows)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (150 percent) as x", + "exception": "\nSampling fraction (1.5) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (150 percent) as x\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (300M)", + "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BYTES sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (300M)\n ^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (43 percent) as x", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (43 percent) repeatable (123)", + "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_REPEATABLE] The feature is not supported: TABLESAMPLE SYSTEM does not support the REPEATABLE clause. Use TABLESAMPLE BERNOULLI for repeatable sampling with a seed. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (43 percent) repeatable (123)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (bucket 3 out of 32 on rand())", + "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BUCKET sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (bucket 3 out of 32 on rand())\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (bucket 4 out of 10 on x)", + "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BUCKET sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (bucket 4 out of 10 on x)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, + { + "input": "select * from t tablesample system (bucket 4 out of 10)", + "exception": "\n[UNSUPPORTED_FEATURE.TABLESAMPLE_SYSTEM_SAMPLE_METHOD] The feature is not supported: TABLESAMPLE SYSTEM does not support BUCKET sampling. Only PERCENT sampling is supported. SQLSTATE: 0A000\n== SQL (line 1, position 17) ==\nselect * from t tablesample system (bucket 4 out of 10)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found system at 28:34 expected '('" + } + }, { "input": "select * from t tablesample(100 rows)", "output": { @@ -4303,8 +5153,132 @@ } }, { - "input": "select * from t tablesample(bucket 11 out of 10) as x", - "exception": "\nSampling fraction (1.1) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 11 out of 10) as x\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "input": "select * from t tablesample(43 percent) repeatable (10) as x", + "output": { + "success": { + "query": { + "project": { + "input": { + "tableAlias": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": { + "method": { + "percent": { + "value": { + "literal": { + "int32": { + "value": 43 + } + } + } + } + }, + "seed": 10 + }, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "name": "x", + "columns": [] + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "select * from t tablesample(bucket 11 out of 10) as x", + "exception": "\nSampling fraction (1.1) must be on interval [0, 1].\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 11 out of 10) as x\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "success": { + "query": { + "project": { + "input": { + "tableAlias": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": { + "method": { + "bucket": { + "numerator": 11, + "denominator": 10 + } + }, + "seed": null + }, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "name": "x", + "columns": [] + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "select * from t tablesample(bucket 4 out of 10 on x) as x", + "exception": "\nTABLESAMPLE(BUCKET x OUT OF y ON colname) is not supported.\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 4 out of 10 on x) as x\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "failure": "invalid argument: found on at 47:49 expected ')'" + } + }, + { + "input": "select * from t tablesample(bucket 4 out of 10) as x", "output": { "success": { "query": { @@ -4321,7 +5295,7 @@ "sample": { "method": { "bucket": { - "numerator": 11, + "numerator": 4, "denominator": 10 } }, @@ -4360,14 +5334,7 @@ } }, { - "input": "select * from t tablesample(bucket 4 out of 10 on x) as x", - "exception": "\nTABLESAMPLE(BUCKET x OUT OF y ON colname) is not supported.\n== SQL (line 1, position 17) ==\nselect * from t tablesample(bucket 4 out of 10 on x) as x\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - "output": { - "failure": "invalid argument: found on at 47:49 expected ')'" - } - }, - { - "input": "select * from t tablesample(bucket 4 out of 10) as x", + "input": "select * from t tablesample(bucket 4 out of 10) repeatable (99) as x", "output": { "success": { "query": { @@ -4388,7 +5355,7 @@ "denominator": 10 } }, - "seed": null + "seed": 99 }, "options": [] }, @@ -5347,6 +6314,153 @@ } } }, + { + "input": "select approx from t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "approx" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "select approx, distance, exact, nearest, similarity from t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "approx" + ], + "planId": null, + "isMetadataColumn": false + } + }, + { + "unresolvedAttribute": { + "name": [ + "distance" + ], + "planId": null, + "isMetadataColumn": false + } + }, + { + "unresolvedAttribute": { + "name": [ + "exact" + ], + "planId": null, + "isMetadataColumn": false + } + }, + { + "unresolvedAttribute": { + "name": [ + "nearest" + ], + "planId": null, + "isMetadataColumn": false + } + }, + { + "unresolvedAttribute": { + "name": [ + "similarity" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } + }, + { + "input": "select distance from t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "distance" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } + }, { "input": "select distinct a, b from db.c", "output": { @@ -5402,6 +6516,43 @@ } } }, + { + "input": "select exact from t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "exact" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } + }, { "input": "select from tbl", "output": { @@ -5520,12 +6671,86 @@ "failure": "invalid argument: found 'S' at 18:21 expected '->', '.', '(', '[', '::', 'ESCAPE', 'IS', 'NOT', 'IN', '*', '/', '%', 'DIV', '+', '-', '||', '>>>', '>>', '<<', '&', '^', '|', '!=', '!>', '!<', '==', '=', '>=', '>', '<=>', '<=', '<>', '<', 'BETWEEN', 'LIKE', 'ILIKE', 'RLIKE', 'REGEXP', 'SIMILAR', 'AND', 'OR', '=>', ',', 'RESPECT', 'IGNORE', or ')'" } }, + { + "input": "select nearest from t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "nearest" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } + }, { "input": "select rtrim(trailing 'S' from 'SS abc S'", "exception": "\n[PARSE_SYNTAX_ERROR] Syntax error at or near 'from'. SQLSTATE: 42601 (line 1, pos 26)\n\n== SQL ==\nselect rtrim(trailing 'S' from 'SS abc S'\n--------------------------^^^\n", "output": { "failure": "invalid argument: found 'S' at 22:25 expected '->', '.', '(', '[', '::', 'ESCAPE', 'IS', 'NOT', 'IN', '*', '/', '%', 'DIV', '+', '-', '||', '>>>', '>>', '<<', '&', '^', '|', '!=', '!>', '!<', '==', '=', '>=', '>', '<=>', '<=', '<>', '<', 'BETWEEN', 'LIKE', 'ILIKE', 'RLIKE', 'REGEXP', 'SIMILAR', 'AND', 'OR', '=>', ',', 'RESPECT', 'IGNORE', or ')'" } + }, + { + "input": "select similarity from t", + "output": { + "success": { + "query": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "t" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedAttribute": { + "name": [ + "similarity" + ], + "planId": null, + "isMetadataColumn": false + } + } + ] + }, + "planId": null + } + } + } } ] } diff --git a/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json b/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json index be7b7ff171..749f7cb726 100644 --- a/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json +++ b/crates/sail-spark-connect/tests/gold_data/plan/plan_with.json @@ -376,6 +376,116 @@ } } }, + { + "input": "with CTE1 (select 1), cte1 as (select 2) select * from cte1", + "exception": "\n[DUPLICATED_CTE_NAMES] CTE definition can't have duplicate names: `cte1`. SQLSTATE: 42602\n== SQL (line 1, position 1) ==\nwith CTE1 (select 1), cte1 as (select 2) select * from cte1\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + "output": { + "success": { + "query": { + "withCtes": { + "input": { + "project": { + "input": { + "read": { + "namedTable": { + "name": [ + "cte1" + ], + "temporal": null, + "sample": null, + "options": [] + }, + "isStreaming": false + }, + "planId": null + }, + "expressions": [ + { + "unresolvedStar": { + "target": null, + "planId": null, + "wildcardOptions": { + "ilikePattern": null, + "excludeColumns": null, + "exceptColumns": null, + "replaceColumns": null, + "renameColumns": null + } + } + } + ] + }, + "planId": null + }, + "recursive": false, + "ctes": [ + [ + "CTE1", + { + "tableAlias": { + "input": { + "project": { + "input": { + "empty": { + "produceOneRow": true + }, + "planId": null + }, + "expressions": [ + { + "literal": { + "int32": { + "value": 1 + } + } + } + ] + }, + "planId": null + }, + "name": "CTE1", + "columns": [] + }, + "planId": null + } + ], + [ + "cte1", + { + "tableAlias": { + "input": { + "project": { + "input": { + "empty": { + "produceOneRow": true + }, + "planId": null + }, + "expressions": [ + { + "literal": { + "int32": { + "value": 2 + } + } + } + ] + }, + "planId": null + }, + "name": "cte1", + "columns": [] + }, + "planId": null + } + ] + ] + }, + "planId": null + } + } + } + }, { "input": "with cte1 (select 1) select * from cte1", "output": { diff --git a/docker/dev/Dockerfile b/docker/dev/Dockerfile index ed071dd7ac..0513d399a7 100644 --- a/docker/dev/Dockerfile +++ b/docker/dev/Dockerfile @@ -1,7 +1,7 @@ ARG RUST_VERSION=1.96.0 ARG RUST_PROFILE=dev ARG RUSTFLAGS="" -ARG PYSPARK_VERSION=4.1.1 +ARG PYSPARK_VERSION=4.2.0 FROM python:3.14-slim AS builder diff --git a/docker/quickstart/Dockerfile b/docker/quickstart/Dockerfile index 31ebb1ea2d..9c819f1825 100644 --- a/docker/quickstart/Dockerfile +++ b/docker/quickstart/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.14-slim ARG PYSAIL_VERSION -ARG PYSPARK_VERSION=4.1.1 +ARG PYSPARK_VERSION=4.2.0 RUN python3 -m pip install --no-cache-dir "pysail==${PYSAIL_VERSION}" RUN python3 -m pip install --no-cache-dir "pyspark-client==${PYSPARK_VERSION}" diff --git a/docker/release/Dockerfile b/docker/release/Dockerfile index 8c84b2051b..23b0d841db 100644 --- a/docker/release/Dockerfile +++ b/docker/release/Dockerfile @@ -2,7 +2,7 @@ ARG RELEASE_TAG ARG RUST_VERSION=1.96.0 ARG RUST_PROFILE=release ARG RUSTFLAGS="" -ARG PYSPARK_VERSION=4.1.1 +ARG PYSPARK_VERSION=4.2.0 FROM python:3.14-slim AS builder diff --git a/docs/development/build/python.md b/docs/development/build/python.md index 1f4ec10e8f..6546668d26 100644 --- a/docs/development/build/python.md +++ b/docs/development/build/python.md @@ -128,7 +128,7 @@ The `PYSPARK_SUBMIT_ARGS` environment variable is no longer needed since Spark 4 ```bash env SPARK_REMOTE="local" \ - hatch run test.spark-4.1.1:pytest --pyargs pysail + hatch run test.spark-4.2.0:pytest --pyargs pysail ``` ::: info diff --git a/docs/development/spark-tests/spark-patch.md b/docs/development/spark-tests/spark-patch.md index e878ae6112..c8aa399c02 100644 --- a/docs/development/spark-tests/spark-patch.md +++ b/docs/development/spark-tests/spark-patch.md @@ -11,15 +11,15 @@ Here are the commands that can be helpful for this purpose. ```bash # Apply the patch. # You can now modify the Spark source code. -git -C opt/spark apply ../../scripts/spark-tests/spark-4.1.1.patch +git -C opt/spark apply ../../scripts/spark-tests/spark-4.2.0.patch # Update the Spark patch file with your local modification. git -C opt/spark add . -git -C opt/spark diff --staged -p > scripts/spark-tests/spark-4.1.1.patch +git -C opt/spark diff --staged -p > scripts/spark-tests/spark-4.2.0.patch # Revert the patch. git -C opt/spark reset -git -C opt/spark apply -R ../../scripts/spark-tests/spark-4.1.1.patch +git -C opt/spark apply -R ../../scripts/spark-tests/spark-4.2.0.patch ``` However, note that we should keep the patch minimal. diff --git a/docs/development/spark-tests/spark-setup.md b/docs/development/spark-tests/spark-setup.md index ad0534a189..f29da9bde3 100644 --- a/docs/development/spark-tests/spark-setup.md +++ b/docs/development/spark-tests/spark-setup.md @@ -15,7 +15,7 @@ git clone git@github.com:ibis-project/testing-data.git opt/ibis-testing-data ::: info -You may use options such as `--branch v4.1.1 --depth 1` to get a shallow copy of the Spark repository. But you would need to take care of fetching other branches if you need to work with multiple Spark versions. +You may use options such as `--branch v4.2.0 --depth 1` to get a shallow copy of the Spark repository. But you would need to take care of fetching other branches if you need to work with multiple Spark versions. ::: @@ -26,7 +26,7 @@ Python tests are also included in the patched package. You can choose to run any or all of the commands, depending on the Spark versions you want to test against. ```bash -env SPARK_VERSION=4.1.1 scripts/spark-tests/build-pyspark.sh +env SPARK_VERSION=4.2.0 scripts/spark-tests/build-pyspark.sh env SPARK_VERSION=3.5.7 scripts/spark-tests/build-pyspark.sh ``` diff --git a/docs/development/spark-tests/test-spark.md b/docs/development/spark-tests/test-spark.md index 7c1aef42be..0cf0c09748 100644 --- a/docs/development/spark-tests/test-spark.md +++ b/docs/development/spark-tests/test-spark.md @@ -18,7 +18,7 @@ We have developed custom scripts to run the tests using [pytest](https://docs.py Before running Spark tests, you need to install the [patched PySpark package](./spark-setup) to the `test-spark` matrix environments that supports testing against multiple Spark versions. ```bash -hatch run test-spark.spark-4.1.1:install-pyspark +hatch run test-spark.spark-4.2.0:install-pyspark hatch run test-spark.spark-3.5.7:install-pyspark ``` @@ -39,7 +39,7 @@ hatch run scripts/spark-tests/run-server.sh After running the Spark Connect server, start another terminal and use the following command to run the Spark tests. ```bash -hatch run test-spark.spark-4.1.1:scripts/spark-tests/run-tests.sh +hatch run test-spark.spark-4.2.0:scripts/spark-tests/run-tests.sh ``` The command runs a default set of test suites for Spark Connect. @@ -56,7 +56,7 @@ You can also use `PYTEST_` environment variables to customize the test execution For example, `PYTEST_ADDOPTS="-k "` can be used to run specific tests matching ``. ```bash -hatch run test-spark.spark-4.1.1:env \ +hatch run test-spark.spark-4.2.0:env \ TEST_RUN_NAME=selected \ scripts/spark-tests/run-tests.sh \ --pyargs pyspark.sql.tests.connect -v -k "test_sql" @@ -73,7 +73,7 @@ As a comparison, you can run the tests against the original JVM-based Spark libr by setting the `SPARK_TESTING_REMOTE_PORT` environment variable to an empty string. ```bash -hatch run test-spark.spark-4.1.1:env \ +hatch run test-spark.spark-4.2.0:env \ SPARK_TESTING_REMOTE_PORT= \ scripts/spark-tests/run-tests.sh ``` @@ -86,7 +86,7 @@ The steps above start the server in the `default` Hatch environment. There are a few PySpark UDF tests that would fail in this setup, since they import testing UDFs available only in the **patched** PySpark library (installed in the `test-spark` Hatch environment). There are also a few data-dependent tests that would fail, since the data files in the `python/test_support` directory are only available in the **patched** PySpark library. -Moreover, when the server is started in the `default` environment which has the PySpark 4.1.1 library installed, the tests for PySpark 3.5.7 does not work. +Moreover, when the server is started in the `default` environment which has the PySpark 4.2.0 library installed, the tests for PySpark 3.5.7 does not work. To use the same PySpark library for both the server and the tests, run the server and the tests in the same `test-spark` environment. diff --git a/docs/guide/integrations/mcp-server.md b/docs/guide/integrations/mcp-server.md index 9dbba6bf87..4ecd74fd2a 100644 --- a/docs/guide/integrations/mcp-server.md +++ b/docs/guide/integrations/mcp-server.md @@ -36,7 +36,7 @@ If you have installed Sail via `pip` in a Python virtual environment, the path i Please note that you must install the PySail library with MCP dependencies. ```bash-vue -pip install "pysail[mcp]=={{ libVersion }}" "pyspark-client==4.1.1" +pip install "pysail[mcp]=={{ libVersion }}" "pyspark-client==4.2.0" ``` ::: info diff --git a/docs/introduction/getting-started/index.md b/docs/introduction/getting-started/index.md index 0e81f3145b..c87c95b975 100644 --- a/docs/introduction/getting-started/index.md +++ b/docs/introduction/getting-started/index.md @@ -16,14 +16,14 @@ See the [Installation](/introduction/installation/) page for full installation i ::: code-group -```bash-vue [Spark 4.1 (Client) ] +```bash-vue [Spark 4.2 (Client) ] pip install "pysail=={{ libVersion }}" -pip install "pyspark-client==4.1.1" +pip install "pyspark-client==4.2.0" ``` -```bash-vue [Spark 4.1] +```bash-vue [Spark 4.2] pip install "pysail=={{ libVersion }}" -pip install "pyspark[connect]==4.1.1" +pip install "pyspark[connect]==4.2.0" ``` ```bash-vue [Spark 3.5] diff --git a/pyproject.toml b/pyproject.toml index 1fce98207c..2c37d9bb66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ [project.optional-dependencies] test = [ # The dependencies for testing the installed package. - "pyspark-client==4.1.1", + "pyspark-client==4.2.0", "pandas<3", "duckdb>=1.0,<2", "pytest>=9.1,<10", @@ -93,7 +93,7 @@ python = "3.11" installer = "pip" skip-install = true dependencies = [ - "pyspark[connect]==4.1.1", + "pyspark[connect]==4.2.0", "ibis-framework>=12,<13", "pandas<3", "pytest>=9.1,<10", @@ -173,18 +173,20 @@ dependencies = [ ] [[tool.hatch.envs.test.matrix]] -spark = ["3.5.7", "4.0.1", "4.1.1"] +spark = ["3.5.7", "4.0.1", "4.1.1", "4.2.0"] [tool.hatch.envs.test.overrides] matrix.spark.path = [ { value = ".venvs/test.spark-3.5.7", if = ["3.5.7"] }, { value = ".venvs/test.spark-4.0.1", if = ["4.0.1"] }, { value = ".venvs/test.spark-4.1.1", if = ["4.1.1"] }, + { value = ".venvs/test.spark-4.2.0", if = ["4.2.0"] }, ] matrix.spark.extra-dependencies = [ { value = "pyspark[connect]==3.5.7", if = ["3.5.7"] }, { value = "pyspark[connect]==4.0.1", if = ["4.0.1"] }, { value = "pyspark[connect]==4.1.1", if = ["4.1.1"] }, + { value = "pyspark[connect]==4.2.0", if = ["4.2.0"] }, ] env.CI.installer = "uv" @@ -201,7 +203,7 @@ dependencies = [ ] [[tool.hatch.envs.test-spark.matrix]] -spark = ["3.5.7", "4.1.1"] +spark = ["3.5.7", "4.2.0"] [tool.hatch.envs.test-spark.extra-scripts] install-pyspark = "\"{env:HATCH_UV}\" pip install --force-reinstall 'pyspark[connect] @ opt/spark/python/dist/pyspark-{matrix:spark}.tar.gz' 'pandas<3'" @@ -209,7 +211,7 @@ install-pyspark = "\"{env:HATCH_UV}\" pip install --force-reinstall 'pyspark[con [tool.hatch.envs.test-spark.overrides] matrix.spark.path = [ { value = ".venvs/test-spark.spark-3.5.7", if = ["3.5.7"] }, - { value = ".venvs/test-spark.spark-4.1.1", if = ["4.1.1"] }, + { value = ".venvs/test-spark.spark-4.2.0", if = ["4.2.0"] }, ] env.CI.installer = "uv" diff --git a/python/pysail/tests/spark/catalog/hms/conftest.py b/python/pysail/tests/spark/catalog/hms/conftest.py index e9ba521dda..9902ec2920 100644 --- a/python/pysail/tests/spark/catalog/hms/conftest.py +++ b/python/pysail/tests/spark/catalog/hms/conftest.py @@ -387,8 +387,8 @@ def _wait_for_hms_catalog(remote: str) -> None: "4.1": ("io.delta", "delta-spark_4.1_2.13", "4.1.0"), } if _SPARK_MINOR not in _DELTA_SPARK_COORDINATES: - raise RuntimeError( - f"No Delta Maven coordinate mapping for Spark {_SPARK_VERSION}; " + pytest.skip( + reason=f"No Delta Maven coordinate mapping for Spark {_SPARK_VERSION}; " "add an entry to _DELTA_SPARK_COORDINATES in " "python/pysail/tests/spark/catalog/hms/conftest.py." ) diff --git a/python/pysail/tests/spark/dataframe/test_udt.txt b/python/pysail/tests/spark/dataframe/test_udt.txt index e0260434ce..fa5fc99d02 100644 --- a/python/pysail/tests/spark/dataframe/test_udt.txt +++ b/python/pysail/tests/spark/dataframe/test_udt.txt @@ -1,5 +1,6 @@ >>> from pyspark.ml.linalg import VectorUDT ->>> from pyspark.sql.types import StringType, LongType, StructType, UserDefinedType +>>> from pyspark.sql.types import LongType, StructType +>>> from pysail.tests.spark.dataframe.udt import NamedPythonUDT, UnnamedPythonUDT >>> schema = StructType().add("key", LongType()).add("vec", VectorUDT()) >>> df = spark.createDataFrame(data=[], schema=schema) >>> df @@ -8,14 +9,6 @@ DataFrame[key: bigint, vec: vector] root |-- key: long (nullable = true) |-- vec: vector (nullable = true) ->>> class UnnamedPythonUDT(UserDefinedType): -... @classmethod -... def sqlType(cls): -... return StringType() -... @classmethod -... def module(cls): -... return "__main__" -... >>> schema = StructType().add("key", LongType()).add("a", UnnamedPythonUDT()) >>> df = spark.createDataFrame(data=[], schema=schema) >>> df @@ -24,10 +17,6 @@ DataFrame[key: bigint, a: udt] root |-- key: long (nullable = true) |-- a: pythonuserdefined (nullable = true) ->>> class NamedPythonUDT(UnnamedPythonUDT): -... def simpleString(cls): -... return "foo" -... >>> schema = StructType().add("key", LongType()).add("b", NamedPythonUDT()) >>> df = spark.createDataFrame(data=[], schema=schema) >>> df diff --git a/python/pysail/tests/spark/dataframe/udt.py b/python/pysail/tests/spark/dataframe/udt.py new file mode 100644 index 0000000000..4f868a2b0c --- /dev/null +++ b/python/pysail/tests/spark/dataframe/udt.py @@ -0,0 +1,25 @@ +"""Importable Python UDTs used by doctests. + +Spark resolves Python UDTs by importing their declared module and no longer +unpickles UDT classes embedded in schema metadata. These test UDTs therefore +must be defined in a module instead of directly in the doctest namespace. + +Reference: +""" + +from pyspark.sql.types import StringType, UserDefinedType + + +class UnnamedPythonUDT(UserDefinedType): + @classmethod + def sqlType(cls): # noqa: N802 + return StringType() + + @classmethod + def module(cls): + return __name__ + + +class NamedPythonUDT(UnnamedPythonUDT): + def simpleString(self): # noqa: N802 + return "foo" diff --git a/python/pysail/tests/spark/datasource/test_python.py b/python/pysail/tests/spark/datasource/test_python.py index 6f99154c88..ead765716e 100644 --- a/python/pysail/tests/spark/datasource/test_python.py +++ b/python/pysail/tests/spark/datasource/test_python.py @@ -15,6 +15,7 @@ import pytest from pysail.testing.spark.session import spark_session_factory +from pysail.testing.spark.utils.common import pyspark_version from pysail.testing.spark.utils.sql import escape_sql_string_literal try: @@ -446,7 +447,7 @@ def read(self, partition): # noqa: ARG002 def test_python_default_single_partition(spark): """Test that missing partitions() defaults to a single partition.""" import pyarrow as pa - from pyspark.sql.datasource import DataSource, DataSourceReader + from pyspark.sql.datasource import DataSource, DataSourceReader, InputPartition class DefaultPartitionDataSource(DataSource): """DataSource that relies on the default partitions().""" @@ -463,8 +464,12 @@ def reader(self, schema): # noqa: ARG002 class DefaultPartitionReader(DataSourceReader): def read(self, partition): - if partition is not None: - msg = f"Expected default partition to be None, got {partition!r}" + if pyspark_version() >= (4, 2): + is_default_partition = isinstance(partition, InputPartition) and partition.value is None + else: + is_default_partition = partition is None + if not is_default_partition: + msg = f"Expected default partition, got {partition!r}" raise ValueError(msg) batch = pa.RecordBatch.from_pydict({"id": [1, 2, 3]}, schema=pa.schema([("id", pa.int32())])) yield batch diff --git a/python/pysail/tests/spark/udf/test_udf_kwargs.py b/python/pysail/tests/spark/udf/test_udf_kwargs.py index 4cc0ebba2a..055ac6fedb 100644 --- a/python/pysail/tests/spark/udf/test_udf_kwargs.py +++ b/python/pysail/tests/spark/udf/test_udf_kwargs.py @@ -109,14 +109,23 @@ def test_udtf_kwargs_reversed_order(udtf_concat): def test_udtf_row_output_invalid_scalar_raises_pickle_exception(): - from pyspark.errors.exceptions.connect import PickleException + if pyspark_version() >= (4, 2): + from pyspark.errors.exceptions.connect import PythonException + + exception = PythonException + match = "UDTF_ARROW_DATA_CONVERSION_ERROR" + else: + from pyspark.errors.exceptions.connect import PickleException + + exception = PickleException + match = "PickleException" @udtf(returnType="x: boolean") class RowOutputUDTF: def eval(self): yield (Row(a=0, b=1.1, c=2),) - with pytest.raises(PickleException, match="PickleException"): + with pytest.raises(exception, match=match): RowOutputUDTF().collect() @@ -132,10 +141,30 @@ class ArrowOutputUDTF: def eval(self): yield (1,) - with pytest.raises(PythonException, match="UDTF_ARROW_TYPE_CONVERSION_ERROR"): + with pytest.raises(PythonException, match=r"UDTF_ARROW_(TYPE|DATA)_CONVERSION_ERROR"): ArrowOutputUDTF().collect() +@pytest.mark.parametrize("use_arrow", [False, True]) +def test_arrow_udtf_lateral_view_uses_argument_schema(spark, use_arrow): + """Passthrough columns must not shift Arrow UDTF argument conversions.""" + + @udtf(returnType="output: string", useArrow=use_arrow) + class EchoUDTF: + def eval(self, value): + yield (value,) + + function_name = f"echo_udtf_{'' if use_arrow else 'not_'}use_arrow" + spark.udtf.register(function_name, EchoUDTF) + + df = spark.sql( + f"""SELECT passthrough, value, output + FROM VALUES (X'706173737468726f756768', 'value') AS input(passthrough, value) + LATERAL VIEW {function_name}(value) AS output""" # noqa: S608 + ) + assert df.collect() == [Row(passthrough=b"passthrough", value="value", output="value")] + + @pytest.mark.skipif( pyspark_version() < (4, 1), reason="UDTF analyze tests require PySpark 4.1+", diff --git a/scripts/spark-gold-data/bootstrap.sh b/scripts/spark-gold-data/bootstrap.sh index 17bffdeb32..0cf99dbb64 100755 --- a/scripts/spark-gold-data/bootstrap.sh +++ b/scripts/spark-gold-data/bootstrap.sh @@ -15,7 +15,7 @@ output_path="${project_path}/crates/sail-spark-connect/tests/gold_data" source "${project_path}/scripts/shell-tools/git-patch.sh" -apply_git_patch "${project_path}"/opt/spark "v4.1.1" "${scripts_path}/spark-4.1.1.patch" +apply_git_patch "${project_path}"/opt/spark "v4.2.0" "${scripts_path}/spark-4.2.0.patch" cd "${project_path}"/opt/spark diff --git a/scripts/spark-gold-data/spark-4.1.1.patch b/scripts/spark-gold-data/spark-4.2.0.patch similarity index 90% rename from scripts/spark-gold-data/spark-4.1.1.patch rename to scripts/spark-gold-data/spark-4.2.0.patch index 04de941b1d..95ca8ababe 100644 --- a/scripts/spark-gold-data/spark-4.1.1.patch +++ b/scripts/spark-gold-data/spark-4.2.0.patch @@ -1,8 +1,8 @@ diff --git a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala -index 810bdabebb3..079df78998d 100644 +index 3fac57dbe5d..38584fb1b8c 100644 --- a/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala +++ b/common/utils/src/main/scala/org/apache/spark/internal/Logging.scala -@@ -408,6 +408,8 @@ private[spark] object Logging { +@@ -413,6 +413,8 @@ private[spark] object Logging { @volatile private[spark] var sparkShellThresholdLevel: Level = null @volatile private[spark] var setLogLevelPrinted: Boolean = false @@ -11,13 +11,13 @@ index 810bdabebb3..079df78998d 100644 val initLock = new Object() try { // We use reflection here to handle the case where users remove the -diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala -index 3b0ec30ee86..bdb6c5e200b 100644 ---- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala -+++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala -@@ -89,15 +89,54 @@ abstract class SparkFunSuite - - protected val regenerateGoldenFiles: Boolean = System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1" +diff --git a/core/src/test/scala/org/apache/spark/SparkTestSuite.scala b/core/src/test/scala/org/apache/spark/SparkTestSuite.scala +index 10504684be9..f3bb4a8f15e 100644 +--- a/core/src/test/scala/org/apache/spark/SparkTestSuite.scala ++++ b/core/src/test/scala/org/apache/spark/SparkTestSuite.scala +@@ -91,15 +91,54 @@ trait SparkTestSuite + protected def regenerateGoldenFiles: Boolean = + System.getenv("SPARK_GENERATE_GOLDEN_FILES") == "1" + /** + * The environment variable for specifying the directory to write test suite data. @@ -34,7 +34,7 @@ index 3b0ec30ee86..bdb6c5e200b 100644 + ) + } + val className = this.getClass.getName.stripSuffix("$").split('.').last -+ new java.io.File(outputDir, s"$className.jsonl") ++ new File(outputDir, s"$className.jsonl") + } + + /** Write test data to the output file. */ @@ -71,7 +71,7 @@ index 3b0ec30ee86..bdb6c5e200b 100644 // Avoid leaking map entries in tests that use accumulators without SparkContext AccumulatorContext.clear() diff --git a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala -index 32270df0a98..a52bb7920f6 100644 +index c997a102a70..93403dc7454 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/catalyst/parser/parsers.scala @@ -35,13 +35,25 @@ import org.apache.spark.sql.errors.QueryParsingErrors @@ -111,10 +111,10 @@ index 32270df0a98..a52bb7920f6 100644 } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala -index 216136d8a7c..35d9aa9c7c4 100644 +index 29bf924f244..dee33a596e2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AbstractSqlParser.scala -@@ -27,6 +27,7 @@ import org.apache.spark.sql.errors.QueryParsingErrors +@@ -28,6 +28,7 @@ import org.apache.spark.sql.errors.QueryParsingErrors import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType @@ -122,7 +122,7 @@ index 216136d8a7c..35d9aa9c7c4 100644 /** * Base class for all ANTLR4 [[ParserInterface]] implementations. */ -@@ -35,7 +36,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { +@@ -36,7 +37,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { /** Creates Expression for a given SQL string. */ override def parseExpression(sqlText: String): Expression = @@ -131,7 +131,7 @@ index 216136d8a7c..35d9aa9c7c4 100644 val ctx = parser.singleExpression() withErrorHandling(ctx, Some(sqlText)) { astBuilder.visitSingleExpression(ctx) -@@ -44,7 +45,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { +@@ -45,7 +46,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { /** Creates TableIdentifier for a given SQL string. */ override def parseTableIdentifier(sqlText: String): TableIdentifier = @@ -140,7 +140,7 @@ index 216136d8a7c..35d9aa9c7c4 100644 val ctx = parser.singleTableIdentifier() withErrorHandling(ctx, Some(sqlText)) { astBuilder.visitSingleTableIdentifier(ctx) -@@ -53,7 +54,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { +@@ -54,7 +55,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { /** Creates FunctionIdentifier for a given SQL string. */ override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = { @@ -149,7 +149,7 @@ index 216136d8a7c..35d9aa9c7c4 100644 val ctx = parser.singleFunctionIdentifier() withErrorHandling(ctx, Some(sqlText)) { astBuilder.visitSingleFunctionIdentifier(ctx) -@@ -63,7 +64,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { +@@ -64,7 +65,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { /** Creates a multi-part identifier for a given SQL string */ override def parseMultipartIdentifier(sqlText: String): Seq[String] = { @@ -158,7 +158,7 @@ index 216136d8a7c..35d9aa9c7c4 100644 val ctx = parser.singleMultipartIdentifier() withErrorHandling(ctx, Some(sqlText)) { astBuilder.visitSingleMultipartIdentifier(ctx) -@@ -73,7 +74,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { +@@ -74,7 +75,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { /** Creates LogicalPlan for a given SQL string of query. */ override def parseQuery(sqlText: String): LogicalPlan = @@ -167,7 +167,7 @@ index 216136d8a7c..35d9aa9c7c4 100644 if (!SQLConf.get.getConf(SQLConf.LEGACY_PARSE_QUERY_WITHOUT_EOF)) { val ctx = parser.singleQuery() -@@ -90,7 +91,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { +@@ -91,7 +92,7 @@ abstract class AbstractSqlParser extends AbstractParser with ParserInterface { } /** Creates LogicalPlan for a given SQL string. */ diff --git a/scripts/spark-tests/spark-4.1.1.patch b/scripts/spark-tests/spark-4.2.0.patch similarity index 91% rename from scripts/spark-tests/spark-4.1.1.patch rename to scripts/spark-tests/spark-4.2.0.patch index 28e550b647..ae1e1dae3a 100644 --- a/scripts/spark-tests/spark-4.1.1.patch +++ b/scripts/spark-tests/spark-4.2.0.patch @@ -1,8 +1,8 @@ diff --git a/python/packaging/classic/setup.py b/python/packaging/classic/setup.py -index 54ec4abe3be..28edac8f441 100755 +index 122c4ca6d4b..8a17c9d3878 100755 --- a/python/packaging/classic/setup.py +++ b/python/packaging/classic/setup.py -@@ -206,6 +206,9 @@ try: +@@ -214,6 +214,9 @@ try: copyfile("pyspark/shell.py", "pyspark/python/pyspark/shell.py") if in_spark: @@ -12,7 +12,7 @@ index 54ec4abe3be..28edac8f441 100755 # !!HACK ALTERT!! # `setup.py` has to be located with the same directory with the package. # Therefore, we copy the current file, and place it at `spark/python` directory. -@@ -308,6 +311,7 @@ try: +@@ -316,6 +319,7 @@ try: "pyspark.pandas.typedef", "pyspark.pandas.usage_logging", "pyspark.pipelines", @@ -20,7 +20,7 @@ index 54ec4abe3be..28edac8f441 100755 "pyspark.python.pyspark", "pyspark.python.lib", "pyspark.testing", -@@ -318,6 +322,62 @@ try: +@@ -326,6 +330,64 @@ try: "pyspark.errors.exceptions", "pyspark.examples.src.main.python", "pyspark.logger", @@ -64,6 +64,7 @@ index 54ec4abe3be..28edac8f441 100755 + "pyspark.resource.tests", + "pyspark.sql.tests", + "pyspark.sql.tests.arrow", ++ "pyspark.sql.tests.coercion", + "pyspark.sql.tests.connect", + "pyspark.sql.tests.connect.arrow", + "pyspark.sql.tests.connect.client", @@ -76,14 +77,15 @@ index 54ec4abe3be..28edac8f441 100755 + "pyspark.sql.tests.pandas.streaming", + "pyspark.sql.tests.plot", + "pyspark.sql.tests.streaming", -+ "pyspark.sql.tests.udf_type_tests", + "pyspark.streaming.tests", + "pyspark.testing.tests", + "pyspark.tests", ++ "pyspark.tests.upstream", ++ "pyspark.tests.upstream.pyarrow", ], include_package_data=True, package_dir={ -@@ -338,10 +398,29 @@ try: +@@ -346,10 +408,29 @@ try: "start-history-server.sh", "stop-history-server.sh", ], @@ -106,7 +108,7 @@ index 54ec4abe3be..28edac8f441 100755 "pyspark.python.lib": ["*.zip"], - "pyspark.data": ["*.txt", "*.data"], + "pyspark.data": ["*.txt", "*.data", "artifact-tests/*.jar"], - "pyspark.licenses": ["*.txt"], + "pyspark.licenses": ["*"], "pyspark.examples.src.main.python": ["*.py", "*/*.py"], + "pyspark.ml.tests": ["typing/*.yml"], + "pyspark.sql.tests": ["typing/*.yml"], @@ -115,10 +117,10 @@ index 54ec4abe3be..28edac8f441 100755 scripts=scripts, license="Apache-2.0", diff --git a/python/pyspark/sql/connect/client/core.py b/python/pyspark/sql/connect/client/core.py -index 80d83c69c45..2625eae10ad 100644 +index b2e30a8325d..b99eed69504 100644 --- a/python/pyspark/sql/connect/client/core.py +++ b/python/pyspark/sql/connect/client/core.py -@@ -344,6 +344,10 @@ class DefaultChannelBuilder(ChannelBuilder): +@@ -352,6 +352,10 @@ class DefaultChannelBuilder(ChannelBuilder): session = PySparkSession._instantiatedSession if session is not None: @@ -129,17 +131,8 @@ index 80d83c69c45..2625eae10ad 100644 jvm = PySparkSession._instantiatedSession._jvm # type: ignore[union-attr] return getattr( getattr( -@@ -1249,6 +1253,8 @@ class SparkConnectClient(object): - """ - Close the channel. - """ -+ if self._closed: -+ return - ExecutePlanResponseReattachableIterator.shutdown() - self._channel.close() - self._closed = True diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py -index b789d7919c9..a4ec3bc4bf7 100755 +index 516c3ad3751..72aac8bfe73 100755 --- a/python/pyspark/sql/tests/connect/test_connect_basic.py +++ b/python/pyspark/sql/tests/connect/test_connect_basic.py @@ -82,6 +82,11 @@ class SparkConnectSQLTestCase(ReusedMixedTestCase, PandasOnSparkTestUtils): @@ -165,7 +158,7 @@ index b789d7919c9..a4ec3bc4bf7 100755 @classmethod diff --git a/python/pyspark/testing/objects.py b/python/pyspark/testing/objects.py -index 5b97664afbd..b9deeef91c8 100644 +index 2acbd0ba1b4..3232d6c2478 100644 --- a/python/pyspark/testing/objects.py +++ b/python/pyspark/testing/objects.py @@ -45,7 +45,7 @@ class ExamplePointUDT(UserDefinedType):