chore(query): Set the default value of enable_experimental_virtual_column as true

Author: b41sh

Cargo.lock

@@ -657,6 +657,7 @@ backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "7226
 color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
 deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "9954bff" }
 display-more = { git = "https://github.com/databendlabs/display-more", tag = "v0.2.0" }
+jsonb = { git = "https://github.com/b41sh/jsonb", rev = "ef5482fe9c07d87daa84115b86fd9af55dea277c" }
 map-api = { git = "https://github.com/databendlabs/map-api", tag = "v0.4.2" }
 openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.9" }
 orc-rust = { git = "https://github.com/datafuse-extras/orc-rust", rev = "d82aa6d" }

Cargo.toml

@@ -484,6 +484,13 @@ impl FromToProtoEnum for ex::VariantDataType {
             variant_data_type::Dt::ArrayT(dt) => {
                 ex::VariantDataType::Array(Box::new(ex::VariantDataType::from_pb_enum(*dt)?))
             }
+            variant_data_type::Dt::DecimalT(dt) => {
+                ex::VariantDataType::Decimal(ex::types::decimal::DecimalDataType::from_pb(dt)?)
+            }
+            variant_data_type::Dt::BinaryT(_) => ex::VariantDataType::Binary,
+            variant_data_type::Dt::DateT(_) => ex::VariantDataType::Date,
+            variant_data_type::Dt::TimestampT(_) => ex::VariantDataType::Timestamp,
+            variant_data_type::Dt::IntervalT(_) => ex::VariantDataType::Interval,
         })
     }
 
@@ -498,6 +505,14 @@ impl FromToProtoEnum for ex::VariantDataType {
             VariantDataType::Array(dt) => {
                 pb::variant_data_type::Dt::ArrayT(Box::new(dt.to_pb_enum()?))
             }
+            VariantDataType::Decimal(n) => {
+                let x = n.to_pb()?;
+                pb::variant_data_type::Dt::DecimalT(x)
+            }
+            VariantDataType::Binary => pb::variant_data_type::Dt::BinaryT(pb::Empty {}),
+            VariantDataType::Date => pb::variant_data_type::Dt::DateT(pb::Empty {}),
+            VariantDataType::Timestamp => pb::variant_data_type::Dt::TimestampT(pb::Empty {}),
+            VariantDataType::Interval => pb::variant_data_type::Dt::IntervalT(pb::Empty {}),
         };
 
         Ok(pb::VariantDataType { dt: Some(dt) })

src/meta/proto-conv/src/schema_from_to_protobuf_impl.rs

@@ -32,6 +32,11 @@ message VariantDataType {
       Empty float64_t = 5;
       Empty string_t = 6;
       VariantDataType array_t = 7;
+      Decimal decimal_t = 8;
+      Empty binary_t = 9;
+      Empty date_t = 10;
+      Empty timestamp_t = 11;
+      Empty interval_t = 12;
   }
 }
 

src/meta/protos/proto/virtual_schema.proto

@@ -34,10 +34,6 @@ async fn test_fuse_do_refresh_virtual_column() -> Result<()> {
         .default_session()
         .get_settings()
         .set_data_retention_time_in_days(0)?;
-    fixture
-        .default_session()
-        .get_settings()
-        .set_enable_experimental_virtual_column(1)?;
     fixture.create_default_database().await?;
     fixture.create_variant_table().await?;
 

src/query/ee/tests/it/storages/fuse/operations/virtual_columns.rs

@@ -16,6 +16,8 @@ use std::str::FromStr;
 
 use databend_common_base::base::tokio;
 use databend_common_exception::Result;
+use databend_common_expression::types::DecimalDataType;
+use databend_common_expression::types::DecimalSize;
 use databend_common_expression::types::Int32Type;
 use databend_common_expression::types::VariantType;
 use databend_common_expression::ColumnId;
@@ -32,11 +34,6 @@ use jsonb::OwnedJsonb;
 #[tokio::test(flavor = "multi_thread")]
 async fn test_virtual_column_builder() -> Result<()> {
     let fixture = TestFixture::setup_with_custom(EESetup::new()).await?;
-
-    fixture
-        .default_session()
-        .get_settings()
-        .set_enable_experimental_virtual_column(1)?;
     fixture.create_default_database().await?;
     fixture.create_variant_table().await?;
 
@@ -247,7 +244,10 @@ async fn test_virtual_column_builder() -> Result<()> {
         "['geo']['lat']",
     )
     .unwrap();
-    assert_eq!(meta_geo_lat.data_type, VariantDataType::Jsonb);
+    assert_eq!(
+        meta_geo_lat.data_type,
+        VariantDataType::Decimal(DecimalDataType::from(DecimalSize::new_unchecked(18, 1)))
+    );
 
     let entries = vec![
         Int32Type::from_data(vec![1, 2, 3, 4, 5, 6, 7, 8]).into(),
@@ -316,11 +316,6 @@ async fn test_virtual_column_builder() -> Result<()> {
 #[tokio::test(flavor = "multi_thread")]
 async fn test_virtual_column_builder_stream_write() -> Result<()> {
     let fixture = TestFixture::setup_with_custom(EESetup::new()).await?;
-
-    fixture
-        .default_session()
-        .get_settings()
-        .set_enable_experimental_virtual_column(1)?;
     fixture.create_default_database().await?;
     fixture.create_variant_table().await?;
 

src/query/ee/tests/it/storages/fuse/operations/virtual_columns_builder.rs

@@ -149,7 +149,7 @@ pub struct DataSchema {
     pub(crate) metadata: BTreeMap<String, String>,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Serialize, Deserialize)]
+#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
 pub enum VariantDataType {
     Jsonb,
     Boolean,
@@ -158,6 +158,11 @@ pub enum VariantDataType {
     Float64,
     String,
     Array(Box<VariantDataType>),
+    Decimal(DecimalDataType),
+    Binary,
+    Date,
+    Timestamp,
+    Interval,
 }
 
 #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)]

src/query/expression/src/schema.rs

@@ -1541,6 +1541,8 @@ impl Decimal for i256 {
     Copy,
     PartialEq,
     Eq,
+    PartialOrd,
+    Ord,
     Hash,
     Serialize,
     Deserialize,

src/query/expression/src/types/decimal.rs

@@ -734,6 +734,11 @@ impl Display for VariantDataType {
             VariantDataType::Float64 => write!(f, "Float64"),
             VariantDataType::String => write!(f, "String"),
             VariantDataType::Array(inner) => write!(f, "Array({inner})"),
+            VariantDataType::Decimal(inner) => write!(f, "Decimal({inner})"),
+            VariantDataType::Binary => write!(f, "Binary"),
+            VariantDataType::Date => write!(f, "Date"),
+            VariantDataType::Timestamp => write!(f, "Timestamp"),
+            VariantDataType::Interval => write!(f, "Interval"),
         }
     }
 }

src/query/expression/src/utils/display.rs

@@ -1079,13 +1079,6 @@ impl DefaultSettings {
                     scope: SettingScope::Both,
                     range: Some(SettingRange::String(vec!["None".into(), "LZ4".into(), "ZSTD".into()])),
                 }),
-                ("enable_refresh_virtual_column_after_write", DefaultSettingValue {
-                    value: UserSettingValue::UInt64(1),
-                    desc: "Refresh virtual column after new data written",
-                    mode: SettingMode::Both,
-                    scope: SettingScope::Both,
-                    range: Some(SettingRange::Numeric(0..=1)),
-                }),
                 ("enable_refresh_aggregating_index_after_write", DefaultSettingValue {
                     value: UserSettingValue::UInt64(1),
                     desc: "Refresh aggregating index after new data written",
@@ -1446,7 +1439,7 @@ impl DefaultSettings {
                     range: Some(SettingRange::Numeric(1..=1024)),
                 }),
                 ("enable_experimental_virtual_column", DefaultSettingValue {
-                    value: UserSettingValue::UInt64(0),
+                    value: UserSettingValue::UInt64(1),
                     desc: "Enables experimental virtual column",
                     mode: SettingMode::Both,
                     scope: SettingScope::Both,