From 99755920e26a242aa10044c62438d6f2eaa27a97 Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Mon, 25 Dec 2023 11:23:00 +0800
Subject: [PATCH 001/299] [fix](nereids)subquery unnest need handle subquery in
 Not expr correnctly (#28713)

---
 .../rules/analysis/SubqueryToApply.java       | 35 ++++++-
 .../nereids_p0/subquery/test_subquery.out     |  4 +
 .../nereids_p0/subquery/test_subquery.groovy  | 93 +++++++++++++++++++
 3 files changed, 127 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
index c236e1b325d206..5f55f86cc91257 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.rules.RuleType;
 import org.apache.doris.nereids.trees.TreeNode;
 import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.BinaryOperator;
 import org.apache.doris.nereids.trees.expressions.Exists;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -30,6 +31,7 @@
 import org.apache.doris.nereids.trees.expressions.ListQuery;
 import org.apache.doris.nereids.trees.expressions.MarkJoinSlotReference;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Not;
 import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.ScalarSubquery;
 import org.apache.doris.nereids.trees.expressions.Slot;
@@ -38,7 +40,6 @@
 import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
 import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.algebra.Aggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalApply;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -73,7 +74,8 @@ public List<Rule> buildRules() {
             RuleType.FILTER_SUBQUERY_TO_APPLY.build(
                 logicalFilter().thenApply(ctx -> {
                     LogicalFilter<Plan> filter = ctx.root;
-
+                    boolean shouldOutputMarkJoinSlot = filter.getConjuncts().stream()
+                            .anyMatch(expr -> shouldOutputMarkJoinSlot(expr, SearchState.SearchNot));
                     ImmutableList<Set<SubqueryExpr>> subqueryExprsList = filter.getConjuncts().stream()
                             .<Set<SubqueryExpr>>map(e -> e.collect(SubqueryToApply::canConvertToSupply))
                             .collect(ImmutableList.toImmutableList());
@@ -98,7 +100,7 @@ public List<Rule> buildRules() {
                         // first step: Replace the subquery of predicate in LogicalFilter
                         // second step: Replace subquery with LogicalApply
                         ReplaceSubquery replaceSubquery = new ReplaceSubquery(
-                                ctx.statementContext, false);
+                                ctx.statementContext, shouldOutputMarkJoinSlot);
                         SubqueryContext context = new SubqueryContext(subqueryExprs);
                         Expression conjunct = replaceSubquery.replace(oldConjuncts.get(i), context);
 
@@ -343,7 +345,7 @@ private boolean nonMarkJoinExistsWithAgg(SubqueryExpr exists,
                 && hasTopLevelAggWithoutGroupBy(exists.getQueryPlan());
     }
 
-    private boolean hasTopLevelAggWithoutGroupBy(Plan plan) {
+    private static boolean hasTopLevelAggWithoutGroupBy(Plan plan) {
         if (plan instanceof LogicalAggregate) {
             return ((LogicalAggregate) plan).getGroupByExpressions().isEmpty();
         } else if (plan instanceof LogicalProject || plan instanceof LogicalSort) {
@@ -427,7 +429,7 @@ public Expression visitExistsSubquery(Exists exists, SubqueryContext context) {
             // it will always consider the returned result to be true
             boolean needCreateMarkJoinSlot = isMarkJoin || shouldOutputMarkJoinSlot;
             MarkJoinSlotReference markJoinSlotReference = null;
-            if (exists.getQueryPlan().anyMatch(Aggregate.class::isInstance) && needCreateMarkJoinSlot) {
+            if (hasTopLevelAggWithoutGroupBy(exists.getQueryPlan()) && needCreateMarkJoinSlot) {
                 markJoinSlotReference =
                         new MarkJoinSlotReference(statementContext.generateColumnName(), true);
             } else if (needCreateMarkJoinSlot) {
@@ -505,4 +507,27 @@ private void setSubqueryToMarkJoinSlot(SubqueryExpr subquery,
 
     }
 
+    private enum SearchState {
+        SearchNot,
+        SearchAnd,
+        SearchExistsOrInSubquery
+    }
+
+    private boolean shouldOutputMarkJoinSlot(Expression expr, SearchState searchState) {
+        if (searchState == SearchState.SearchNot && expr instanceof Not) {
+            if (shouldOutputMarkJoinSlot(((Not) expr).child(), SearchState.SearchAnd)) {
+                return true;
+            }
+        } else if (searchState == SearchState.SearchAnd && expr instanceof And) {
+            for (Expression child : expr.children()) {
+                if (shouldOutputMarkJoinSlot(child, SearchState.SearchExistsOrInSubquery)) {
+                    return true;
+                }
+            }
+        } else if (searchState == SearchState.SearchExistsOrInSubquery
+                && (expr instanceof InSubquery || expr instanceof Exists)) {
+            return true;
+        }
+        return false;
+    }
 }
diff --git a/regression-test/data/nereids_p0/subquery/test_subquery.out b/regression-test/data/nereids_p0/subquery/test_subquery.out
index 2344cedddd5077..004aa0bcad05bf 100644
--- a/regression-test/data/nereids_p0/subquery/test_subquery.out
+++ b/regression-test/data/nereids_p0/subquery/test_subquery.out
@@ -26,3 +26,7 @@ true	15	1992	3021	11011920	0.000	true	9999-12-12	2015-04-02T00:00		3.141592653	2
 -- !sql_mark_join --
 1
 
+-- !select_sub --
+1	9
+2	\N
+
diff --git a/regression-test/suites/nereids_p0/subquery/test_subquery.groovy b/regression-test/suites/nereids_p0/subquery/test_subquery.groovy
index dfb79c771d8eac..36a863a211d351 100644
--- a/regression-test/suites/nereids_p0/subquery/test_subquery.groovy
+++ b/regression-test/suites/nereids_p0/subquery/test_subquery.groovy
@@ -155,4 +155,97 @@ suite("test_subquery") {
     qt_sql_mark_join """with A as (select count(*) n1 from test_one_row_relation where exists (select 1 from test_one_row_relation t where t.user_id = test_one_row_relation.user_id) or 1 = 1) select * from A;"""
 
     sql """drop table if exists test_one_row_relation;"""
+
+    sql """drop table if exists subquery_test_t1;"""
+    sql """drop table if exists subquery_test_t2;"""
+    sql """create table subquery_test_t1 (
+                id int
+            )
+            UNIQUE KEY (`id`)
+            DISTRIBUTED BY HASH(`id`) BUCKETS 1
+            PROPERTIES ("replication_allocation" = "tag.location.default: 1");"""
+    sql """create table subquery_test_t2 (
+                id int
+            )
+            UNIQUE KEY (`id`)
+            DISTRIBUTED BY HASH(`id`) BUCKETS 1
+            PROPERTIES ("replication_allocation" = "tag.location.default: 1");"""
+
+    explain {
+        sql("""analyzed plan select subquery_test_t1.id from subquery_test_t1
+                where
+                    not (
+                            exists(select 1 from subquery_test_t2 where subquery_test_t1.id = subquery_test_t2.id and subquery_test_t2.id = 5) 
+                            and
+                            exists(select 1 from subquery_test_t2 where subquery_test_t1.id = subquery_test_t2.id and subquery_test_t2.id = 6)
+                        ); """)
+        contains("isMarkJoin=true")
+    }
+    explain {
+        sql("""analyzed plan select subquery_test_t1.id from subquery_test_t1
+                where
+                    not ( 
+                            subquery_test_t1.id > 10 
+                            and 
+                            exists(select 1 from subquery_test_t2 where subquery_test_t1.id = subquery_test_t2.id and subquery_test_t2.id = 6)
+                        );""")
+        contains("isMarkJoin=true")
+    }
+    explain {
+        sql("""analyzed plan select subquery_test_t1.id from subquery_test_t1
+                where
+                    not ( 
+                            subquery_test_t1.id > 10 
+                            and 
+                            subquery_test_t1.id in (select 1 from subquery_test_t2 where subquery_test_t1.id = subquery_test_t2.id and subquery_test_t2.id = 6)
+                        );  """)
+        contains("isMarkJoin=true")
+    }
+    explain {
+        sql("""analyzed plan select subquery_test_t1.id from subquery_test_t1
+                where
+                    not ( 
+                            subquery_test_t1.id > 10 
+                            and 
+                            subquery_test_t1.id in (select 1 from subquery_test_t2 where subquery_test_t1.id = subquery_test_t2.id and subquery_test_t2.id = 6)
+                        ); """)
+        contains("isMarkJoin=true")
+    }
+    explain {
+        sql("""analyzed plan select subquery_test_t1.id from subquery_test_t1
+                where
+                    not ( 
+                            subquery_test_t1.id > 10 
+                            and 
+                            ( subquery_test_t1.id < 100 or subquery_test_t1.id in (select 1 from subquery_test_t2 where subquery_test_t1.id = subquery_test_t2.id and subquery_test_t2.id = 6) )
+                        ); """)
+        contains("isMarkJoin=true")
+    }
+    explain {
+        sql("""analyzed plan select subquery_test_t1.id from subquery_test_t1
+                where
+                    not ( 
+                            subquery_test_t1.id > 10 
+                            and 
+                            ( subquery_test_t1.id < 100 or case when subquery_test_t1.id in (select 1 from subquery_test_t2 where subquery_test_t1.id = subquery_test_t2.id and subquery_test_t2.id = 6) then 1 else 0 end )
+                        );""")
+        contains("isMarkJoin=true")
+    }
+
+    sql """drop table if exists table_23_undef_undef"""
+    sql """create table table_23_undef_undef (`pk` int,`col_int_undef_signed` int  ,`col_varchar_10__undef_signed` varchar(10)  ,`col_varchar_1024__undef_signed` varchar(1024)  ) engine=olap distributed by hash(pk) buckets 10 properties(    'replication_num' = '1');"""
+    sql """drop table if exists table_20_undef_undef"""
+    sql """create table table_20_undef_undef (`pk` int,`col_int_undef_signed` int  ,`col_varchar_10__undef_signed` varchar(10)  ,`col_varchar_1024__undef_signed` varchar(1024)  ) engine=olap distributed by hash(pk) buckets 10 properties(    'replication_num' = '1');"""
+    sql """drop table if exists table_9_undef_undef"""
+    sql """create table table_9_undef_undef (`pk` int,`col_int_undef_signed` int  ,`col_varchar_10__undef_signed` varchar(10)  ,`col_varchar_1024__undef_signed` varchar(1024)  ) engine=olap distributed by hash(pk) buckets 10 properties(    'replication_num' = '1');"""
+
+    sql """insert into table_23_undef_undef values (0,0,'t','p'),(1,6,'q',"really"),(2,3,'p',"of"),(3,null,"he",'k'),(4,8,"this","don't"),(5,6,"see","this"),(6,5,'s','q'),(7,null,'o','j'),(8,9,'l',"could"),(9,null,"one",'l'),(10,7,"can't",'f'),(11,2,"going","not"),(12,null,'g','r'),(13,3,"ok",'s'),(14,6,"she",'k'),(15,null,"she",'p'),(16,8,"what","him"),(17,null,"from","to"),(18,5,"so","up"),(19,null,"my","is"),(20,null,'h',"see"),(21,null,"as","to"),(22,0,"know","the");"""
+    sql """insert into table_20_undef_undef values (0,null,'r','x'),(1,null,'m',"say"),(2,2,"mean",'h'),(3,null,'n','b'),(4,8,"do","do"),(5,9,'h',"were"),(6,null,"was","one"),(7,2,'o',"she"),(8,0,"who","me"),(9,null,'n',"that"),(10,null,"will",'l'),(11,4,'m',"if"),(12,5,"the","got"),(13,null,"why",'f'),(14,0,"of","for"),(15,null,"or","ok"),(16,null,'c','u'),(17,3,'f','c'),(18,null,"see",'f'),(19,2,'f','z');"""
+    sql """insert into table_9_undef_undef values (0,3,"his",'g'),(1,8,'p','n'),(2,null,"get","got"),(3,3,'r','r'),(4,null,"or","get"),(5,0,'j',"yeah"),(6,null,'w','x'),(7,8,'q',"for"),(8,3,'p',"that");"""
+
+    qt_select_sub"""SELECT DISTINCT alias1.`pk` AS field1,     alias2.`col_int_undef_signed` AS field2 FROM table_23_undef_undef AS alias1,     table_20_undef_undef AS alias2 WHERE (         EXISTS (             SELECT DISTINCT SQ1_alias1.`col_varchar_10__undef_signed` AS SQ1_field1             FROM table_9_undef_undef AS SQ1_alias1             WHERE SQ1_alias1.`col_varchar_10__undef_signed` = alias1.`col_varchar_10__undef_signed`             )         )     OR alias1.`col_varchar_1024__undef_signed` = "TmxRwcNZHC"     AND (         alias1.`col_varchar_10__undef_signed` <> "rnZeukOcuM"         AND alias2.`col_varchar_10__undef_signed` != "dbPAEpzstk"         ) ORDER BY alias1.`pk`,     field1,     field2 LIMIT 2 OFFSET 7; """
+    sql """drop table if exists table_23_undef_undef"""
+    sql """drop table if exists table_20_undef_undef"""
+    sql """drop table if exists table_9_undef_undef"""
+
 }

From c53611dcb3906637648b7e4bd77e01e3b5399388 Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Mon, 25 Dec 2023 11:23:27 +0800
Subject: [PATCH 002/299] [feature](nereids)support decimalv2 (#28726)

---
 .../executable/NumericArithmetic.java         |  3 +
 .../expressions/literal/DecimalLiteral.java   |  5 +-
 .../expressions/literal/DecimalV3Literal.java | 28 +++++-
 .../apache/doris/nereids/types/DataType.java  | 18 ++++
 .../doris/nereids/types/DecimalV2Type.java    | 52 +++++++----
 .../nereids/parser/NereidsParserTest.java     |  8 ++
 .../nereids_p0/datatype/test_decimalv2.out    | 52 +++++++++++
 .../decimalv2/test_decimalv2_overflow.groovy  |  2 +-
 .../nereids_p0/datatype/test_decimalv2.groovy | 88 +++++++++++++++++++
 9 files changed, 237 insertions(+), 19 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/datatype/test_decimalv2.out
 create mode 100644 regression-test/suites/nereids_p0/datatype/test_decimalv2.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java
index 228c8be3d09d0d..0aad2849823ed6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/executable/NumericArithmetic.java
@@ -570,6 +570,9 @@ public static Expression divideDouble(DoubleLiteral first, DoubleLiteral second)
         return new DoubleLiteral(result);
     }
 
+    /**
+     * Executable arithmetic functions divide
+     */
     @ExecFunction(name = "divide", argTypes = {"DECIMAL", "DECIMAL"}, returnType = "DECIMAL")
     public static Expression divideDecimal(DecimalLiteral first, DecimalLiteral second) {
         if (first.getValue().compareTo(BigDecimal.ZERO) == 0) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteral.java
index 472ea3a5dc7a7c..711673cc2ff672 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalLiteral.java
@@ -74,13 +74,14 @@ public double getDouble() {
     /**
      * check precision and scale is enough for value.
      */
-    public static void checkPrecisionAndScale(int precision, int scale, BigDecimal value) throws AnalysisException {
+    private static void checkPrecisionAndScale(int precision, int scale, BigDecimal value) throws AnalysisException {
         Preconditions.checkNotNull(value);
         int realPrecision = value.precision();
         int realScale = value.scale();
         boolean valid = true;
         if (precision != -1 && scale != -1) {
-            if (precision < realPrecision || scale < realScale) {
+            if (precision < realPrecision || scale < realScale
+                    || realPrecision - realScale > DecimalV2Type.MAX_PRECISION - DecimalV2Type.MAX_SCALE) {
                 valid = false;
             }
         } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java
index 741d2c3c4c11c5..65296b4f79b7f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DecimalV3Literal.java
@@ -18,9 +18,12 @@
 package org.apache.doris.nereids.trees.expressions.literal;
 
 import org.apache.doris.analysis.LiteralExpr;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.DecimalV3Type;
 
+import com.google.common.base.Preconditions;
+
 import java.math.BigDecimal;
 import java.math.RoundingMode;
 import java.util.Objects;
@@ -43,7 +46,7 @@ public DecimalV3Literal(BigDecimal value) {
     public DecimalV3Literal(DecimalV3Type dataType, BigDecimal value) {
         super(DecimalV3Type.createDecimalV3TypeLooseCheck(dataType.getPrecision(), dataType.getScale()));
         Objects.requireNonNull(value, "value not be null");
-        DecimalLiteral.checkPrecisionAndScale(dataType.getPrecision(), dataType.getScale(), value);
+        checkPrecisionAndScale(dataType.getPrecision(), dataType.getScale(), value);
         BigDecimal adjustedValue = value.scale() < 0 ? value
                 : value.setScale(dataType.getScale(), RoundingMode.HALF_UP);
         this.value = Objects.requireNonNull(adjustedValue);
@@ -80,4 +83,27 @@ public DecimalV3Literal roundFloor(int newScale) {
                 .createDecimalV3Type(((DecimalV3Type) dataType).getPrecision(), newScale),
                 value.setScale(newScale, RoundingMode.FLOOR));
     }
+
+    /**
+     * check precision and scale is enough for value.
+     */
+    private static void checkPrecisionAndScale(int precision, int scale, BigDecimal value) throws AnalysisException {
+        Preconditions.checkNotNull(value);
+        int realPrecision = value.precision();
+        int realScale = value.scale();
+        boolean valid = true;
+        if (precision != -1 && scale != -1) {
+            if (precision < realPrecision || scale < realScale) {
+                valid = false;
+            }
+        } else {
+            valid = false;
+        }
+
+        if (!valid) {
+            throw new AnalysisException(
+                    String.format("Invalid precision and scale - expect (%d, %d), but (%d, %d)",
+                            precision, scale, realPrecision, realScale));
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index b22c6093f9a75c..999e8881569736 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -156,6 +156,24 @@ public static DataType convertPrimitiveFromStrings(List<String> types, boolean u
                         throw new AnalysisException("Nereids do not support type: " + type);
                 }
                 break;
+            case "decimalv2":
+                // NOTICE, maybe convert to decimalv3, so do not truc here.
+                switch (types.size()) {
+                    case 1:
+                        dataType = DecimalV2Type.CATALOG_DEFAULT_NOT_CONVERSION;
+                        break;
+                    case 2:
+                        dataType = DecimalV2Type.createDecimalV2TypeWithoutTruncate(
+                                Integer.parseInt(types.get(1)), 0, false);
+                        break;
+                    case 3:
+                        dataType = DecimalV2Type.createDecimalV2TypeWithoutTruncate(
+                                Integer.parseInt(types.get(1)), Integer.parseInt(types.get(2)), false);
+                        break;
+                    default:
+                        throw new AnalysisException("Nereids do not support type: " + type);
+                }
+                break;
             case "decimalv3":
                 switch (types.size()) {
                     case 1:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java
index 00b043db7a67f3..523f113d996541 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DecimalV2Type.java
@@ -37,17 +37,21 @@ public class DecimalV2Type extends FractionalType {
 
     public static int MAX_PRECISION = 27;
     public static int MAX_SCALE = 9;
-    public static final DecimalV2Type SYSTEM_DEFAULT = new DecimalV2Type(MAX_PRECISION, MAX_SCALE);
-    public static final DecimalV2Type CATALOG_DEFAULT = new DecimalV2Type(DEFAULT_PRECISION, DEFAULT_SCALE);
-
-    private static final DecimalV2Type BOOLEAN_DECIMAL = new DecimalV2Type(1, 0);
-    private static final DecimalV2Type TINYINT_DECIMAL = new DecimalV2Type(3, 0);
-    private static final DecimalV2Type SMALLINT_DECIMAL = new DecimalV2Type(5, 0);
-    private static final DecimalV2Type INTEGER_DECIMAL = new DecimalV2Type(10, 0);
-    private static final DecimalV2Type BIGINT_DECIMAL = new DecimalV2Type(20, 0);
-    private static final DecimalV2Type LARGEINT_DECIMAL = new DecimalV2Type(27, 0);
-    private static final DecimalV2Type FLOAT_DECIMAL = new DecimalV2Type(14, 7);
-    private static final DecimalV2Type DOUBLE_DECIMAL = new DecimalV2Type(27, 9);
+    public static final DecimalV2Type SYSTEM_DEFAULT = new DecimalV2Type(MAX_PRECISION, MAX_SCALE, true);
+    public static final DecimalV2Type SYSTEM_DEFAULT_NOT_CONVERSION =
+            new DecimalV2Type(MAX_PRECISION, MAX_SCALE, false);
+    public static final DecimalV2Type CATALOG_DEFAULT = new DecimalV2Type(DEFAULT_PRECISION, DEFAULT_SCALE, true);
+    public static final DecimalV2Type CATALOG_DEFAULT_NOT_CONVERSION =
+            new DecimalV2Type(DEFAULT_PRECISION, DEFAULT_SCALE, false);
+
+    private static final DecimalV2Type BOOLEAN_DECIMAL = new DecimalV2Type(1, 0, true);
+    private static final DecimalV2Type TINYINT_DECIMAL = new DecimalV2Type(3, 0, true);
+    private static final DecimalV2Type SMALLINT_DECIMAL = new DecimalV2Type(5, 0, true);
+    private static final DecimalV2Type INTEGER_DECIMAL = new DecimalV2Type(10, 0, true);
+    private static final DecimalV2Type BIGINT_DECIMAL = new DecimalV2Type(20, 0, true);
+    private static final DecimalV2Type LARGEINT_DECIMAL = new DecimalV2Type(27, 0, true);
+    private static final DecimalV2Type FLOAT_DECIMAL = new DecimalV2Type(14, 7, true);
+    private static final DecimalV2Type DOUBLE_DECIMAL = new DecimalV2Type(27, 9, true);
 
     private static final int WIDTH = 16;
 
@@ -68,14 +72,17 @@ public class DecimalV2Type extends FractionalType {
     private final int precision;
     private final int scale;
 
+    private final boolean shouldConversion;
+
     /**
      * constructors.
      */
-    private DecimalV2Type(int precision, int scale) {
+    private DecimalV2Type(int precision, int scale, boolean shouldConversion) {
         Preconditions.checkArgument(precision >= scale, "precision should not smaller than scale,"
                 + " but precision is " + precision, ", scale is " + scale);
         this.precision = precision;
         this.scale = scale;
+        this.shouldConversion = shouldConversion;
     }
 
     /** createDecimalV2Type. */
@@ -86,7 +93,7 @@ public static DecimalV2Type createDecimalV2Type(int precision, int scale) {
         if (precision == CATALOG_DEFAULT.precision && scale == CATALOG_DEFAULT.scale) {
             return CATALOG_DEFAULT;
         }
-        return new DecimalV2Type(Math.min(precision, MAX_PRECISION), Math.min(scale, MAX_SCALE));
+        return new DecimalV2Type(Math.min(precision, MAX_PRECISION), Math.min(scale, MAX_SCALE), true);
     }
 
     public static DecimalV2Type createDecimalV2Type(BigDecimal bigDecimal) {
@@ -105,7 +112,22 @@ public static DecimalV2Type createDecimalV2TypeWithoutTruncate(int precision, in
         if (precision == CATALOG_DEFAULT.precision && scale == CATALOG_DEFAULT.scale) {
             return CATALOG_DEFAULT;
         }
-        return new DecimalV2Type(precision, scale);
+        return new DecimalV2Type(precision, scale, true);
+    }
+
+    /**
+     * create DecimalV2Type with appropriate scale, precision and shouldConversion flag,
+     * not truncate to MAX_PRECISION, MAX_SCALE.
+     */
+    public static DecimalV2Type createDecimalV2TypeWithoutTruncate(int precision, int scale,
+            boolean shouldConversion) {
+        if (precision == SYSTEM_DEFAULT.precision && scale == SYSTEM_DEFAULT.scale) {
+            return shouldConversion ? SYSTEM_DEFAULT : SYSTEM_DEFAULT_NOT_CONVERSION;
+        }
+        if (precision == CATALOG_DEFAULT.precision && scale == CATALOG_DEFAULT.scale) {
+            return shouldConversion ? CATALOG_DEFAULT : CATALOG_DEFAULT_NOT_CONVERSION;
+        }
+        return new DecimalV2Type(precision, scale, shouldConversion);
     }
 
     /**
@@ -153,7 +175,7 @@ public int getScale() {
 
     @Override
     public DataType conversion() {
-        if (Config.enable_decimal_conversion) {
+        if (Config.enable_decimal_conversion && shouldConversion) {
             return DecimalV3Type.createDecimalV3Type(precision, scale);
         }
         Preconditions.checkArgument(precision > 0 && precision <= MAX_PRECISION,
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index bae4d7459ce9f0..9baee503cfc491 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -331,6 +331,14 @@ public void testDatetimev1() {
 
     }
 
+    @Test
+    public void testDecimalv2() {
+        String decv2 = "SELECT CAST('1.234' AS decimalv2(10,5))";
+        NereidsParser nereidsParser = new NereidsParser();
+        LogicalPlan logicalPlan = (LogicalPlan) nereidsParser.parseSingle(decv2).child(0);
+        Assertions.assertTrue(logicalPlan.getExpressions().get(0).getDataType().isDecimalV2Type());
+    }
+
     @Test
     public void parseSetOperation() {
         String union = "select * from t1 union select * from t2 union all select * from t3";
diff --git a/regression-test/data/nereids_p0/datatype/test_decimalv2.out b/regression-test/data/nereids_p0/datatype/test_decimalv2.out
new file mode 100644
index 00000000000000..70c7ed37cbbf78
--- /dev/null
+++ b/regression-test/data/nereids_p0/datatype/test_decimalv2.out
@@ -0,0 +1,52 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql1 --
+1.230
+
+-- !sql2 --
+1.230
+
+-- !sql1 --
+1	1.230
+2	2.340
+3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
+-- !sql2 --
+1	1.230	1	1.230
+2	2.340	2	2.340
+3	3.450	3	3.450
+
diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy
index 2b23bf2d5dce7d..53cd87fac48478 100644
--- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy
+++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow.groovy
@@ -36,7 +36,7 @@ suite("test_decimalv2_overflow", "nonConcurrent") {
 	def tblName2 = "test_decimalv2_overflow2"
 	sql "drop table if exists ${tblName2}"
     sql """ CREATE  TABLE ${tblName2} (
-              `c2`  decimalv2(20, 2),
+              `c2`  decimalv2(20, 2)
           ) ENGINE=OLAP
         UNIQUE KEY(`c2`)
         DISTRIBUTED BY HASH(`c2`) BUCKETS 10
diff --git a/regression-test/suites/nereids_p0/datatype/test_decimalv2.groovy b/regression-test/suites/nereids_p0/datatype/test_decimalv2.groovy
new file mode 100644
index 00000000000000..3bdab1ccd51045
--- /dev/null
+++ b/regression-test/suites/nereids_p0/datatype/test_decimalv2.groovy
@@ -0,0 +1,88 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_decimalv2") {
+    def tbName = "test_decimalv2_exprs"
+
+    sql """set enable_nereids_planner=true"""
+    sql """set enable_fallback_to_original_planner=false"""
+    sql """set enable_nereids_dml=true"""
+
+    sql "DROP TABLE IF EXISTS ${tbName}"
+    sql """
+            create table ${tbName}(k1 decimalv2(10,3), k2 int) distributed by hash(k1) buckets 1 properties("replication_num" = "1");
+        """
+    sql """ insert into ${tbName} values("1.23", 1); """
+
+    qt_sql1 """ select dt
+             from
+             (
+             select cast(k1 as decimalv2(5,3)) as dt
+             from ${tbName}
+             ) r; """
+    qt_sql2 """ select dt
+             from
+             (
+             select cast(k1 as decimal(5,3)) as dt
+             from ${tbName}
+             ) r; """
+    sql "DROP TABLE ${tbName}"
+
+    tbName = "test_decimalv2_runtime_filter"
+    sql "DROP TABLE IF EXISTS ${tbName}"
+    sql """
+            CREATE TABLE IF NOT EXISTS ${tbName} (
+                c0 int,
+                c2 decimalv2(10, 3)
+            )
+            DISTRIBUTED BY HASH(c0) BUCKETS 5 properties("replication_num" = "1");
+        """
+    sql "insert into ${tbName} values(1, '1.23')"
+    sql "insert into ${tbName} values(2, '2.34')"
+    sql "insert into ${tbName} values(3, '3.45')"
+
+    qt_sql1 "select * from ${tbName} ORDER BY c0"
+
+    sql " set runtime_filter_type = 1; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql " set runtime_filter_type = 2; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql " set runtime_filter_type = 4; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql " set runtime_filter_type = 8; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql " set runtime_filter_wait_time_ms = 0; "
+
+    sql " set runtime_filter_type = 1; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql " set runtime_filter_type = 2; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql " set runtime_filter_type = 4; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql " set runtime_filter_type = 8; "
+    qt_sql2 "select * from ${tbName} a, ${tbName} b WHERE a.c2 = b.c2 ORDER BY a.c0"
+
+    sql "DROP TABLE ${tbName}"
+}

From 48f8f8f2ad7e8d71b08b0d9a3a861f2dbe98095e Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Mon, 25 Dec 2023 11:23:44 +0800
Subject: [PATCH 003/299] [fix](nereids)group by expr may be bound twice in
 bind agg slot (#28771)

---
 .../rules/analysis/BindExpression.java        | 38 +++++++++++++++----
 .../nereids_syntax_p0/analyze_agg.groovy      | 11 ++++++
 2 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
index 0437f7cdcdac3f..83b0eb1ea7924e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
@@ -292,10 +292,22 @@ protected boolean condition(Rule rule, Plan plan) {
                         if (alias.child().anyMatch(expr -> expr instanceof AggregateFunction)) {
                             continue;
                         }
-                        // NOTICE: must use unbound expressions, because we will bind them in binding group by expr.
-                        childOutputsToExpr.putIfAbsent(alias.getName(), agg.getOutputExpressions().get(i).child(0));
+                        /*
+                            Alias(x) has been bound by binding agg's output
+                            we add x to childOutputsToExpr, so when binding group by exprs later, we can use x directly
+                            and won't bind it again
+                            select
+                              p_cycle_time / (select max(p_cycle_time) from log_event_8)  as 'x',
+                              count(distinct case_id) as 'y'
+                            from
+                              log_event_8
+                            group by
+                              x
+                         */
+                        childOutputsToExpr.putIfAbsent(alias.getName(), output.get(i).child(0));
                     }
 
+                    Set<Expression> boundedGroupByExpressions = Sets.newHashSet();
                     List<Expression> replacedGroupBy = agg.getGroupByExpressions().stream()
                             .map(groupBy -> {
                                 if (groupBy instanceof UnboundSlot) {
@@ -303,7 +315,9 @@ protected boolean condition(Rule rule, Plan plan) {
                                     if (unboundSlot.getNameParts().size() == 1) {
                                         String name = unboundSlot.getNameParts().get(0);
                                         if (childOutputsToExpr.containsKey(name)) {
-                                            return childOutputsToExpr.get(name);
+                                            Expression expression = childOutputsToExpr.get(name);
+                                            boundedGroupByExpressions.add(expression);
+                                            return expression;
                                         }
                                     }
                                 }
@@ -336,16 +350,24 @@ protected boolean condition(Rule rule, Plan plan) {
 
                     List<Expression> groupBy = replacedGroupBy.stream()
                             .map(expression -> {
-                                Expression e = binder.bind(expression);
-                                if (e instanceof UnboundSlot) {
-                                    return childBinder.bind(e);
+                                if (boundedGroupByExpressions.contains(expression)) {
+                                    // expr has been bound by binding agg's output
+                                    return expression;
+                                } else {
+                                    // bind slot for unbound exprs
+                                    Expression e = binder.bind(expression);
+                                    if (e instanceof UnboundSlot) {
+                                        return childBinder.bind(e);
+                                    }
+                                    return e;
                                 }
-                                return e;
                             })
                             .collect(Collectors.toList());
                     groupBy.forEach(expression -> checkBoundExceptLambda(expression, ctx.root));
                     groupBy = groupBy.stream()
-                            .map(expr -> bindFunction(expr, ctx.root, ctx.cascadesContext))
+                            // bind function for unbound exprs or return old expr if it's bound by binding agg's output
+                            .map(expr -> boundedGroupByExpressions.contains(expr) ? expr
+                                    : bindFunction(expr, ctx.root, ctx.cascadesContext))
                             .collect(ImmutableList.toImmutableList());
                     checkIfOutputAliasNameDuplicatedForGroupBy(groupBy, output);
                     return agg.withGroupByAndOutput(groupBy, output);
diff --git a/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy b/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy
index 9f426e23e3d133..9a79df6bad524b 100644
--- a/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy
+++ b/regression-test/suites/nereids_syntax_p0/analyze_agg.groovy
@@ -77,4 +77,15 @@ suite("analyze_agg") {
 
     // should not bind g /g in group by again, otherwise will throw exception
     sql "select g / g as nu, sum(c) from t2 group by nu"
+    sql """
+            select
+                1,
+                id / (select max(id) from t2)  as 'x',
+                count(distinct c) as 'y'
+            from
+                t2
+            group by
+                1,
+                x
+        """
 }
\ No newline at end of file

From 6de797844d10dd55f4c0cc58d5fd89078ab55e0d Mon Sep 17 00:00:00 2001
From: meiyi <myimeiyi@gmail.com>
Date: Mon, 25 Dec 2023 11:34:11 +0800
Subject: [PATCH 004/299] [doc](insert) Add group commit docs (#25949)

---
 docs/en/docs/admin-manual/config/be-config.md |  13 +
 .../import/import-way/group-commit-manual.md  | 415 ++++++++++++++++++
 docs/sidebars.json                            |   3 +-
 .../docs/admin-manual/config/be-config.md     |  13 +
 .../import/import-way/group-commit-manual.md  | 413 +++++++++++++++++
 5 files changed, 856 insertions(+), 1 deletion(-)
 create mode 100644 docs/en/docs/data-operate/import/import-way/group-commit-manual.md
 create mode 100644 docs/zh-CN/docs/data-operate/import/import-way/group-commit-manual.md

diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md
index 3c72ac3a82aacf..1b8ee9423dc76e 100644
--- a/docs/en/docs/admin-manual/config/be-config.md
+++ b/docs/en/docs/admin-manual/config/be-config.md
@@ -1504,3 +1504,16 @@ Indicates how many tablets failed to load in the data directory. At the same tim
 
 * Description: BE Whether to enable the use of java-jni. When enabled, mutual calls between c++ and java are allowed. Currently supports hudi, java-udf, jdbc, max-compute, paimon, preload, avro
 * Default value: true
+
+#### `group_commit_wal_path`
+
+* The `WAL` directory of group commit.
+* Default: A directory named `wal` is created under each directory of the `storage_root_path`. Configuration examples:
+  ```
+  group_commit_wal_path=/data1/storage/wal;/data2/storage/wal;/data3/storage/wal
+  ```
+
+#### `group_commit_memory_rows_for_max_filter_ratio`
+
+* Description: The `max_filter_ratio` limit can only work if the total rows of `group commit` is less than this value. See [Group Commit](../../data-operate/import/import-way/group-commit-manual.md) for more details
+* Default: 10000
diff --git a/docs/en/docs/data-operate/import/import-way/group-commit-manual.md b/docs/en/docs/data-operate/import/import-way/group-commit-manual.md
new file mode 100644
index 00000000000000..9b0e007e15b6c0
--- /dev/null
+++ b/docs/en/docs/data-operate/import/import-way/group-commit-manual.md
@@ -0,0 +1,415 @@
+---
+{
+    "title": "Group Commit",
+    "language": "en"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Group Commit
+
+Group commit load does not introduce a new data import method, but an extension of `INSERT INTO tbl VALUS(...)`, `Stream Load` and `Http Stream`. It is a way to improve the write performance of Doris with high-concurrency and small-data writes. Your application can directly use JDBC to do high-concurrency insert operation into Doris, at the same time, combining PreparedStatement can get even higher performance. In logging scenarios, you can also do high-concurrency Stream Load or Http Stream into Doris. 
+
+## Group Commit Mode
+
+Group Commit provides 3 modes:
+
+* `off_mode`
+
+Disable group commit, keep the original behavior for `INSERT INTO VALUES`, `Stream Load` and `Http Stream`.
+
+* `sync_mode`
+
+Doris groups multiple loads into one transaction commit based on the `group_commit_interval` table property. The load is returned after the transaction commit. This mode is suitable for high-concurrency writing scenarios and requires immediate data visibility after the load is finished.
+
+* `async_mode`
+
+Doris writes data to the Write Ahead Log (WAL) firstly, then the load is returned. Doris groups multiple loads into one transaction commit based on the `group_commit_interval` table property, and the data is visible after the commit. To prevent excessive disk space usage by the WAL, it automatically switches to `sync_mode`. This is suitable for latency-sensitive and high-frequency writing.
+
+## Basic operations
+
+If the table schema is:
+```sql
+CREATE TABLE `dt` (
+    `id` int(11) NOT NULL,
+    `name` varchar(50) NULL,
+    `score` int(11) NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`id`)
+DISTRIBUTED BY HASH(`id`) BUCKETS 1
+PROPERTIES (
+    "replication_num" = "1"
+);
+```
+
+### INSERT INTO VALUES
+
+* async_mode
+```sql
+# Config session variable to enable the async group commit, the default value is off_mode
+mysql> set group_commit = async_mode;
+
+# The retured label is start with 'group_commit', which is the label of the real load job
+mysql> insert into dt values(1, 'Bob', 90), (2, 'Alice', 99);
+Query OK, 2 rows affected (0.05 sec)
+{'label':'group_commit_a145ce07f1c972fc-bd2c54597052a9ad', 'status':'PREPARE', 'txnId':'181508'}
+
+# The returned label and txn_id are the same as the above, which means they are handled in on load job  
+mysql> insert into dt(id, name) values(3, 'John');
+Query OK, 1 row affected (0.01 sec)
+{'label':'group_commit_a145ce07f1c972fc-bd2c54597052a9ad', 'status':'PREPARE', 'txnId':'181508'}
+
+# The data is not visible
+mysql> select * from dt;
+Empty set (0.01 sec)
+
+# After about 10 seconds, the data is visible
+mysql> select * from dt;
++------+-------+-------+
+| id   | name  | score |
++------+-------+-------+
+|    1 | Bob   |    90 |
+|    2 | Alice |    99 |
+|    3 | John  |  NULL |
++------+-------+-------+
+3 rows in set (0.02 sec)
+```
+
+* sync_mode
+```sql
+# Config session variable to enable the sync group commit
+mysql> set group_commit = sync_mode;
+
+# The retured label is start with 'group_commit', which is the label of the real load job. 
+# The insert costs at least the group_commit_interval_ms of table property.
+mysql> insert into dt values(4, 'Bob', 90), (5, 'Alice', 99);
+Query OK, 2 rows affected (10.06 sec)
+{'label':'group_commit_d84ab96c09b60587_ec455a33cb0e9e87', 'status':'PREPARE', 'txnId':'3007', 'query_id':'fc6b94085d704a94-a69bfc9a202e66e2'}
+
+# The data is visible after the insert is returned
+mysql> select * from dt;
++------+-------+-------+
+| id   | name  | score |
++------+-------+-------+
+|    1 | Bob   |    90 |
+|    2 | Alice |    99 |
+|    3 | John  |  NULL |
+|    4 | Bob   |    90 |
+|    5 | Alice |    99 |
++------+-------+-------+
+5 rows in set (0.03 sec)
+```
+
+* off_mode
+```sql
+mysql> set group_commit = off_mode;
+```
+
+### Stream Load
+
+If the content of `data.csv` is:
+```sql
+6,Amy,60
+7,Ross,98
+```
+
+* async_mode
+```sql
+# Add 'group_commit:async_mode' configuration in the http header
+
+curl --location-trusted -u {user}:{passwd} -T data.csv -H "group_commit:async_mode"  -H "column_separator:,"  http://{fe_host}:{http_port}/api/db/dt/_stream_load
+{
+    "TxnId": 7009,
+    "Label": "group_commit_c84d2099208436ab_96e33fda01eddba8",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 35,
+    "StreamLoadPutTimeMs": 5,
+    "ReadDataTimeMs": 0,
+    "WriteDataTimeMs": 26
+}
+
+# The returned 'GroupCommit' is 'true', which means this is a group commit load
+# The retured label is start with 'group_commit', which is the label of the real load job
+```
+
+* sync_mode
+```sql
+# Add 'group_commit:sync_mode' configuration in the http header
+
+curl --location-trusted -u {user}:{passwd} -T data.csv -H "group_commit:sync_mode"  -H "column_separator:,"  http://{fe_host}:{http_port}/api/db/dt/_stream_load
+{
+    "TxnId": 3009,
+    "Label": "group_commit_d941bf17f6efcc80_ccf4afdde9881293",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 10044,
+    "StreamLoadPutTimeMs": 4,
+    "ReadDataTimeMs": 0,
+    "WriteDataTimeMs": 10038
+}
+
+# The returned 'GroupCommit' is 'true', which means this is a group commit load
+# The retured label is start with 'group_commit', which is the label of the real load job
+```
+
+See [Stream Load](stream-load-manual.md) for more detailed syntax used by **Stream Load**.
+
+### Http Stream
+
+* async_mode
+```sql
+# Add 'group_commit:async_mode' configuration in the http header
+
+curl --location-trusted -u {user}:{passwd} -T data.csv  -H "group_commit:async_mode" -H "sql:insert into db.dt select * from http_stream('column_separator'=',', 'format' = 'CSV')"  http://{fe_host}:{http_port}/api/_http_stream
+{
+    "TxnId": 7011,
+    "Label": "group_commit_3b45c5750d5f15e5_703428e462e1ebb0",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 65,
+    "StreamLoadPutTimeMs": 41,
+    "ReadDataTimeMs": 47,
+    "WriteDataTimeMs": 23
+}
+
+# The returned 'GroupCommit' is 'true', which means this is a group commit load
+# The retured label is start with 'group_commit', which is the label of the real load job
+```
+
+* sync_mode
+```sql
+# Add 'group_commit:sync_mode' configuration in the http header
+
+curl --location-trusted -u {user}:{passwd} -T data.csv  -H "group_commit:sync_mode" -H "sql:insert into db.dt select * from http_stream('column_separator'=',', 'format' = 'CSV')"  http://{fe_host}:{http_port}/api/_http_stream
+{
+    "TxnId": 3011,
+    "Label": "group_commit_fe470e6752aadbe6_a8f3ac328b02ea91",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 10066,
+    "StreamLoadPutTimeMs": 31,
+    "ReadDataTimeMs": 32,
+    "WriteDataTimeMs": 10034
+}
+
+# The returned 'GroupCommit' is 'true', which means this is a group commit load
+# The retured label is start with 'group_commit', which is the label of the real load job
+```
+
+See [Stream Load](stream-load-manual.md) for more detailed syntax used by **Http Stream**.
+
+### Use `PreparedStatement`
+
+To reduce the CPU cost of SQL parsing and query planning, we provide the `PreparedStatement` in the FE. When using `PreparedStatement`, the SQL and its plan will be cached in the session level memory cache and will be reused later on, which reduces the CPU cost of FE. The following is an example of using PreparedStatement in JDBC:
+
+1. Setup JDBC url and enable server side prepared statement
+
+```
+url = jdbc:mysql://127.0.0.1:9030/db?useServerPrepStmts=true
+```
+
+2. Set `group_commit` session variable, there are two ways to do it:
+
+* Add `sessionVariables=group_commit=async_mode` in JDBC url
+
+```
+url = jdbc:mysql://127.0.0.1:9030/db?useServerPrepStmts=true&sessionVariables=group_commit=async_mode
+```
+
+* Use `SET group_commit = async_mode;` command
+
+```
+try (Statement statement = conn.createStatement()) {
+    statement.execute("SET group_commit = async_mode;");
+}
+```
+
+3. Using `PreparedStatement`
+
+```java
+private static final String JDBC_DRIVER = "com.mysql.jdbc.Driver";
+private static final String URL_PATTERN = "jdbc:mysql://%s:%d/%s?useServerPrepStmts=true";
+private static final String HOST = "127.0.0.1";
+private static final int PORT = 9087;
+private static final String DB = "db";
+private static final String TBL = "dt";
+private static final String USER = "root";
+private static final String PASSWD = "";
+private static final int INSERT_BATCH_SIZE = 10;
+
+private static void groupCommitInsert() throws Exception {
+    Class.forName(JDBC_DRIVER);
+    try (Connection conn = DriverManager.getConnection(String.format(URL_PATTERN, HOST, PORT, DB), USER, PASSWD)) {
+        // set session variable 'group_commit'
+        try (Statement statement = conn.createStatement()) {
+            statement.execute("SET group_commit = async_mode;");
+        }
+
+        String query = "insert into " + TBL + " values(?, ?, ?)";
+        try (PreparedStatement stmt = conn.prepareStatement(query)) {
+            for (int i = 0; i < INSERT_BATCH_SIZE; i++) {
+                stmt.setInt(1, i);
+                stmt.setString(2, "name" + i);
+                stmt.setInt(3, i + 10);
+                int result = stmt.executeUpdate();
+                System.out.println("rows: " + result);
+            }
+        }
+    } catch (Exception e) {
+        e.printStackTrace();
+    }
+}   
+
+private static void groupCommitInsertBatch() throws Exception {
+    Class.forName(JDBC_DRIVER);
+    // add rewriteBatchedStatements=true and cachePrepStmts=true in JDBC url
+    // set session variables by sessionVariables=group_commit=async_mode in JDBC url
+    try (Connection conn = DriverManager.getConnection(
+            String.format(URL_PATTERN + "&rewriteBatchedStatements=true&cachePrepStmts=true&sessionVariables=group_commit=async_mode", HOST, PORT, DB), USER, PASSWD)) {
+
+        String query = "insert into " + TBL + " values(?, ?, ?)";
+        try (PreparedStatement stmt = conn.prepareStatement(query)) {
+            for (int j = 0; j < 5; j++) {
+                // 10 rows per insert
+                for (int i = 0; i < INSERT_BATCH_SIZE; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, "name" + i);
+                    stmt.setInt(3, i + 10);
+                    stmt.addBatch();
+                }
+                int[] result = stmt.executeBatch();
+            }
+        }
+    } catch (Exception e) {
+        e.printStackTrace();
+    }
+}
+```
+
+See [Synchronize Data Using Insert Method](../import-scenes/jdbc-load.md) for more details about **JDBC**.
+
+## Modify the group commit interval
+
+The default group commit interval is 10 seconds. Users can modify the configuration of the table:
+
+```sql
+# Modify the group commit interval to 2 seconds
+ALTER TABLE dt SET ("group_commit_interval_ms"="2000");
+```
+
+## Limitations
+
+* When the group commit is enabled, some `INSERT INTO VALUES` sqls are not executed in the group commit way if they meet the following conditions:
+
+  * Transaction insert, such as `BEGIN`, `INSERT INTO VALUES`, `COMMIT`
+
+  * Specify the label, such as `INSERT INTO dt WITH LABEL {label} VALUES`
+
+  * Expressions within VALUES, such as `INSERT INTO dt VALUES (1 + 100)`
+
+  * Column update
+
+  * Tables that do not support light schema changes
+
+* When the group commit is enabled, some `Stream Load` and `Http Stream` are not executed in the group commit way if they meet the following conditions:
+
+  * Two phase commit
+
+  * Specify the label
+
+  * Column update
+
+  * Tables that do not support light schema changes
+
+* For unique table, because the group commit can not guarantee the commit order, users can use sequence column to ensure the data consistency.
+
+* The limit of `max_filter_ratio`
+
+  * For non group commit load, filter_ratio is calculated by the failed rows and total rows when load is finished. If the filter_ratio does not match, the transaction will not commit
+
+  * In the group commit mode, multiple user loads are executed by one internal load. The internal load will commit all user loads.
+
+  * Currently, group commit supports a certain degree of max_filter_ratio semantics. When the total number of rows does not exceed group_commit_memory_rows_for_max_filter_ratio (configured in `be.conf`, defaulting to `10000` rows), max_filter_ratio will work.
+
+* The limit of WAL
+
+  * For async_mode group commit, data is written to the Write Ahead Log (WAL). If the internal load succeeds, the WAL is immediately deleted. If the internal load fails, data is recovery by importing the WAL.
+
+  * Currently, WAL files are stored only on one disk of one BE. If the BE's disk is damaged or the file is mistakenly deleted, it may result in data loss.
+
+  * When decommissioning a BE node, please use the [`DECOMMISSION`](../../../sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DECOMMISSION-BACKEND.md) command to safely decommission the node. This prevents potential data loss if the WAL files are not processed before the node is taken offline.
+
+  * For async_mode group commit writes, to protect disk space, it switches to sync_mode under the following conditions:
+
+    * For an import with large amount of data: exceeding 80% of the disk space of a WAL directory. 
+
+    * Chunked stream loads with an unknown data amount.
+
+    * Insufficient disk space, even with it is an import with small amount of data.
+
+  * During hard weight schema changes (adding or dropping columns, modifying varchar length, and renaming columns are lightweight schema changes, others are hard weight), to ensure WAL file is compatibility with the table's schema, the final stage of metadata modification in FE will reject group commit writes. Clients get `insert table ${table_name} is blocked on schema change` exception and can retry the import.
+
+## Relevant system configuration
+
+### BE configuration
+
+#### `group_commit_wal_path`
+
+* The `WAL` directory of group commit.
+* Default: A directory named `wal` is created under each directory of the `storage_root_path`. Configuration examples:
+  ```
+  group_commit_wal_path=/data1/storage/wal;/data2/storage/wal;/data3/storage/wal
+  ```
+
+#### `group_commit_memory_rows_for_max_filter_ratio`
+
+* Description: The `max_filter_ratio` limit can only work if the total rows of `group commit` is less than this value.
+* Default: 10000
+
diff --git a/docs/sidebars.json b/docs/sidebars.json
index da910610f08645..7e5286f7a4c73a 100644
--- a/docs/sidebars.json
+++ b/docs/sidebars.json
@@ -91,7 +91,8 @@
                                 "data-operate/import/import-way/mysql-load-manual",
                                 "data-operate/import/import-way/s3-load-manual",
                                 "data-operate/import/import-way/insert-into-manual",
-                                "data-operate/import/import-way/load-json-format"
+                                "data-operate/import/import-way/load-json-format",
+                                "data-operate/import/import-way/group-commit-manual"
                             ]
                         },
                         {
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md
index 04d74f49d60632..7b99c1f2cb9068 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -1533,3 +1533,16 @@ load tablets from header failed, failed tablets size: xxx, path=xxx
 
 * 描述: BE 是否开启使用java-jni，开启后允许 c++ 与 java 之间的相互调用。目前已经支持hudi、java-udf、jdbc、max-compute、paimon、preload、avro
 * 默认值: true
+
+#### `group_commit_wal_path`
+
+* 描述:  group commit 存放 WAL 文件的目录，请参考 [Group Commit](../../data-operate/import/import-way/group-commit-manual.md)
+* 默认值: 默认在用户配置的`storage_root_path`的各个目录下创建一个名为`wal`的目录。配置示例：
+  ```
+  group_commit_wal_path=/data1/storage/wal;/data2/storage/wal;/data3/storage/wal
+  ```
+
+#### `group_commit_memory_rows_for_max_filter_ratio`
+
+* 描述:  当 group commit 导入的总行数不高于该值，`max_filter_ratio` 正常工作，否则不工作，请参考 [Group Commit](../../data-operate/import/import-way/group-commit-manual.md)
+* 默认值: 10000
diff --git a/docs/zh-CN/docs/data-operate/import/import-way/group-commit-manual.md b/docs/zh-CN/docs/data-operate/import/import-way/group-commit-manual.md
new file mode 100644
index 00000000000000..e657b6a6199ae6
--- /dev/null
+++ b/docs/zh-CN/docs/data-operate/import/import-way/group-commit-manual.md
@@ -0,0 +1,413 @@
+---
+{
+    "title": "Group Commit",
+    "language": "zh-CN"
+}
+---
+
+<!-- 
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Group Commit
+
+Group Commit 不是一种新的导入方式，而是对`INSERT INTO tbl VALUES(...)`、`Stream Load`、`Http Stream`的扩展，大幅提升了高并发小写入的性能。您的应用程序可以直接使用 JDBC 将数据高频写入 Doris，同时通过使用 PreparedStatement 可以获得更高的性能。在日志场景下，您也可以利用 Stream Load 或者 Http Stream 将数据高频写入 Doris。
+
+## Group Commit 模式
+
+Group Commit 写入有三种模式，分别是：
+
+* 关闭模式（`off_mode`）
+
+不开启 Group Commit，保持以上三种导入方式的默认行为。
+
+* 同步模式（`sync_mode`）
+
+Doris 根据负载和表的 `group_commit_interval`属性将多个导入在一个事务提交，事务提交后导入返回。这适用于高并发写入场景，且在导入完成后要求数据立即可见。
+
+* 异步模式（`async_mode`）
+
+Doris 首先将数据写入 WAL (`Write Ahead Log`)，然后导入立即返回。Doris 会根据负载和表的`group_commit_interval`属性异步提交数据，提交之后数据可见。为了防止 WAL 占用较大的磁盘空间，单次导入数据量较大时，会自动切换为`sync_mode`。这适用于写入延迟敏感以及高频写入的场景。
+
+## Group Commit 使用方式
+
+假如表的结构为：
+```sql
+CREATE TABLE `dt` (
+    `id` int(11) NOT NULL,
+    `name` varchar(50) NULL,
+    `score` int(11) NULL
+) ENGINE=OLAP
+DUPLICATE KEY(`id`)
+DISTRIBUTED BY HASH(`id`) BUCKETS 1
+PROPERTIES (
+    "replication_num" = "1"
+);
+```
+
+### INSERT INTO VALUES
+
+* 异步模式
+```sql
+# 配置session变量开启 group commit (默认为off_mode),开启异步模式
+mysql> set group_commit = async_mode;
+
+# 这里返回的label是 group_commit 开头的，可以区分出是否使用了 group commit
+mysql> insert into dt values(1, 'Bob', 90), (2, 'Alice', 99);
+Query OK, 2 rows affected (0.05 sec)
+{'label':'group_commit_a145ce07f1c972fc-bd2c54597052a9ad', 'status':'PREPARE', 'txnId':'181508'}
+
+# 可以看出这个 label, txn_id 和上一个相同，说明是攒到了同一个导入任务中
+mysql> insert into dt(id, name) values(3, 'John');
+Query OK, 1 row affected (0.01 sec)
+{'label':'group_commit_a145ce07f1c972fc-bd2c54597052a9ad', 'status':'PREPARE', 'txnId':'181508'}
+
+# 不能立刻查询到
+mysql> select * from dt;
+Empty set (0.01 sec)
+
+# 10秒后可以查询到，可以通过表属性 group_commit_interval 控制数据可见延迟。
+mysql> select * from dt;
++------+-------+-------+
+| id   | name  | score |
++------+-------+-------+
+|    1 | Bob   |    90 |
+|    2 | Alice |    99 |
+|    3 | John  |  NULL |
++------+-------+-------+
+3 rows in set (0.02 sec)
+```
+
+* 同步模式
+```sql
+# 配置session变量开启 group commit (默认为off_mode),开启同步模式
+mysql> set group_commit = sync_mode;
+
+# 这里返回的 label 是 group_commit 开头的，可以区分出是否谁用了 group commit，导入耗时至少是表属性 group_commit_interval。
+mysql> insert into dt values(4, 'Bob', 90), (5, 'Alice', 99);
+Query OK, 2 rows affected (10.06 sec)
+{'label':'group_commit_d84ab96c09b60587_ec455a33cb0e9e87', 'status':'PREPARE', 'txnId':'3007', 'query_id':'fc6b94085d704a94-a69bfc9a202e66e2'}
+
+# 数据可以立刻读出
+mysql> select * from dt;
++------+-------+-------+
+| id   | name  | score |
++------+-------+-------+
+|    1 | Bob   |    90 |
+|    2 | Alice |    99 |
+|    3 | John  |  NULL |
+|    4 | Bob   |    90 |
+|    5 | Alice |    99 |
++------+-------+-------+
+5 rows in set (0.03 sec)
+```
+
+* 关闭模式
+```sql
+mysql> set group_commit = off_mode;
+```
+
+### Stream Load
+
+假如`data.csv`的内容为：
+```sql
+6,Amy,60
+7,Ross,98
+```
+
+* 异步模式
+```sql
+# 导入时在header中增加"group_commit:async_mode"配置
+
+curl --location-trusted -u {user}:{passwd} -T data.csv -H "group_commit:async_mode"  -H "column_separator:,"  http://{fe_host}:{http_port}/api/db/dt/_stream_load
+{
+    "TxnId": 7009,
+    "Label": "group_commit_c84d2099208436ab_96e33fda01eddba8",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 35,
+    "StreamLoadPutTimeMs": 5,
+    "ReadDataTimeMs": 0,
+    "WriteDataTimeMs": 26
+}
+
+# 返回的GroupCommit为true，说明进入了group commit的流程
+# 返回的Label是group_commit开头的，是真正消费数据的导入关联的label
+```
+
+* 同步模式
+```sql
+# 导入时在header中增加"group_commit:sync_mode"配置
+
+curl --location-trusted -u {user}:{passwd} -T data.csv -H "group_commit:sync_mode"  -H "column_separator:,"  http://{fe_host}:{http_port}/api/db/dt/_stream_load
+{
+    "TxnId": 3009,
+    "Label": "group_commit_d941bf17f6efcc80_ccf4afdde9881293",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 10044,
+    "StreamLoadPutTimeMs": 4,
+    "ReadDataTimeMs": 0,
+    "WriteDataTimeMs": 10038
+}
+
+# 返回的GroupCommit为true，说明进入了group commit的流程
+# 返回的Label是group_commit开头的，是真正消费数据的导入关联的label
+```
+
+关于 Stream Load 使用的更多详细语法及最佳实践，请参阅 [Stream Load](stream-load-manual.md)。
+
+### Http Stream
+
+* 异步模式
+```sql
+# 导入时在header中增加"group_commit:async_mode"配置
+
+curl --location-trusted -u {user}:{passwd} -T data.csv  -H "group_commit:async_mode" -H "sql:insert into db.dt select * from http_stream('column_separator'=',', 'format' = 'CSV')"  http://{fe_host}:{http_port}/api/_http_stream
+{
+    "TxnId": 7011,
+    "Label": "group_commit_3b45c5750d5f15e5_703428e462e1ebb0",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 65,
+    "StreamLoadPutTimeMs": 41,
+    "ReadDataTimeMs": 47,
+    "WriteDataTimeMs": 23
+}
+
+# 返回的GroupCommit为true，说明进入了group commit的流程
+# 返回的Label是group_commit开头的，是真正消费数据的导入关联的label
+```
+
+* 同步模式
+```sql
+# 导入时在header中增加"group_commit:sync_mode"配置
+
+curl --location-trusted -u {user}:{passwd} -T data.csv  -H "group_commit:sync_mode" -H "sql:insert into db.dt select * from http_stream('column_separator'=',', 'format' = 'CSV')"  http://{fe_host}:{http_port}/api/_http_stream
+{
+    "TxnId": 3011,
+    "Label": "group_commit_fe470e6752aadbe6_a8f3ac328b02ea91",
+    "Comment": "",
+    "GroupCommit": true,
+    "Status": "Success",
+    "Message": "OK",
+    "NumberTotalRows": 2,
+    "NumberLoadedRows": 2,
+    "NumberFilteredRows": 0,
+    "NumberUnselectedRows": 0,
+    "LoadBytes": 19,
+    "LoadTimeMs": 10066,
+    "StreamLoadPutTimeMs": 31,
+    "ReadDataTimeMs": 32,
+    "WriteDataTimeMs": 10034
+}
+
+# 返回的GroupCommit为true，说明进入了group commit的流程
+# 返回的Label是group_commit开头的，是真正消费数据的导入关联的label
+```
+
+关于 Http Stream 使用的更多详细语法及最佳实践，请参阅 [Stream Load](stream-load-manual.md)。
+
+### 使用`PreparedStatement`
+
+当用户使用 JDBC `insert into values`方式写入时，为了减少 SQL 解析和生成规划的开销， 我们在 FE 端支持了 MySQL 协议的`PreparedStatement`特性。当使用`PreparedStatement`时，SQL 和其导入规划将被缓存到 Session 级别的内存缓存中，后续的导入直接使用缓存对象，降低了 FE 的 CPU 压力。下面是在 JDBC 中使用 PreparedStatement 的例子：
+
+1. 设置 JDBC url 并在 Server 端开启 prepared statement
+
+```
+url = jdbc:mysql://127.0.0.1:9030/db?useServerPrepStmts=true
+```
+
+2. 配置 `group_commit` session变量，有如下两种方式：
+
+* 通过 JDBC url 设置，增加`sessionVariables=group_commit=async_mode`
+
+```
+url = jdbc:mysql://127.0.0.1:9030/db?useServerPrepStmts=true&sessionVariables=group_commit=async_mode
+```
+
+* 通过执行 SQL 设置
+
+```
+try (Statement statement = conn.createStatement()) {
+    statement.execute("SET group_commit = async_mode;");
+}
+```
+
+3. 使用 `PreparedStatement`
+
+```java
+private static final String JDBC_DRIVER = "com.mysql.jdbc.Driver";
+private static final String URL_PATTERN = "jdbc:mysql://%s:%d/%s?useServerPrepStmts=true";
+private static final String HOST = "127.0.0.1";
+private static final int PORT = 9087;
+private static final String DB = "db";
+private static final String TBL = "dt";
+private static final String USER = "root";
+private static final String PASSWD = "";
+private static final int INSERT_BATCH_SIZE = 10;
+
+private static void groupCommitInsert() throws Exception {
+    Class.forName(JDBC_DRIVER);
+    try (Connection conn = DriverManager.getConnection(String.format(URL_PATTERN, HOST, PORT, DB), USER, PASSWD)) {
+        // set session variable 'group_commit'
+        try (Statement statement = conn.createStatement()) {
+            statement.execute("SET group_commit = async_mode;");
+        }
+
+        String query = "insert into " + TBL + " values(?, ?, ?)";
+        try (PreparedStatement stmt = conn.prepareStatement(query)) {
+            for (int i = 0; i < INSERT_BATCH_SIZE; i++) {
+                stmt.setInt(1, i);
+                stmt.setString(2, "name" + i);
+                stmt.setInt(3, i + 10);
+                int result = stmt.executeUpdate();
+                System.out.println("rows: " + result);
+            }
+        }
+    } catch (Exception e) {
+        e.printStackTrace();
+    }
+}   
+
+private static void groupCommitInsertBatch() throws Exception {
+    Class.forName(JDBC_DRIVER);
+    // add rewriteBatchedStatements=true and cachePrepStmts=true in JDBC url
+    // set session variables by sessionVariables=group_commit=async_mode in JDBC url
+    try (Connection conn = DriverManager.getConnection(
+            String.format(URL_PATTERN + "&rewriteBatchedStatements=true&cachePrepStmts=true&sessionVariables=group_commit=async_mode", HOST, PORT, DB), USER, PASSWD)) {
+
+        String query = "insert into " + TBL + " values(?, ?, ?)";
+        try (PreparedStatement stmt = conn.prepareStatement(query)) {
+            for (int j = 0; j < 5; j++) {
+                // 10 rows per insert
+                for (int i = 0; i < INSERT_BATCH_SIZE; i++) {
+                    stmt.setInt(1, i);
+                    stmt.setString(2, "name" + i);
+                    stmt.setInt(3, i + 10);
+                    stmt.addBatch();
+                }
+                int[] result = stmt.executeBatch();
+            }
+        }
+    } catch (Exception e) {
+        e.printStackTrace();
+    }
+}
+```
+
+关于**JDBC**的更多用法，参考[使用Insert方式同步数据](../import-scenes/jdbc-load.md)。
+
+## 修改group commit默认提交间隔
+
+group commit 的默认提交间隔为 10 秒，用户可以通过修改表的配置，调整 group commit 的提交间隔：
+
+```sql
+# 修改提交间隔为 2 秒
+ALTER TABLE dt SET ("group_commit_interval_ms"="2000");
+```
+
+## 使用限制
+
+* 当开启了 group commit 模式，系统会判断用户发起的`INSERT INTO VALUES`语句是否符合 group commit 的条件，如果符合，该语句的执行会进入到 group commit 写入中。符合以下条件会自动退化为非 group commit 方式：
+
+  + 事务写入，即`Begin`; `INSERT INTO VALUES`; `COMMIT`方式
+
+  + 指定 label，即`INSERT INTO dt WITH LABEL {label} VALUES`
+
+  + VALUES 中包含表达式，即`INSERT INTO dt VALUES (1 + 100)`
+
+  + 列更新写入
+
+  + 表不支持 light schema change
+
+* 当开启了 group commit 模式，系统会判断用户发起的`Stream Load`和`Http Stream`是否符合 group commit 的条件，如果符合，该导入的执行会进入到 group commit 写入中。符合以下条件的会自动退化为非 group commit 方式：
+
+  + 两阶段提交
+
+  + 指定 label
+
+  + 列更新写入
+
+  + 表不支持 light schema change
+
++ 对于 unique 模型，由于 group commit 不能保证提交顺序，用户可以配合 sequence 列使用来保证数据一致性
+
+* 对`max_filter_ratio`语义的支持
+
+  * 在默认的导入中，`filter_ratio`是导入完成后，通过失败的行数和总行数计算，决定是否提交本次写入
+
+  * 在 group commit 模式下，由于多个用户发起的导入会被一个内部导入执行，虽然可以计算出每个导入的`filter_ratio`，但是数据一旦进入内部导入，就只能 commit transaction
+
+  * group commit 模式支持了一定程度的`max_filter_ratio`语义，当导入的总行数不高于`group_commit_memory_rows_for_max_filter_ratio`(配置在`be.conf`中，默认为`10000`行)，`max_filter_ratio` 工作
+
+* WAL 限制
+
+  * 对于`async_mode`的 group commit 写入，会把数据写入 WAL。如果内部导入成功，则 WAL 被立刻删除；如果内部导入失败，通过导入 WAL 的方法来恢复数据
+
+  * 目前 WAL 文件只存储在一个 BE 上，如果这个 BE 磁盘损坏或文件误删等，可能导入丢失部分数据
+
+  * 当下线 BE 节点时，请使用[`DECOMMISSION`](../../../sql-manual/sql-reference/Cluster-Management-Statements/ALTER-SYSTEM-DECOMMISSION-BACKEND.md)命令，安全下线节点，防止该节点下线前 WAL 文件还没有全部处理完成，导致部分数据丢失
+
+  * 对于`async_mode`的 group commit 写入，为了保护磁盘空间，当遇到以下情况时，会切换成`sync_mode`
+
+    * 导入数据量过大，即超过 WAL 单目录的80%空间
+
+    * 不知道数据量的 chunked stream load
+
+    * 导入数据量不大，但磁盘可用空间不足
+
+  * 当发生重量级 schema change（目前加减列、修改 varchar 长度和重命名列是轻量级 schema change，其它的是重量级 schema change） 时，为了保证 WAL 能够适配表的 schema，在 schema change 最后的 fe 修改元数据阶段，会拒绝 group commit 写入，客户端收到`insert table ${table_name} is blocked on schema change`异常，客户端重试即可
+
+## 相关系统配置
+
+### BE 配置
+
+#### `group_commit_wal_path`
+
+* 描述:  group commit 存放 WAL 文件的目录
+* 默认值: 默认在用户配置的`storage_root_path`的各个目录下创建一个名为`wal`的目录。配置示例：
+  ```
+  group_commit_wal_path=/data1/storage/wal;/data2/storage/wal;/data3/storage/wal
+  ```
+
+#### `group_commit_memory_rows_for_max_filter_ratio`
+
+* 描述:  当 group commit 导入的总行数不高于该值，`max_filter_ratio` 正常工作，否则不工作
+* 默认值: 10000

From e9e1e2894bfa729613d4e3c48a543dbb54b72452 Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Mon, 25 Dec 2023 11:50:41 +0800
Subject: [PATCH 005/299] [performance](variant) support topn 2phase read for
 variant column (#28318)

 [performance](variant) support topn 2phase read for variant column
---
 .../segment_v2/hierarchical_data_reader.h     |  2 +-
 be/src/olap/rowset/segment_v2/segment.cpp     | 67 ++++++++++++-
 be/src/olap/rowset/segment_v2/segment.h       | 14 ++-
 .../rowset/segment_v2/segment_iterator.cpp    |  3 +-
 .../olap/rowset/segment_v2/segment_iterator.h |  4 +-
 be/src/olap/tablet_schema.cpp                 | 14 +++
 be/src/olap/tablet_schema.h                   |  6 ++
 be/src/runtime/descriptors.cpp                |  4 +
 be/src/service/internal_service.cpp           | 94 ++++++++++++-------
 be/src/vec/columns/column_object.cpp          |  9 +-
 be/src/vec/exec/scan/new_olap_scanner.cpp     | 30 ++----
 be/src/vec/exec/scan/new_olap_scanner.h       |  1 -
 be/src/vec/json/path_in_data.cpp              | 11 +++
 be/src/vec/json/path_in_data.h                |  1 +
 .../org/apache/doris/analysis/SelectStmt.java |  3 +-
 gensrc/proto/descriptors.proto                |  1 +
 regression-test/data/variant_p0/load.out      | 22 ++---
 .../data/variant_p0/sql/gh_data.out           | 68 ++++++++++----
 regression-test/suites/variant_p0/load.groovy |  8 +-
 .../suites/variant_p0/sql/gh_data.sql         |  9 +-
 20 files changed, 261 insertions(+), 110 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
index 58da2a43435ae5..99f926f44cf5e6 100644
--- a/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
+++ b/be/src/olap/rowset/segment_v2/hierarchical_data_reader.h
@@ -229,7 +229,7 @@ class ExtractReader : public ColumnIterator {
 private:
     Status extract_to(vectorized::MutableColumnPtr& dst, size_t nrows);
 
-    const TabletColumn& _col;
+    TabletColumn _col;
     // may shared among different column iterators
     std::unique_ptr<StreamReader> _root_reader;
 };
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index e3d1ce0c599523..de2593447c5175 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -61,10 +61,12 @@
 #include "util/slice.h" // Slice
 #include "vec/columns/column.h"
 #include "vec/common/string_ref.h"
+#include "vec/core/field.h"
 #include "vec/data_types/data_type.h"
 #include "vec/data_types/data_type_factory.hpp"
 #include "vec/data_types/data_type_nullable.h"
 #include "vec/data_types/data_type_object.h"
+#include "vec/json/path_in_data.h"
 #include "vec/olap/vgeneric_iterators.h"
 
 namespace doris {
@@ -332,17 +334,18 @@ Status Segment::_load_index_impl() {
 
 // Return the storage datatype of related column to field.
 // Return nullptr meaning no such storage infomation for this column
-vectorized::DataTypePtr Segment::get_data_type_of(const Field& field, bool ignore_children) const {
+vectorized::DataTypePtr Segment::get_data_type_of(vectorized::PathInData path, bool is_nullable,
+                                                  bool ignore_children) const {
     // Path has higher priority
-    if (!field.path().empty()) {
-        auto node = _sub_column_tree.find_leaf(field.path());
+    if (!path.empty()) {
+        auto node = _sub_column_tree.find_leaf(path);
         if (node) {
             if (ignore_children || node->children.empty()) {
                 return node->data.file_column_type;
             }
         }
         // it contains children or column missing in storage, so treat it as variant
-        return field.is_nullable()
+        return is_nullable
                        ? vectorized::make_nullable(std::make_shared<vectorized::DataTypeObject>())
                        : std::make_shared<vectorized::DataTypeObject>();
     }
@@ -686,7 +689,8 @@ Status Segment::read_key_by_rowid(uint32_t row_id, std::string* key) {
 
 bool Segment::same_with_storage_type(int32_t cid, const Schema& schema,
                                      bool ignore_children) const {
-    auto file_column_type = get_data_type_of(*schema.column(cid), ignore_children);
+    auto file_column_type = get_data_type_of(schema.column(cid)->path(),
+                                             schema.column(cid)->is_nullable(), ignore_children);
     auto expected_type = Schema::get_data_type_ptr(*schema.column(cid));
 #ifndef NDEBUG
     if (file_column_type && !file_column_type->equals(*expected_type)) {
@@ -700,5 +704,58 @@ bool Segment::same_with_storage_type(int32_t cid, const Schema& schema,
     return same;
 }
 
+Status Segment::seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor* slot,
+                                       uint32_t row_id, vectorized::MutableColumnPtr& result,
+                                       OlapReaderStatistics& stats,
+                                       std::unique_ptr<ColumnIterator>& iterator_hint) {
+    StorageReadOptions storage_read_opt;
+    storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
+    segment_v2::ColumnIteratorOptions opt {
+            .use_page_cache = !config::disable_storage_page_cache,
+            .file_reader = file_reader().get(),
+            .stats = &stats,
+            .io_ctx = io::IOContext {.reader_type = ReaderType::READER_QUERY},
+    };
+    std::vector<segment_v2::rowid_t> single_row_loc {row_id};
+    if (!slot->column_paths().empty()) {
+        vectorized::PathInData path(schema.column_by_uid(slot->col_unique_id()).name_lower_case(),
+                                    slot->column_paths());
+        auto storage_type = get_data_type_of(path, slot->is_nullable(), false);
+        vectorized::MutableColumnPtr file_storage_column = storage_type->create_column();
+        DCHECK(storage_type != nullptr);
+        TabletColumn column = TabletColumn::create_materialized_variant_column(
+                schema.column_by_uid(slot->col_unique_id()).name_lower_case(), slot->column_paths(),
+                slot->col_unique_id());
+        if (iterator_hint == nullptr) {
+            RETURN_IF_ERROR(new_column_iterator(column, &iterator_hint, &storage_read_opt));
+            RETURN_IF_ERROR(iterator_hint->init(opt));
+        }
+        RETURN_IF_ERROR(
+                iterator_hint->read_by_rowids(single_row_loc.data(), 1, file_storage_column));
+        // iterator_hint.reset(nullptr);
+        // Get it's inner field, for JSONB case
+        vectorized::Field field = remove_nullable(storage_type)->get_default();
+        file_storage_column->get(0, field);
+        result->insert(field);
+    } else {
+        int index = (slot->col_unique_id() >= 0) ? schema.field_index(slot->col_unique_id())
+                                                 : schema.field_index(slot->col_name());
+        if (index < 0) {
+            std::stringstream ss;
+            ss << "field name is invalid. field=" << slot->col_name()
+               << ", field_name_to_index=" << schema.get_all_field_names();
+            return Status::InternalError(ss.str());
+        }
+        storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
+        if (iterator_hint == nullptr) {
+            RETURN_IF_ERROR(
+                    new_column_iterator(schema.column(index), &iterator_hint, &storage_read_opt));
+            RETURN_IF_ERROR(iterator_hint->init(opt));
+        }
+        RETURN_IF_ERROR(iterator_hint->read_by_rowids(single_row_loc.data(), 1, result));
+    }
+    return Status::OK();
+}
+
 } // namespace segment_v2
 } // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index 42b7fc83ac0cae..47ccab71be68a2 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -39,11 +39,14 @@
 #include "olap/rowset/segment_v2/page_handle.h"
 #include "olap/schema.h"
 #include "olap/tablet_schema.h"
+#include "runtime/descriptors.h"
 #include "util/once.h"
 #include "util/slice.h"
+#include "vec/columns/column.h"
 #include "vec/columns/subcolumn_tree.h"
 #include "vec/data_types/data_type.h"
 #include "vec/data_types/data_type_nullable.h"
+#include "vec/json/path_in_data.h"
 
 namespace doris {
 namespace vectorized {
@@ -123,6 +126,10 @@ class Segment : public std::enable_shared_from_this<Segment> {
 
     Status read_key_by_rowid(uint32_t row_id, std::string* key);
 
+    Status seek_and_read_by_rowid(const TabletSchema& schema, SlotDescriptor* slot, uint32_t row_id,
+                                  vectorized::MutableColumnPtr& result, OlapReaderStatistics& stats,
+                                  std::unique_ptr<ColumnIterator>& iterator_hint);
+
     Status load_index();
 
     Status load_pk_index_and_bf();
@@ -146,7 +153,8 @@ class Segment : public std::enable_shared_from_this<Segment> {
     // ignore_chidren set to false will treat field as variant
     // when it contains children with field paths.
     // nullptr will returned if storage type does not contains such column
-    std::shared_ptr<const vectorized::IDataType> get_data_type_of(const Field& filed,
+    std::shared_ptr<const vectorized::IDataType> get_data_type_of(vectorized::PathInData path,
+                                                                  bool is_nullable,
                                                                   bool ignore_children) const;
 
     // Check is schema read type equals storage column type
@@ -157,8 +165,8 @@ class Segment : public std::enable_shared_from_this<Segment> {
     bool can_apply_predicate_safely(int cid, Predicate* pred, const Schema& schema,
                                     ReaderType read_type) const {
         const Field* col = schema.column(cid);
-        vectorized::DataTypePtr storage_column_type =
-                get_data_type_of(*col, read_type != ReaderType::READER_QUERY);
+        vectorized::DataTypePtr storage_column_type = get_data_type_of(
+                col->path(), col->is_nullable(), read_type != ReaderType::READER_QUERY);
         if (storage_column_type == nullptr) {
             // Default column iterator
             return true;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 483e0e502df8da..f85ed5abce5982 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -342,7 +342,8 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
         const Field* col = _schema->column(i);
         if (col) {
             auto storage_type = _segment->get_data_type_of(
-                    *col, _opts.io_ctx.reader_type != ReaderType::READER_QUERY);
+                    col->path(), col->is_nullable(),
+                    _opts.io_ctx.reader_type != ReaderType::READER_QUERY);
             if (storage_type == nullptr) {
                 storage_type = vectorized::DataTypeFactory::instance().create_data_type(*col);
             }
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 3fca034a18fd39..c953e0b9bcc8ba 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -251,8 +251,8 @@ class SegmentIterator : public RowwiseIterator {
             if (block_cid >= block->columns()) {
                 continue;
             }
-            vectorized::DataTypePtr storage_type =
-                    _segment->get_data_type_of(*_schema->column(cid), false);
+            vectorized::DataTypePtr storage_type = _segment->get_data_type_of(
+                    _schema->column(cid)->path(), _schema->column(cid)->is_nullable(), false);
             if (storage_type && !storage_type->equals(*block->get_by_position(block_cid).type)) {
                 // Do additional cast
                 vectorized::MutableColumnPtr tmp = storage_type->create_column();
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 63682d8c7128f8..78e7e938caae5f 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -554,6 +554,20 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
     }
 }
 
+TabletColumn TabletColumn::create_materialized_variant_column(const std::string& root,
+                                                              const std::vector<std::string>& paths,
+                                                              int32_t parent_unique_id) {
+    TabletColumn subcol;
+    subcol.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
+    subcol.set_is_nullable(true);
+    subcol.set_unique_id(-1);
+    subcol.set_parent_unique_id(parent_unique_id);
+    vectorized::PathInData path(root, paths);
+    subcol.set_path_info(path);
+    subcol.set_name(path.get_path());
+    return subcol;
+}
+
 void TabletColumn::to_schema_pb(ColumnPB* column) const {
     column->set_unique_id(_unique_id);
     column->set_name(_col_name);
diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h
index 3f56568080019b..21970b5cbac5eb 100644
--- a/be/src/olap/tablet_schema.h
+++ b/be/src/olap/tablet_schema.h
@@ -36,6 +36,7 @@
 #include "gutil/stringprintf.h"
 #include "olap/olap_common.h"
 #include "runtime/define_primitive_type.h"
+#include "runtime/descriptors.h"
 #include "util/string_util.h"
 #include "vec/aggregate_functions/aggregate_function.h"
 #include "vec/common/string_utils/string_utils.h"
@@ -91,6 +92,11 @@ class TabletColumn {
                _type == FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE ||
                _type == FieldType::OLAP_FIELD_TYPE_AGG_STATE;
     }
+    // Such columns are not exist in frontend schema info, so we need to
+    // add them into tablet_schema for later column indexing.
+    static TabletColumn create_materialized_variant_column(const std::string& root,
+                                                           const std::vector<std::string>& paths,
+                                                           int32_t parent_unique_id);
     bool has_default_value() const { return _has_default_value; }
     std::string default_value() const { return _default_value; }
     size_t length() const { return _length; }
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
index 721950abbee741..09113f85eea1d3 100644
--- a/be/src/runtime/descriptors.cpp
+++ b/be/src/runtime/descriptors.cpp
@@ -85,6 +85,7 @@ SlotDescriptor::SlotDescriptor(const PSlotDescriptor& pdesc)
           _is_materialized(pdesc.is_materialized()),
           _is_key(pdesc.is_key()),
           _need_materialize(true),
+          _column_paths(pdesc.column_paths().begin(), pdesc.column_paths().end()),
           _is_auto_increment(pdesc.is_auto_increment()) {}
 
 void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const {
@@ -103,6 +104,9 @@ void SlotDescriptor::to_protobuf(PSlotDescriptor* pslot) const {
     pslot->set_is_key(_is_key);
     pslot->set_is_auto_increment(_is_auto_increment);
     pslot->set_col_type(_col_type);
+    for (const std::string& path : _column_paths) {
+        pslot->add_column_paths(path);
+    }
 }
 
 vectorized::MutableColumnPtr SlotDescriptor::get_empty_mutable_column() const {
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index 8d352ba1d53186..bca48737bf508a 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -25,6 +25,7 @@
 #include <butil/errno.h>
 #include <butil/iobuf.h>
 #include <fcntl.h>
+#include <fmt/core.h>
 #include <gen_cpp/MasterService_types.h>
 #include <gen_cpp/PaloInternalService_types.h>
 #include <gen_cpp/PlanNodes_types.h>
@@ -46,6 +47,7 @@
 #include <set>
 #include <sstream>
 #include <string>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -73,6 +75,7 @@
 #include "olap/rowset/segment_v2/common.h"
 #include "olap/rowset/segment_v2/inverted_index_desc.h"
 #include "olap/rowset/segment_v2/segment.h"
+#include "olap/rowset/segment_v2/segment_iterator.h"
 #include "olap/segment_loader.h"
 #include "olap/storage_engine.h"
 #include "olap/tablet.h"
@@ -1711,6 +1714,38 @@ auto scope_timer_run(Func fn, int64_t* cost) -> decltype(fn()) {
     return res;
 }
 
+struct IteratorKey {
+    int64_t tablet_id;
+    RowsetId rowset_id;
+    uint64_t segment_id;
+    int slot_id;
+
+    // unordered map std::equal_to
+    bool operator==(const IteratorKey& rhs) const {
+        return tablet_id == rhs.tablet_id && rowset_id == rhs.rowset_id &&
+               segment_id == rhs.segment_id && slot_id == rhs.slot_id;
+    }
+};
+
+struct HashOfIteratorKey {
+    size_t operator()(const IteratorKey& key) const {
+        size_t seed = 0;
+        seed = HashUtil::hash64(&key.tablet_id, sizeof(key.tablet_id), seed);
+        seed = HashUtil::hash64(&key.rowset_id.hi, sizeof(key.rowset_id.hi), seed);
+        seed = HashUtil::hash64(&key.rowset_id.mi, sizeof(key.rowset_id.mi), seed);
+        seed = HashUtil::hash64(&key.rowset_id.lo, sizeof(key.rowset_id.lo), seed);
+        seed = HashUtil::hash64(&key.segment_id, sizeof(key.segment_id), seed);
+        seed = HashUtil::hash64(&key.slot_id, sizeof(key.slot_id), seed);
+        return seed;
+    }
+};
+
+struct IteratorItem {
+    std::unique_ptr<ColumnIterator> iterator;
+    // for holding the reference of segment to avoid use after release
+    SegmentSharedPtr segment;
+};
+
 Status PInternalServiceImpl::_multi_get(const PMultiGetRequest& request,
                                         PMultiGetResponse* response) {
     OlapReaderStatistics stats;
@@ -1735,6 +1770,7 @@ Status PInternalServiceImpl::_multi_get(const PMultiGetRequest& request,
         full_read_schema.append_column(TabletColumn(column_pb));
     }
 
+    std::unordered_map<IteratorKey, IteratorItem, HashOfIteratorKey> iterator_map;
     // read row by row
     for (size_t i = 0; i < request.row_locs_size(); ++i) {
         const auto& row_loc = request.row_locs(i);
@@ -1773,8 +1809,8 @@ Status PInternalServiceImpl::_multi_get(const PMultiGetRequest& request,
                 },
                 &acquire_segments_ms));
         // find segment
-        auto it = std::find_if(segment_cache.get_segments().begin(),
-                               segment_cache.get_segments().end(),
+        auto it = std::find_if(segment_cache.get_segments().cbegin(),
+                               segment_cache.get_segments().cend(),
                                [&row_loc](const segment_v2::SegmentSharedPtr& seg) {
                                    return seg->id() == row_loc.segment_id();
                                });
@@ -1802,37 +1838,28 @@ Status PInternalServiceImpl::_multi_get(const PMultiGetRequest& request,
         if (result_block.is_empty_column()) {
             result_block = vectorized::Block(desc.slots(), request.row_locs().size());
         }
+        VLOG_DEBUG << "Read row location "
+                   << fmt::format("{}, {}, {}, {}", row_location.tablet_id,
+                                  row_location.row_location.rowset_id.to_string(),
+                                  row_location.row_location.segment_id,
+                                  row_location.row_location.row_id);
         for (int x = 0; x < desc.slots().size(); ++x) {
-            int index = -1;
-            if (desc.slots()[x]->col_unique_id() >= 0) {
-                // light sc enabled
-                index = full_read_schema.field_index(desc.slots()[x]->col_unique_id());
-            } else {
-                index = full_read_schema.field_index(desc.slots()[x]->col_name());
-            }
-            if (index < 0) {
-                std::stringstream ss;
-                ss << "field name is invalid. field=" << desc.slots()[x]->col_name()
-                   << ", field_name_to_index=" << full_read_schema.get_all_field_names();
-                return Status::InternalError(ss.str());
-            }
-            std::unique_ptr<segment_v2::ColumnIterator> column_iterator;
+            auto row_id = static_cast<segment_v2::rowid_t>(row_loc.ordinal_id());
             vectorized::MutableColumnPtr column =
                     result_block.get_by_position(x).column->assume_mutable();
-            StorageReadOptions storage_read_opt;
-            storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
-            RETURN_IF_ERROR(segment->new_column_iterator(full_read_schema.column(index),
-                                                         &column_iterator, &storage_read_opt));
-            segment_v2::ColumnIteratorOptions opt {
-                    .use_page_cache = !config::disable_storage_page_cache,
-                    .file_reader = segment->file_reader().get(),
-                    .stats = &stats,
-                    .io_ctx = io::IOContext {.reader_type = ReaderType::READER_QUERY},
-            };
-            static_cast<void>(column_iterator->init(opt));
-            std::vector<segment_v2::rowid_t> single_row_loc {
-                    static_cast<segment_v2::rowid_t>(row_loc.ordinal_id())};
-            RETURN_IF_ERROR(column_iterator->read_by_rowids(single_row_loc.data(), 1, column));
+            IteratorKey iterator_key {.tablet_id = tablet->tablet_id(),
+                                      .rowset_id = rowset_id,
+                                      .segment_id = row_loc.segment_id(),
+                                      .slot_id = desc.slots()[x]->id()};
+            IteratorItem& iterator_item = iterator_map[iterator_key];
+            if (iterator_item.segment == nullptr) {
+                // hold the reference
+                iterator_map[iterator_key].segment = segment;
+            }
+            segment = iterator_item.segment;
+            RETURN_IF_ERROR(segment->seek_and_read_by_rowid(full_read_schema, desc.slots()[x],
+                                                            row_id, column, stats,
+                                                            iterator_item.iterator));
         }
     }
     // serialize block if not empty
@@ -1852,11 +1879,13 @@ Status PInternalServiceImpl::_multi_get(const PMultiGetRequest& request,
                          "hit_cached_pages:{}, total_pages_read:{}, compressed_bytes_read:{}, "
                          "io_latency:{}ns, "
                          "uncompressed_bytes_read:{},"
+                         "bytes_read:{},"
                          "acquire_tablet_ms:{}, acquire_rowsets_ms:{}, acquire_segments_ms:{}, "
                          "lookup_row_data_ms:{}",
                          stats.cached_pages_num, stats.total_pages_num, stats.compressed_bytes_read,
-                         stats.io_ns, stats.uncompressed_bytes_read, acquire_tablet_ms,
-                         acquire_rowsets_ms, acquire_segments_ms, lookup_row_data_ms);
+                         stats.io_ns, stats.uncompressed_bytes_read, stats.bytes_read,
+                         acquire_tablet_ms, acquire_rowsets_ms, acquire_segments_ms,
+                         lookup_row_data_ms);
     return Status::OK();
 }
 
@@ -1865,6 +1894,7 @@ void PInternalServiceImpl::multiget_data(google::protobuf::RpcController* contro
                                          PMultiGetResponse* response,
                                          google::protobuf::Closure* done) {
     bool ret = _light_work_pool.try_offer([request, response, done, this]() {
+        signal::set_signal_task_id(request->query_id());
         // multi get data by rowid
         MonotonicStopWatch watch;
         watch.start();
diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index f44fc62ed982ea..782a2967f03230 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -645,11 +645,12 @@ void ColumnObject::for_each_subcolumn(ColumnCallback callback) {
 }
 
 void ColumnObject::insert_from(const IColumn& src, size_t n) {
-    const auto& src_v = assert_cast<const ColumnObject&>(src);
+    const auto* src_v = check_and_get_column<ColumnObject>(src);
     // optimize when src and this column are scalar variant, since try_insert is inefficiency
-    if (src_v.is_scalar_variant() && is_scalar_variant() &&
-        src_v.get_root_type()->equals(*get_root_type()) && src_v.is_finalized() && is_finalized()) {
-        assert_cast<ColumnNullable&>(*get_root()).insert_from(*src_v.get_root(), n);
+    if (src_v != nullptr && src_v->is_scalar_variant() && is_scalar_variant() &&
+        src_v->get_root_type()->equals(*get_root_type()) && src_v->is_finalized() &&
+        is_finalized()) {
+        assert_cast<ColumnNullable&>(*get_root()).insert_from(*src_v->get_root(), n);
         ++num_rows;
         return;
     }
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp
index c6e401f3f5484b..5566abec3ab383 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -58,6 +58,7 @@
 #include "vec/exec/scan/new_olap_scan_node.h"
 #include "vec/exec/scan/vscan_node.h"
 #include "vec/exprs/vexpr_context.h"
+#include "vec/json/path_in_data.h"
 #include "vec/olap/block_reader.h"
 
 namespace doris::vectorized {
@@ -411,16 +412,6 @@ Status NewOlapScanner::_init_tablet_reader_params(
     return Status::OK();
 }
 
-vectorized::PathInData NewOlapScanner::_build_path(SlotDescriptor* slot,
-                                                   const std::string& root_name) {
-    PathInDataBuilder path_builder;
-    path_builder.append(root_name, false);
-    for (const std::string& path : slot->column_paths()) {
-        path_builder.append(path, false);
-    }
-    return path_builder.build();
-}
-
 Status NewOlapScanner::_init_variant_columns() {
     auto& tablet_schema = _tablet_reader_params.tablet_schema;
     // Parent column has path info to distinction from each other
@@ -434,16 +425,10 @@ Status NewOlapScanner::_init_variant_columns() {
         if (slot->type().is_variant_type()) {
             // Such columns are not exist in frontend schema info, so we need to
             // add them into tablet_schema for later column indexing.
-            TabletColumn subcol;
-            subcol.set_type(FieldType::OLAP_FIELD_TYPE_VARIANT);
-            subcol.set_is_nullable(true);
-            subcol.set_unique_id(-1);
-            subcol.set_parent_unique_id(slot->col_unique_id());
-            PathInData path = _build_path(
-                    slot, tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case());
-            subcol.set_path_info(path);
-            subcol.set_name(path.get_path());
-            if (tablet_schema->field_index(path) < 0) {
+            TabletColumn subcol = TabletColumn::create_materialized_variant_column(
+                    tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case(),
+                    slot->column_paths(), slot->col_unique_id());
+            if (tablet_schema->field_index(subcol.path_info()) < 0) {
                 tablet_schema->append_column(subcol, TabletSchema::ColumnType::VARIANT);
             }
         }
@@ -465,8 +450,9 @@ Status NewOlapScanner::_init_return_columns() {
         int32_t index = 0;
         auto& tablet_schema = _tablet_reader_params.tablet_schema;
         if (slot->type().is_variant_type()) {
-            index = tablet_schema->field_index(_build_path(
-                    slot, tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case()));
+            index = tablet_schema->field_index(PathInData(
+                    tablet_schema->column_by_uid(slot->col_unique_id()).name_lower_case(),
+                    slot->column_paths()));
         } else {
             index = slot->col_unique_id() >= 0 ? tablet_schema->field_index(slot->col_unique_id())
                                                : tablet_schema->field_index(slot->col_name());
diff --git a/be/src/vec/exec/scan/new_olap_scanner.h b/be/src/vec/exec/scan/new_olap_scanner.h
index ae094b84f0390e..9eab71f3c34eb2 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.h
+++ b/be/src/vec/exec/scan/new_olap_scanner.h
@@ -93,7 +93,6 @@ class NewOlapScanner : public VScanner {
                                       const std::vector<FunctionFilter>& function_filters);
 
     [[nodiscard]] Status _init_return_columns();
-    vectorized::PathInData _build_path(SlotDescriptor* slot, const std::string& root_name);
     [[nodiscard]] Status _init_variant_columns();
 
     std::vector<OlapScanRange*> _key_ranges;
diff --git a/be/src/vec/json/path_in_data.cpp b/be/src/vec/json/path_in_data.cpp
index 1c02febd446946..ae91b44499493c 100644
--- a/be/src/vec/json/path_in_data.cpp
+++ b/be/src/vec/json/path_in_data.cpp
@@ -46,11 +46,22 @@ PathInData::PathInData(const PathInData& other) : path(other.path) {
     build_parts(other.get_parts());
 }
 
+PathInData::PathInData(const std::string& root, const std::vector<std::string>& paths) {
+    PathInDataBuilder path_builder;
+    path_builder.append(root, false);
+    for (const std::string& path : paths) {
+        path_builder.append(path, false);
+    }
+    build_path(path_builder.get_parts());
+    build_parts(path_builder.get_parts());
+}
+
 PathInData::PathInData(const std::vector<std::string>& paths) {
     PathInDataBuilder path_builder;
     for (size_t i = 0; i < paths.size(); ++i) {
         path_builder.append(paths[i], false);
     }
+    build_path(path_builder.get_parts());
     build_parts(path_builder.get_parts());
 }
 
diff --git a/be/src/vec/json/path_in_data.h b/be/src/vec/json/path_in_data.h
index 6531a8bfc6a8a3..267ab1fab3f761 100644
--- a/be/src/vec/json/path_in_data.h
+++ b/be/src/vec/json/path_in_data.h
@@ -61,6 +61,7 @@ class PathInData {
     explicit PathInData(std::string_view path_);
     explicit PathInData(const Parts& parts_);
     explicit PathInData(const std::vector<std::string>& paths);
+    explicit PathInData(const std::string& root, const std::vector<std::string>& paths);
     PathInData(const PathInData& other);
     PathInData& operator=(const PathInData& other);
     static UInt128 get_parts_hash(const Parts& parts_);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 893d4c93239f77..dc3c2f52d812ca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -829,8 +829,7 @@ public boolean checkEnableTwoPhaseRead(Analyzer analyzer) {
             LOG.debug("only support duplicate key or MOW model");
             return false;
         }
-        if (!olapTable.getEnableLightSchemaChange() || !Strings.isNullOrEmpty(olapTable.getStoragePolicy())
-                    || olapTable.hasVariantColumns()) {
+        if (!olapTable.getEnableLightSchemaChange()) {
             return false;
         }
         if (getOrderByElements() != null) {
diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto
index 270199cc0204c7..8762a78c0f4ed1 100644
--- a/gensrc/proto/descriptors.proto
+++ b/gensrc/proto/descriptors.proto
@@ -38,6 +38,7 @@ message PSlotDescriptor {
     optional bool is_key = 12;
     optional bool is_auto_increment = 13;
     optional int32 col_type = 14 [default = 0];
+    repeated string column_paths = 15;
 };
 
 message PTupleDescriptor {
diff --git a/regression-test/data/variant_p0/load.out b/regression-test/data/variant_p0/load.out
index e48aafe2e0d423..7def94d2fadeef 100644
--- a/regression-test/data/variant_p0/load.out
+++ b/regression-test/data/variant_p0/load.out
@@ -143,7 +143,7 @@
 [123]
 
 -- !sql_25 --
-50000	55000.00000000013	6150000
+50000	55000.000000002256	6150000
 
 -- !sql_26 --
 5000
@@ -227,16 +227,16 @@
 \N	\N	\N
 
 -- !sql_36_2 --
-7702	{"payload":{"commits":[{"sha":"348743fdce27d3f3c97e366381b1b7b371fc4510","author":{"email":"9b7a0973fc99779f7e1822eb7336ff5d28bd2653@users.noreply.github.com","name":"Łukasz Magiera"},"message":"Create README.md","distinct":true,"url":"https://api.github.com/repos/magik6k/BitBuffer/commits/348743fdce27d3f3c97e366381b1b7b371fc4510"}],"before":"4e150694dacd35e7d5cda4e9f6a2aedb1d35db36","head":"348743fdce27d3f3c97e366381b1b7b371fc4510","size":1,"push_id":536752118,"ref":"refs/heads/master","distinct_size":1},"created_at":"2015-01-01T00:59:58Z","id":"2489395761","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/magik6k","id":3867941,"login":"magik6k","avatar_url":"https://avatars.githubusercontent.com/u/3867941?"},"repo":{"url":"https://api.github.com/repos/magik6k/BitBuffer","id":28677864,"name":"magik6k/BitBuffer"},"type":"PushEvent"}
-7701	{"payload":{"pages":[{"page_name":"Android Development Basics","title":"Android Development Basics","summary":null,"action":"edited","sha":"e4e947a4f29b1a06f560ac1e62bd3bf183e434b6","html_url":"https://github.com/wllmtrng/wllmtrng.github.io/wiki/Android-Development-Basics"}]},"created_at":"2015-01-01T00:59:58Z","id":"2489395760","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/wllmtrng","id":1335855,"login":"wllmtrng","avatar_url":"https://avatars.githubusercontent.com/u/1335855?"},"repo":{"url":"https://api.github.com/repos/wllmtrng/wllmtrng.github.io","id":18089434,"name":"wllmtrng/wllmtrng.github.io"},"type":"GollumEvent"}
-7700	{"payload":{"forkee":{"svn_url":"https://github.com/WangXYZ/TBC","pushed_at":"2014-12-24T18:26:11Z","issues_url":"https://api.github.com/repos/WangXYZ/TBC/issues{/number}","events_url":"https://api.github.com/repos/WangXYZ/TBC/events","labels_url":"https://api.github.com/repos/WangXYZ/TBC/labels{/name}","releases_url":"https://api.github.com/repos/WangXYZ/TBC/releases{/id}","keys_url":"https://api.github.com/repos/WangXYZ/TBC/keys{/key_id}","stargazers_url":"https://api.github.com/repos/WangXYZ/TBC/stargazers","has_downloads":1,"commits_url":"https://api.github.com/repos/WangXYZ/TBC/commits{/sha}","downloads_url":"https://api.github.com/repos/WangXYZ/TBC/downloads","default_branch":"master","open_issues":0,"size":9207,"forks_count":0,"id":28678211,"has_wiki":0,"owner":{"starred_url":"https://api.github.com/users/WangXYZ/starred{/owner}{/repo}","url":"https://api.github.com/users/WangXYZ","repos_url":"https://api.github.com/users/WangXYZ/repos","events_url":"https://api.github.com/users/WangXYZ/events{/privacy}","login":"WangXYZ","avatar_url":"https://avatars.githubusercontent.com/u/8252171?v=3","gravatar_id":"","site_admin":0,"html_url":"https://github.com/WangXYZ","received_events_url":"https://api.github.com/users/WangXYZ/received_events","followers_url":"https://api.github.com/users/WangXYZ/followers","following_url":"https://api.github.com/users/WangXYZ/following{/other_user}","gists_url":"https://api.github.com/users/WangXYZ/gists{/gist_id}","type":"Organization","subscriptions_url":"https://api.github.com/users/WangXYZ/subscriptions","organizations_url":"https://api.github.com/users/WangXYZ/orgs","id":8252171},"languages_url":"https://api.github.com/repos/WangXYZ/TBC/languages","git_tags_url":"https://api.github.com/repos/WangXYZ/TBC/git/tags{/sha}","archive_url":"https://api.github.com/repos/WangXYZ/TBC/{archive_format}{/ref}","git_refs_url":"https://api.github.com/repos/WangXYZ/TBC/git/refs{/sha}","trees_url":"https://api.github.com/repos/WangXYZ/TBC/git/trees{/sha}","updated_at":"2014-09-10T17:41:40Z","description":"ACID Scripts for CMaNGOS TBC","forks_url":"https://api.github.com/repos/WangXYZ/TBC/forks","hooks_url":"https://api.github.com/repos/WangXYZ/TBC/hooks","created_at":"2015-01-01T00:59:57Z","fork":1,"forks":0,"subscription_url":"https://api.github.com/repos/WangXYZ/TBC/subscription","compare_url":"https://api.github.com/repos/WangXYZ/TBC/compare/{base}...{head}","url":"https://api.github.com/repos/WangXYZ/TBC","collaborators_url":"https://api.github.com/repos/WangXYZ/TBC/collaborators{/collaborator}","statuses_url":"https://api.github.com/repos/WangXYZ/TBC/statuses/{sha}","comments_url":"https://api.github.com/repos/WangXYZ/TBC/comments{/number}","blobs_url":"https://api.github.com/repos/WangXYZ/TBC/git/blobs{/sha}","html_url":"https://github.com/WangXYZ/TBC","watchers_count":0,"has_issues":0,"has_pages":0,"contents_url":"https://api.github.com/repos/WangXYZ/TBC/contents/{+path}","issue_events_url":"https://api.github.com/repos/WangXYZ/TBC/issues/events{/number}","ssh_url":"git@github.com:WangXYZ/TBC.git","tags_url":"https://api.github.com/repos/WangXYZ/TBC/tags","name":"TBC","issue_comment_url":"https://api.github.com/repos/WangXYZ/TBC/issues/comments/{number}","git_url":"git://github.com/WangXYZ/TBC.git","subscribers_url":"https://api.github.com/repos/WangXYZ/TBC/subscribers","clone_url":"https://github.com/WangXYZ/TBC.git","notifications_url":"https://api.github.com/repos/WangXYZ/TBC/notifications{?since,all,participating}","full_name":"WangXYZ/TBC","private":0,"teams_url":"https://api.github.com/repos/WangXYZ/TBC/teams","milestones_url":"https://api.github.com/repos/WangXYZ/TBC/milestones{/number}","public":1,"git_commits_url":"https://api.github.com/repos/WangXYZ/TBC/git/commits{/sha}","open_issues_count":0,"watchers":0,"contributors_url":"https://api.github.com/repos/WangXYZ/TBC/contributors","branches_url":"https://api.github.com/repos/WangXYZ/TBC/branches{/branch}","stargazers_count":0,"pulls_url":"https://api.github.com/repos/WangXYZ/TBC/pulls{/number}","merges_url":"https://api.github.com/repos/WangXYZ/TBC/merges","assignees_url":"https://api.github.com/repos/WangXYZ/TBC/assignees{/user}"}},"created_at":"2015-01-01T00:59:57Z","id":"2489395755","org":{"gravatar_id":"","url":"https://api.github.com/orgs/ACID-Scripts","id":8674587,"login":"ACID-Scripts","avatar_url":"https://avatars.githubusercontent.com/u/8674587?"},"public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/WangXYZ","id":8252171,"login":"WangXYZ","avatar_url":"https://avatars.githubusercontent.com/u/8252171?"},"repo":{"url":"https://api.github.com/repos/ACID-Scripts/TBC","id":23724137,"name":"ACID-Scripts/TBC"},"type":"ForkEvent"}
-7699	{"payload":{"description":"Game using the MS Kinect that scans the user and creates a 3D version of them.","ref":"master","ref_type":"branch","pusher_type":"user","master_branch":"master"},"created_at":"2015-01-01T00:59:57Z","id":"2489395752","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/chrisjimenez","id":3580593,"login":"chrisjimenez","avatar_url":"https://avatars.githubusercontent.com/u/3580593?"},"repo":{"url":"https://api.github.com/repos/chrisjimenez/IpsePuppet","id":28678128,"name":"chrisjimenez/IpsePuppet"},"type":"CreateEvent"}
-7698	{"payload":{"issue":{"url":"https://api.github.com/repos/antonioortegajr/beerfind.me/issues/12","created_at":"2015-01-01T00:59:56Z","body":"The bee that are being searched for currently should also have logos and descriptions from the API.","events_url":"https://api.github.com/repos/antonioortegajr/beerfind.me/issues/12/events","labels_url":"https://api.github.com/repos/antonioortegajr/beerfind.me/issues/12/labels{/name}","locked":0,"state":"open","comments_url":"https://api.github.com/repos/antonioortegajr/beerfind.me/issues/12/comments","user":{"starred_url":"https://api.github.com/users/antonioortegajr/starred{/owner}{/repo}","url":"https://api.github.com/users/antonioortegajr","repos_url":"https://api.github.com/users/antonioortegajr/repos","events_url":"https://api.github.com/users/antonioortegajr/events{/privacy}","login":"antonioortegajr","avatar_url":"https://avatars.githubusercontent.com/u/6744175?v=3","gravatar_id":"","site_admin":0,"html_url":"https://github.com/antonioortegajr","received_events_url":"https://api.github.com/users/antonioortegajr/received_events","followers_url":"https://api.github.com/users/antonioortegajr/followers","following_url":"https://api.github.com/users/antonioortegajr/following{/other_user}","gists_url":"https://api.github.com/users/antonioortegajr/gists{/gist_id}","type":"User","subscriptions_url":"https://api.github.com/users/antonioortegajr/subscriptions","organizations_url":"https://api.github.com/users/antonioortegajr/orgs","id":6744175},"title":"add logos and descriptions to beers being searched","id":53210166,"number":12,"comments":0,"updated_at":"2015-01-01T00:59:56Z","html_url":"https://github.com/antonioortegajr/beerfind.me/issues/12"},"action":"opened"},"created_at":"2015-01-01T00:59:56Z","id":"2489395749","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/antonioortegajr","id":6744175,"login":"antonioortegajr","avatar_url":"https://avatars.githubusercontent.com/u/6744175?"},"repo":{"url":"https://api.github.com/repos/antonioortegajr/beerfind.me","id":28573267,"name":"antonioortegajr/beerfind.me"},"type":"IssuesEvent"}
-7697	{"payload":{"action":"closed","pull_request":{"review_comments_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/534/comments","url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/534","body":"","patch_url":"https://github.com/XLabs/Xamarin-Forms-Labs/pull/534.patch","head":{"ref":"master","user":{"starred_url":"https://api.github.com/users/bokmadsen/starred{/owner}{/repo}","url":"https://api.github.com/users/bokmadsen","repos_url":"https://api.github.com/users/bokmadsen/repos","events_url":"https://api.github.com/users/bokmadsen/events{/privacy}","login":"bokmadsen","avatar_url":"https://avatars.githubusercontent.com/u/790740?v=3","gravatar_id":"","site_admin":0,"html_url":"https://github.com/bokmadsen","received_events_url":"https://api.github.com/users/bokmadsen/received_events","followers_url":"https://api.github.com/users/bokmadsen/followers","following_url":"https://api.github.com/users/bokmadsen/following{/other_user}","gists_url":"https://api.github.com/users/bokmadsen/gists{/gist_id}","type":"User","subscriptions_url":"https://api.github.com/users/bokmadsen/subscriptions","organizations_url":"https://api.github.com/users/bokmadsen/orgs","id":790740},"sha":"2a664fd4ac7e8ff340893ad5e4c2f50127ee52c6","label":"bokmadsen:master","repo":{"svn_url":"https://github.com/bokmadsen/Xamarin-Forms-Labs","pushed_at":"2014-12-21T16:59:46Z","url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs","issues_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/issues{/number}","collaborators_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/collaborators{/collaborator}","events_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/events","has_downloads":1,"labels_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/labels{/name}","keys_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/keys{/key_id}","releases_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/releases{/id}","stargazers_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/stargazers","statuses_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/statuses/{sha}","comments_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/comments{/number}","blobs_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/git/blobs{/sha}","html_url":"https://github.com/bokmadsen/Xamarin-Forms-Labs","has_pages":1,"commits_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/commits{/sha}","downloads_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/downloads","contents_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/contents/{+path}","has_issues":0,"issue_events_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/issues/events{/number}","ssh_url":"git@github.com:bokmadsen/Xamarin-Forms-Labs.git","watchers_count":0,"default_branch":"master","open_issues":0,"size":96455,"forks_count":0,"tags_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/tags","id":28305437,"name":"Xamarin-Forms-Labs","has_wiki":1,"issue_comment_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/issues/comments/{number}","owner":{"starred_url":"https://api.github.com/users/bokmadsen/starred{/owner}{/repo}","url":"https://api.github.com/users/bokmadsen","repos_url":"https://api.github.com/users/bokmadsen/repos","events_url":"https://api.github.com/users/bokmadsen/events{/privacy}","login":"bokmadsen","avatar_url":"https://avatars.githubusercontent.com/u/790740?v=3","gravatar_id":"","site_admin":0,"html_url":"https://github.com/bokmadsen","received_events_url":"https://api.github.com/users/bokmadsen/received_events","followers_url":"https://api.github.com/users/bokmadsen/followers","following_url":"https://api.github.com/users/bokmadsen/following{/other_user}","gists_url":"https://api.github.com/users/bokmadsen/gists{/gist_id}","type":"User","subscriptions_url":"https://api.github.com/users/bokmadsen/subscriptions","organizations_url":"https://api.github.com/users/bokmadsen/orgs","id":790740},"languages_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/languages","git_url":"git://github.com/bokmadsen/Xamarin-Forms-Labs.git","subscribers_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/subscribers","clone_url":"https://github.com/bokmadsen/Xamarin-Forms-Labs.git","notifications_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/notifications{?since,all,participating}","git_tags_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/git/tags{/sha}","archive_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/{archive_format}{/ref}","full_name":"bokmadsen/Xamarin-Forms-Labs","private":0,"teams_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/teams","milestones_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/milestones{/number}","git_refs_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/git/refs{/sha}","git_commits_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/git/commits{/sha}","trees_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/git/trees{/sha}","updated_at":"2014-12-21T16:59:46Z","description":"Xamarin Forms Labs is a open source project that aims to provide a powerful and cross platform set of controls and helpers tailored to work with Xamarin Forms.","forks_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/forks","open_issues_count":0,"hooks_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/hooks","watchers":0,"created_at":"2014-12-21T16:32:52Z","contributors_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/contributors","branches_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/branches{/branch}","stargazers_count":0,"fork":1,"forks":0,"compare_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/compare/{base}...{head}","pulls_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/pulls{/number}","subscription_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/subscription","merges_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/merges","assignees_url":"https://api.github.com/repos/bokmadsen/Xamarin-Forms-Labs/assignees{/user}","language":"C#","homepage":""}},"issue_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/issues/534","statuses_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/statuses/2a664fd4ac7e8ff340893ad5e4c2f50127ee52c6","comments_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/issues/534/comments","diff_url":"https://github.com/XLabs/Xamarin-Forms-Labs/pull/534.diff","_links":{"self":{"href":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/534"},"commits":{"href":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/534/commits"},"review_comments":{"href":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/534/comments"},"statuses":{"href":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/statuses/2a664fd4ac7e8ff340893ad5e4c2f50127ee52c6"},"comments":{"href":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/issues/534/comments"},"html":{"href":"https://github.com/XLabs/Xamarin-Forms-Labs/pull/534"},"issue":{"href":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/issues/534"},"review_comment":{"href":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/comments/{number}"}},"title":"Added null check in HybridWebView","base":{"ref":"master","user":{"starred_url":"https://api.github.com/users/XLabs/starred{/owner}{/repo}","url":"https://api.github.com/users/XLabs","repos_url":"https://api.github.com/users/XLabs/repos","events_url":"https://api.github.com/users/XLabs/events{/privacy}","login":"XLabs","avatar_url":"https://avatars.githubusercontent.com/u/7787062?v=3","gravatar_id":"","site_admin":0,"html_url":"https://github.com/XLabs","received_events_url":"https://api.github.com/users/XLabs/received_events","followers_url":"https://api.github.com/users/XLabs/followers","following_url":"https://api.github.com/users/XLabs/following{/other_user}","gists_url":"https://api.github.com/users/XLabs/gists{/gist_id}","type":"Organization","subscriptions_url":"https://api.github.com/users/XLabs/subscriptions","organizations_url":"https://api.github.com/users/XLabs/orgs","id":7787062},"sha":"6906dd38ff69debcc304cb05b6877fae71747acd","label":"XLabs:master","repo":{"svn_url":"https://github.com/XLabs/Xamarin-Forms-Labs","pushed_at":"2015-01-01T00:59:12Z","url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs","issues_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/issues{/number}","collaborators_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/collaborators{/collaborator}","events_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/events","has_downloads":1,"labels_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/labels{/name}","keys_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/keys{/key_id}","releases_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/releases{/id}","stargazers_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/stargazers","statuses_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/statuses/{sha}","comments_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/comments{/number}","blobs_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/git/blobs{/sha}","html_url":"https://github.com/XLabs/Xamarin-Forms-Labs","has_pages":1,"commits_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/commits{/sha}","downloads_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/downloads","contents_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/contents/{+path}","has_issues":1,"issue_events_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/issues/events{/number}","ssh_url":"git@github.com:XLabs/Xamarin-Forms-Labs.git","watchers_count":340,"default_branch":"master","open_issues":92,"size":104805,"forks_count":210,"tags_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/tags","id":20463939,"name":"Xamarin-Forms-Labs","has_wiki":1,"issue_comment_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/issues/comments/{number}","owner":{"starred_url":"https://api.github.com/users/XLabs/starred{/owner}{/repo}","url":"https://api.github.com/users/XLabs","repos_url":"https://api.github.com/users/XLabs/repos","events_url":"https://api.github.com/users/XLabs/events{/privacy}","login":"XLabs","avatar_url":"https://avatars.githubusercontent.com/u/7787062?v=3","gravatar_id":"","site_admin":0,"html_url":"https://github.com/XLabs","received_events_url":"https://api.github.com/users/XLabs/received_events","followers_url":"https://api.github.com/users/XLabs/followers","following_url":"https://api.github.com/users/XLabs/following{/other_user}","gists_url":"https://api.github.com/users/XLabs/gists{/gist_id}","type":"Organization","subscriptions_url":"https://api.github.com/users/XLabs/subscriptions","organizations_url":"https://api.github.com/users/XLabs/orgs","id":7787062},"languages_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/languages","git_url":"git://github.com/XLabs/Xamarin-Forms-Labs.git","subscribers_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/subscribers","clone_url":"https://github.com/XLabs/Xamarin-Forms-Labs.git","notifications_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/notifications{?since,all,participating}","git_tags_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/git/tags{/sha}","archive_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/{archive_format}{/ref}","full_name":"XLabs/Xamarin-Forms-Labs","private":0,"teams_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/teams","milestones_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/milestones{/number}","git_refs_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/git/refs{/sha}","git_commits_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/git/commits{/sha}","trees_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/git/trees{/sha}","updated_at":"2015-01-01T00:59:13Z","description":"Xamarin Forms Labs is a open source project that aims to provide a powerful and cross platform set of controls and helpers tailored to work with Xamarin Forms.","forks_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/forks","open_issues_count":92,"hooks_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/hooks","watchers":340,"created_at":"2014-06-03T23:53:11Z","contributors_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/contributors","branches_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/branches{/branch}","stargazers_count":340,"fork":0,"forks":210,"compare_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/compare/{base}...{head}","pulls_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls{/number}","subscription_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/subscription","merges_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/merges","assignees_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/assignees{/user}","language":"C#","homepage":""}},"updated_at":"2015-01-01T00:59:55Z","html_url":"https://github.com/XLabs/Xamarin-Forms-Labs/pull/534","commits_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/534/commits","created_at":"2014-12-21T17:00:20Z","locked":0,"state":"closed","user":{"starred_url":"https://api.github.com/users/bokmadsen/starred{/owner}{/repo}","url":"https://api.github.com/users/bokmadsen","repos_url":"https://api.github.com/users/bokmadsen/repos","events_url":"https://api.github.com/users/bokmadsen/events{/privacy}","login":"bokmadsen","avatar_url":"https://avatars.githubusercontent.com/u/790740?v=3","gravatar_id":"","site_admin":0,"html_url":"https://github.com/bokmadsen","received_events_url":"https://api.github.com/users/bokmadsen/received_events","followers_url":"https://api.github.com/users/bokmadsen/followers","following_url":"https://api.github.com/users/bokmadsen/following{/other_user}","gists_url":"https://api.github.com/users/bokmadsen/gists{/gist_id}","type":"User","subscriptions_url":"https://api.github.com/users/bokmadsen/subscriptions","organizations_url":"https://api.github.com/users/bokmadsen/orgs","id":790740},"id":26429779,"number":534,"review_comment_url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs/pulls/comments/{number}","additions":16,"changed_files":1,"mergeable_state":"clean","review_comments":0,"deletions":14,"merged":0,"merge_commit_sha":"9cda5a97584b1ad549fff2dcaaf304adeb27ae07","commits":1,"closed_at":"2015-01-01T00:59:55Z","mergeable":1,"comments":0},"number":534},"created_at":"2015-01-01T00:59:55Z","id":"2489395745","org":{"gravatar_id":"","url":"https://api.github.com/orgs/XLabs","id":7787062,"login":"XLabs","avatar_url":"https://avatars.githubusercontent.com/u/7787062?"},"public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/rmarinho","id":1235097,"login":"rmarinho","avatar_url":"https://avatars.githubusercontent.com/u/1235097?"},"repo":{"url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs","id":20463939,"name":"XLabs/Xamarin-Forms-Labs"},"type":"PullRequestEvent"}
-7696	{"payload":{"commits":[{"sha":"c356eac8fa4409a1aa794ab07244250a862da03b","author":{"email":"de8898f6c55e335aa0a2b937fae65fb756ee038f@gmail.com","name":"Zaryafaraj"},"message":"reserve modal styles","distinct":true,"url":"https://api.github.com/repos/Fathalian/Guild/commits/c356eac8fa4409a1aa794ab07244250a862da03b"},{"sha":"9a773fc648910c7a2499401f44a6e5f71eb30460","author":{"email":"de8898f6c55e335aa0a2b937fae65fb756ee038f@gmail.com","name":"Zaryafaraj"},"message":"Merge branch 'master' of https://github.com/Fathalian/Guild\\n\\nConflicts:\\n\\tapp/templates/reserveModal.html","distinct":true,"url":"https://api.github.com/repos/Fathalian/Guild/commits/9a773fc648910c7a2499401f44a6e5f71eb30460"}],"before":"da6d28eba11c89836e132bbba032c22d92e4f233","head":"9a773fc648910c7a2499401f44a6e5f71eb30460","size":2,"push_id":536752114,"ref":"refs/heads/master","distinct_size":2},"created_at":"2015-01-01T00:59:55Z","id":"2489395744","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/Zaryafaraj","id":1356088,"login":"Zaryafaraj","avatar_url":"https://avatars.githubusercontent.com/u/1356088?"},"repo":{"url":"https://api.github.com/repos/Fathalian/Guild","id":26995510,"name":"Fathalian/Guild"},"type":"PushEvent"}
-7695	{"payload":{"commits":[{"sha":"17cf9ea07662d74b2d5bac1cf976f5853a63920d","author":{"email":"af59d1d6805404937849f05dafd5a911888fd7a4@gmail.com","name":"Kevin Hofmaenner"},"message":"minor UI tweak","distinct":true,"url":"https://api.github.com/repos/kevinhofmaenner/blackjack/commits/17cf9ea07662d74b2d5bac1cf976f5853a63920d"}],"before":"6b4f5af0a2a70bbe00cd88281823b436d506ff2d","head":"17cf9ea07662d74b2d5bac1cf976f5853a63920d","size":1,"push_id":536752112,"ref":"refs/heads/master","distinct_size":1},"created_at":"2015-01-01T00:59:55Z","id":"2489395742","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/kevinhofmaenner","id":10161858,"login":"kevinhofmaenner","avatar_url":"https://avatars.githubusercontent.com/u/10161858?"},"repo":{"url":"https://api.github.com/repos/kevinhofmaenner/blackjack","id":28652857,"name":"kevinhofmaenner/blackjack"},"type":"PushEvent"}
-7694	{"payload":{"commits":[{"sha":"aa8ec0de017c8003758776739facc819e33ac7c9","author":{"email":"e0e04a2320844b42511db0376599e166ab5bda54@gmail.com","name":"Runhang Li"},"message":"finish all hamms test","distinct":true,"url":"https://api.github.com/repos/marklrh/ocaml-cohttp-test/commits/aa8ec0de017c8003758776739facc819e33ac7c9"}],"before":"2bb795fc30fc15ab85bcc10f894bfcfa118d69bc","head":"aa8ec0de017c8003758776739facc819e33ac7c9","size":1,"push_id":536752109,"ref":"refs/heads/master","distinct_size":1},"created_at":"2015-01-01T00:59:53Z","id":"2489395735","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/marklrh","id":3656079,"login":"marklrh","avatar_url":"https://avatars.githubusercontent.com/u/3656079?"},"repo":{"url":"https://api.github.com/repos/marklrh/ocaml-cohttp-test","id":27470715,"name":"marklrh/ocaml-cohttp-test"},"type":"PushEvent"}
-7693	{"payload":{"commits":[{"sha":"0b27989723feb4b183d5f87813fef146b670b1d1","author":{"email":"7c5a0c567b5584a13fde407456875318a5bec977@gmail.com","name":"Raphaël Benitte"},"message":"Add time clock widget + Improve stylus theming","distinct":true,"url":"https://api.github.com/repos/plouc/mozaik/commits/0b27989723feb4b183d5f87813fef146b670b1d1"}],"before":"7ddf17eb74fff5adad6e2feb72bcb627d4644800","head":"0b27989723feb4b183d5f87813fef146b670b1d1","size":1,"push_id":536752104,"ref":"refs/heads/master","distinct_size":1},"created_at":"2015-01-01T00:59:51Z","id":"2489395728","public":1,"actor":{"gravatar_id":"","url":"https://api.github.com/users/plouc","id":501642,"login":"plouc","avatar_url":"https://avatars.githubusercontent.com/u/501642?"},"repo":{"url":"https://api.github.com/repos/plouc/mozaik","id":28498113,"name":"plouc/mozaik"},"type":"PushEvent"}
+7702	{"url":"https://api.github.com/repos/magik6k/BitBuffer","id":28677864,"name":"magik6k/BitBuffer"}
+7701	{"url":"https://api.github.com/repos/wllmtrng/wllmtrng.github.io","id":18089434,"name":"wllmtrng/wllmtrng.github.io"}
+7700	{"url":"https://api.github.com/repos/ACID-Scripts/TBC","id":23724137,"name":"ACID-Scripts/TBC"}
+7699	{"url":"https://api.github.com/repos/chrisjimenez/IpsePuppet","id":28678128,"name":"chrisjimenez/IpsePuppet"}
+7698	{"url":"https://api.github.com/repos/antonioortegajr/beerfind.me","id":28573267,"name":"antonioortegajr/beerfind.me"}
+7697	{"url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs","id":20463939,"name":"XLabs/Xamarin-Forms-Labs"}
+7696	{"url":"https://api.github.com/repos/Fathalian/Guild","id":26995510,"name":"Fathalian/Guild"}
+7695	{"url":"https://api.github.com/repos/kevinhofmaenner/blackjack","id":28652857,"name":"kevinhofmaenner/blackjack"}
+7694	{"url":"https://api.github.com/repos/marklrh/ocaml-cohttp-test","id":27470715,"name":"marklrh/ocaml-cohttp-test"}
+7693	{"url":"https://api.github.com/repos/plouc/mozaik","id":28498113,"name":"plouc/mozaik"}
 
 -- !sql_36_3 --
 2	{"updated_value":10}
diff --git a/regression-test/data/variant_p0/sql/gh_data.out b/regression-test/data/variant_p0/sql/gh_data.out
index 7fc5e10dbe180d..4c34edabe48205 100644
--- a/regression-test/data/variant_p0/sql/gh_data.out
+++ b/regression-test/data/variant_p0/sql/gh_data.out
@@ -3,31 +3,37 @@
 0
 
 -- !gh_data_2 --
-5000
+0
 
 -- !gh_data_3 --
+0
+
+-- !gh_data_4 --
+5000
+
+-- !gh_data_5 --
 leonardomso/33-js-concepts	3
 ytdl-org/youtube-dl	3
 Bogdanp/neko	2
 bminossi/AllVideoPocsFromHackerOne	2
 disclose/diodata	2
 
--- !gh_data_4 --
+-- !gh_data_6 --
 14690758274
 
--- !gh_data_5 --
+-- !gh_data_7 --
 73453762334584
 
--- !gh_data_6 --
+-- !gh_data_8 --
 457806339
 
--- !gh_data_7 --
+-- !gh_data_9 --
 0
 
--- !gh_data_8 --
+-- !gh_data_10 --
 19829
 
--- !gh_data_9 --
+-- !gh_data_11 --
 49390617
 64890096
 10696700
@@ -39,19 +45,19 @@ disclose/diodata	2
 42386044
 73801003
 
--- !gh_data_10 --
-27	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746717","public":1,"actor":{"gravatar_id":"","display_login":"sergdudnik","url":"https://api.github.com/users/sergdudnik","id":16341546,"login":"sergdudnik","avatar_url":"https://avatars.githubusercontent.com/u/16341546?"},"repo":{"url":"https://api.github.com/repos/leonardomso/33-js-concepts","id":147350463,"name":"leonardomso/33-js-concepts"},"type":"WatchEvent"}
-36	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746732","public":1,"actor":{"gravatar_id":"","display_login":"juliusHuelsmann","url":"https://api.github.com/users/juliusHuelsmann","id":9212314,"login":"juliusHuelsmann","avatar_url":"https://avatars.githubusercontent.com/u/9212314?"},"repo":{"url":"https://api.github.com/repos/odeke-em/drive","id":26109545,"name":"odeke-em/drive"},"type":"WatchEvent"}
-46	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:27Z","id":"14690746749","org":{"gravatar_id":"","url":"https://api.github.com/orgs/GO-LiFE","id":38434522,"login":"GO-LiFE","avatar_url":"https://avatars.githubusercontent.com/u/38434522?"},"public":1,"actor":{"gravatar_id":"","display_login":"okbean","url":"https://api.github.com/users/okbean","id":75969386,"login":"okbean","avatar_url":"https://avatars.githubusercontent.com/u/75969386?"},"repo":{"url":"https://api.github.com/repos/GO-LiFE/GoFIT_SDK_Android","id":141905736,"name":"GO-LiFE/GoFIT_SDK_Android"},"type":"WatchEvent"}
-56	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:28Z","id":"14690746773","public":1,"actor":{"gravatar_id":"","display_login":"PWDream","url":"https://api.github.com/users/PWDream","id":4903755,"login":"PWDream","avatar_url":"https://avatars.githubusercontent.com/u/4903755?"},"repo":{"url":"https://api.github.com/repos/MrXujiang/h5-Dooring","id":289417971,"name":"MrXujiang/h5-Dooring"},"type":"WatchEvent"}
-86	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746843","public":1,"actor":{"gravatar_id":"","display_login":"Gui-Yom","url":"https://api.github.com/users/Gui-Yom","id":25181283,"login":"Gui-Yom","avatar_url":"https://avatars.githubusercontent.com/u/25181283?"},"repo":{"url":"https://api.github.com/repos/redsaph/cleartext","id":106453399,"name":"redsaph/cleartext"},"type":"WatchEvent"}
-98	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746866","org":{"gravatar_id":"","url":"https://api.github.com/orgs/sherlock-project","id":48293496,"login":"sherlock-project","avatar_url":"https://avatars.githubusercontent.com/u/48293496?"},"public":1,"actor":{"gravatar_id":"","display_login":"humaidk2","url":"https://api.github.com/users/humaidk2","id":12982026,"login":"humaidk2","avatar_url":"https://avatars.githubusercontent.com/u/12982026?"},"repo":{"url":"https://api.github.com/repos/sherlock-project/sherlock","id":162998479,"name":"sherlock-project/sherlock"},"type":"WatchEvent"}
-101	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:29Z","id":"14690746870","public":1,"actor":{"gravatar_id":"","display_login":"hasantezcan","url":"https://api.github.com/users/hasantezcan","id":32804505,"login":"hasantezcan","avatar_url":"https://avatars.githubusercontent.com/u/32804505?"},"repo":{"url":"https://api.github.com/repos/okandavut/react-spotify-nowplaying","id":326215605,"name":"okandavut/react-spotify-nowplaying"},"type":"WatchEvent"}
-112	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:30Z","id":"14690746899","public":1,"actor":{"gravatar_id":"","display_login":"nicholas-robertson","url":"https://api.github.com/users/nicholas-robertson","id":17681331,"login":"nicholas-robertson","avatar_url":"https://avatars.githubusercontent.com/u/17681331?"},"repo":{"url":"https://api.github.com/repos/sentriz/gonic","id":178435468,"name":"sentriz/gonic"},"type":"WatchEvent"}
-122	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:30Z","id":"14690746914","org":{"gravatar_id":"","url":"https://api.github.com/orgs/netlify-labs","id":47546088,"login":"netlify-labs","avatar_url":"https://avatars.githubusercontent.com/u/47546088?"},"public":1,"actor":{"gravatar_id":"","display_login":"javaniecampbell","url":"https://api.github.com/users/javaniecampbell","id":1676496,"login":"javaniecampbell","avatar_url":"https://avatars.githubusercontent.com/u/1676496?"},"repo":{"url":"https://api.github.com/repos/netlify-labs/react-netlify-identity-widget","id":182606378,"name":"netlify-labs/react-netlify-identity-widget"},"type":"WatchEvent"}
-169	{"payload":{"action":"started"},"created_at":"2021-01-02T16:37:32Z","id":"14690747028","org":{"gravatar_id":"","url":"https://api.github.com/orgs/microsoft","id":6154722,"login":"microsoft","avatar_url":"https://avatars.githubusercontent.com/u/6154722?"},"public":1,"actor":{"gravatar_id":"","display_login":"Yxnt","url":"https://api.github.com/users/Yxnt","id":10323352,"login":"Yxnt","avatar_url":"https://avatars.githubusercontent.com/u/10323352?"},"repo":{"url":"https://api.github.com/repos/microsoft/BotBuilder-Samples","id":68730444,"name":"microsoft/BotBuilder-Samples"},"type":"WatchEvent"}
+-- !gh_data_12 --
+27	{"url":"https://api.github.com/repos/leonardomso/33-js-concepts","id":147350463,"name":"leonardomso/33-js-concepts"}
+36	{"url":"https://api.github.com/repos/odeke-em/drive","id":26109545,"name":"odeke-em/drive"}
+46	{"url":"https://api.github.com/repos/GO-LiFE/GoFIT_SDK_Android","id":141905736,"name":"GO-LiFE/GoFIT_SDK_Android"}
+56	{"url":"https://api.github.com/repos/MrXujiang/h5-Dooring","id":289417971,"name":"MrXujiang/h5-Dooring"}
+86	{"url":"https://api.github.com/repos/redsaph/cleartext","id":106453399,"name":"redsaph/cleartext"}
+98	{"url":"https://api.github.com/repos/sherlock-project/sherlock","id":162998479,"name":"sherlock-project/sherlock"}
+101	{"url":"https://api.github.com/repos/okandavut/react-spotify-nowplaying","id":326215605,"name":"okandavut/react-spotify-nowplaying"}
+112	{"url":"https://api.github.com/repos/sentriz/gonic","id":178435468,"name":"sentriz/gonic"}
+122	{"url":"https://api.github.com/repos/netlify-labs/react-netlify-identity-widget","id":182606378,"name":"netlify-labs/react-netlify-identity-widget"}
+169	{"url":"https://api.github.com/repos/microsoft/BotBuilder-Samples","id":68730444,"name":"microsoft/BotBuilder-Samples"}
 
--- !gh_data_11 --
+-- !gh_data_13 --
 2051941	1
 10696700	1
 32271952	2
@@ -63,3 +69,27 @@ disclose/diodata	2
 64890096	1
 73801003	1
 
+-- !gh_data_14 --
+27	14690746717	WatchEvent	leonardomso/33-js-concepts
+36	14690746732	WatchEvent	odeke-em/drive
+46	14690746749	WatchEvent	GO-LiFE/GoFIT_SDK_Android
+56	14690746773	WatchEvent	MrXujiang/h5-Dooring
+86	14690746843	WatchEvent	redsaph/cleartext
+98	14690746866	WatchEvent	sherlock-project/sherlock
+101	14690746870	WatchEvent	okandavut/react-spotify-nowplaying
+112	14690746899	WatchEvent	sentriz/gonic
+122	14690746914	WatchEvent	netlify-labs/react-netlify-identity-widget
+169	14690747028	WatchEvent	microsoft/BotBuilder-Samples
+
+-- !gh_data_15 --
+user
+user
+user
+user
+user
+user
+user
+user
+user
+user
+
diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy
index 0501d3ce52fa5a..b23ae19d12f1a2 100644
--- a/regression-test/suites/variant_p0/load.groovy
+++ b/regression-test/suites/variant_p0/load.groovy
@@ -287,7 +287,7 @@ suite("regression_test_variant", "variant_type"){
         // 12. streamload remote file
         table_name = "logdata"
         create_table.call(table_name, "4")
-        sql "set enable_two_phase_read_opt = false;"
+        // sql "set enable_two_phase_read_opt = false;"
         // no sparse columns
         set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
         load_json_data.call(table_name, """${getS3Url() + '/load/logdata.json'}""")
@@ -340,7 +340,7 @@ suite("regression_test_variant", "variant_type"){
         qt_sql_36_1 "select cast(v:a as int), cast(v:b as int), cast(v:c as int) from ${table_name} order by k limit 10"
         sql "DELETE FROM ${table_name} WHERE k=1"
         sql "select * from ${table_name}"
-        qt_sql_36_2 "select * from ${table_name} where k > 3 order by k desc limit 10"
+        qt_sql_36_2 """select k, json_extract(cast(v as text), "\$.repo") from ${table_name} where k > 3 order by k desc limit 10"""
         sql "insert into ${table_name} select * from ${table_name}"
         sql """UPDATE ${table_name} set v = '{"updated_value" : 10}' where k = 2"""
         qt_sql_36_3 """select * from ${table_name} where k = 2"""
@@ -386,13 +386,13 @@ suite("regression_test_variant", "variant_type"){
         sql """insert into ${table_name} values (2, "abe", '{"c" : 1}')"""
         sql """insert into ${table_name} values (3, "abd", '{"d" : 1}')"""
         sql "delete from ${table_name} where k in (select k from variant_mow where k in (1, 2))"
-        qt_sql_38 "select * from ${table_name} order by k"
+        qt_sql_38 "select * from ${table_name} order by k limit 10"
 
         // read text from sparse col
         set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
         sql """insert into  sparse_columns select 0, '{"a": 1123, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" : {"akakaka" : null, "xxxx" : {"xxx" : 123}}}'  as json_str
             union  all select 0, '{"a" : 1234, "xxxx" : "kaana", "ddd" : {"aaa" : 123, "mxmxm" : [456, "789"]}}' as json_str from numbers("number" = "4096") limit 4096 ;"""
-        qt_sql_31 """select cast(v:xxxx as string) from sparse_columns where cast(v:xxxx as string) != 'null' limit 1;"""
+        qt_sql_31 """select cast(v:xxxx as string) from sparse_columns where cast(v:xxxx as string) != 'null' order by k limit 1;"""
         sql "truncate table sparse_columns"
         set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
     } finally {
diff --git a/regression-test/suites/variant_p0/sql/gh_data.sql b/regression-test/suites/variant_p0/sql/gh_data.sql
index 72a3adb01cff43..2fc0ccac14a629 100644
--- a/regression-test/suites/variant_p0/sql/gh_data.sql
+++ b/regression-test/suites/variant_p0/sql/gh_data.sql
@@ -1,4 +1,6 @@
 set exec_mem_limit=8G;
+set enable_two_phase_read_opt = true;
+set topn_opt_limit_threshold = 1024; 
 SELECT count() from ghdata;
 SELECT cast(v:repo.name as string), count() AS stars FROM ghdata WHERE cast(v:type as string) = 'WatchEvent' GROUP BY cast(v:repo.name as string)  ORDER BY stars DESC, cast(v:repo.name as string) LIMIT 5;
 SELECT max(cast(cast(v:`id` as string) as bigint)) FROM ghdata;
@@ -6,8 +8,9 @@ SELECT sum(cast(cast(v:`id` as string) as bigint)) FROM ghdata;
 SELECT sum(cast(v:payload.member.id as bigint)) FROM ghdata;
 SELECT sum(cast(v:payload.pull_request.milestone.creator.site_admin as bigint)) FROM ghdata;
 SELECT sum(length(v:payload.pull_request.base.repo.html_url)) FROM ghdata;
--- SELECT v:payload.commits.author.name FROM ghdata ORDER BY k LIMIT 10;
 SELECT v:payload.member.id FROM ghdata where cast(v:payload.member.id as string) is not null  ORDER BY k LIMIT 10;
 -- select k, v:payload.commits.author.name AS name, e FROM ghdata as t lateral view  explode(cast(v:payload.commits.author.name as array<string>)) tm1 as e  order by k limit 5;
-select k, v from ghdata WHERE cast(v:type as string) = 'WatchEvent'  order by k limit 10;
-SELECT cast(v:payload.member.id as bigint), count() FROM ghdata where cast(v:payload.member.id as bigint) is not null group by cast(v:payload.member.id as bigint) order by 1, 2 desc LIMIT 10;
\ No newline at end of file
+select k, json_extract(v, '$.repo') from ghdata WHERE cast(v:type as string) = 'WatchEvent'  order by k limit 10;
+SELECT cast(v:payload.member.id as bigint), count() FROM ghdata where cast(v:payload.member.id as bigint) is not null group by cast(v:payload.member.id as bigint) order by 1, 2 desc LIMIT 10;
+select k, cast(v:`id` as string), cast(v:type as string), cast(v:repo.name as string) from ghdata WHERE cast(v:type as string) = 'WatchEvent'  order by k limit 10;
+SELECT cast(v:payload.pusher_type as text) FROM ghdata where cast(v:payload.pusher_type as text) is not null ORDER BY k LIMIT 10;
\ No newline at end of file

From 1d984e0ebb1010f58b860c3089254053566d5b13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= <jianxie0@gmail.com>
Date: Mon, 25 Dec 2023 12:53:14 +0800
Subject: [PATCH 006/299] return residual expr of join (#28760)

---
 .../jobs/joinorder/hypergraph/HyperGraph.java | 158 +++++++++++-------
 .../jobs/joinorder/hypergraph/edge/Edge.java  |  22 +++
 .../joinorder/hypergraph/edge/FilterEdge.java |  15 --
 .../mv/AbstractMaterializedViewRule.java      |   6 +-
 .../exploration/mv/ComparisonResult.java      |  94 +++++++++++
 .../rules/exploration/mv/StructInfo.java      |   2 +-
 .../hypergraph/CompareOuterJoinTest.java      |  13 +-
 .../hypergraph/PullupExpressionTest.java      | 151 +++++++++++++++++
 .../exploration/mv/BuildStructInfoTest.java   |   4 +-
 9 files changed, 380 insertions(+), 85 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
index e534fc1fa84624..7bd33c64b3c593 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
 import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
 import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
 import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
 import org.apache.doris.nereids.trees.expressions.Alias;
@@ -44,18 +45,21 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.Map.Entry;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
-import javax.annotation.Nullable;
 
 /**
  * The graph is a join graph, whose node is the leaf plan and edge is a join operator.
@@ -268,11 +272,11 @@ private void makeFilterConflictRules(JoinEdge joinEdge) {
         filterEdges.forEach(e -> {
             if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes)
                     && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType())) {
-                e.addRejectJoin(joinEdge);
+                e.addRejectEdge(joinEdge);
             }
             if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes)
                     && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType())) {
-                e.addRejectJoin(joinEdge);
+                e.addRejectEdge(joinEdge);
             }
         });
     }
@@ -289,9 +293,11 @@ private void makeJoinConflictRules(JoinEdge edgeB) {
             JoinEdge childA = joinEdges.get(i);
             if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) {
                 leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getLeftSubNodes(joinEdges));
+                childA.addRejectEdge(edgeB);
             }
             if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType())) {
                 leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getRightSubNodes(joinEdges));
+                childA.addRejectEdge(edgeB);
             }
         }
 
@@ -299,9 +305,11 @@ private void makeJoinConflictRules(JoinEdge edgeB) {
             JoinEdge childA = joinEdges.get(i);
             if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) {
                 rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getRightSubNodes(joinEdges));
+                childA.addRejectEdge(edgeB);
             }
             if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType())) {
                 rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getLeftSubNodes(joinEdges));
+                childA.addRejectEdge(edgeB);
             }
         }
         edgeB.setLeftExtendedNodes(leftRequired);
@@ -593,57 +601,75 @@ public int edgeSize() {
      * compare hypergraph
      *
      * @param viewHG the compared hyper graph
-     * @return null represents not compatible, or return some expression which can
-     *         be pull up from this hyper graph
+     * @return Comparison result
      */
-    public @Nullable List<Expression> isLogicCompatible(HyperGraph viewHG, LogicalCompatibilityContext ctx) {
-        Map<Edge, Edge> queryToView = constructEdgeMap(viewHG, ctx.getQueryToViewEdgeExpressionMapping());
+    public ComparisonResult isLogicCompatible(HyperGraph viewHG, LogicalCompatibilityContext ctx) {
+        // 1 try to construct a map which can be mapped from edge to edge
+        Map<Edge, Edge> queryToView = constructMapWithNode(viewHG, ctx.getQueryToViewNodeIDMapping());
 
-        // All edge in view must have a mapped edge in query
-        if (queryToView.size() != viewHG.edgeSize()) {
-            return null;
+        // 2. compare them by expression and extract residual expr
+        ComparisonResult.Builder builder = new ComparisonResult.Builder();
+        ComparisonResult edgeCompareRes = compareEdgesWithExpr(queryToView, ctx.getQueryToViewEdgeExpressionMapping());
+        if (edgeCompareRes.isInvalid()) {
+            return ComparisonResult.INVALID;
         }
+        builder.addComparisonResult(edgeCompareRes);
 
-        boolean allMatch = queryToView.entrySet().stream()
-                .allMatch(entry ->
-                        compareEdgeWithNode(entry.getKey(), entry.getValue(), ctx.getQueryToViewNodeIDMapping()));
-        if (!allMatch) {
-            return null;
+        // 3. pull join edge of view is no sense, so reject them
+        if (!queryToView.values().containsAll(viewHG.joinEdges)) {
+            return ComparisonResult.INVALID;
         }
 
-        // join edges must be identical
-        boolean isJoinIdentical = joinEdges.stream()
-                .allMatch(queryToView::containsKey);
-        if (!isJoinIdentical) {
-            return null;
+        // 4. process residual edges
+        List<Expression> residualQueryJoin =
+                processOrphanEdges(Sets.difference(Sets.newHashSet(joinEdges), queryToView.keySet()));
+        if (residualQueryJoin == null) {
+            return ComparisonResult.INVALID;
         }
+        builder.addQueryExpressions(residualQueryJoin);
 
-        // extract all top filters
-        List<FilterEdge> residualFilterEdges = filterEdges.stream()
-                .filter(e -> !queryToView.containsKey(e))
-                .collect(ImmutableList.toImmutableList());
-        if (residualFilterEdges.stream().anyMatch(e -> !e.isTopFilter())) {
-            return null;
+        List<Expression> residualQueryFilter =
+                processOrphanEdges(Sets.difference(Sets.newHashSet(filterEdges), queryToView.keySet()));
+        if (residualQueryFilter == null) {
+            return ComparisonResult.INVALID;
         }
-        return residualFilterEdges.stream()
-                .flatMap(e -> e.getExpressions().stream())
-                .collect(ImmutableList.toImmutableList());
+        builder.addQueryExpressions(residualQueryFilter);
+
+        List<Expression> residualViewFilter =
+                processOrphanEdges(
+                        Sets.difference(Sets.newHashSet(viewHG.filterEdges), Sets.newHashSet(queryToView.values())));
+        if (residualViewFilter == null) {
+            return ComparisonResult.INVALID;
+        }
+        builder.addViewExpressions(residualViewFilter);
+
+        return builder.build();
     }
 
-    private Map<Edge, Edge> constructEdgeMap(HyperGraph viewHG, Map<Expression, Expression> exprMap) {
-        Map<Expression, Edge> exprToEdge = constructExprMap(viewHG);
-        Map<Edge, Edge> queryToView = new HashMap<>();
-        joinEdges.stream()
-                .filter(e -> !e.getExpressions().isEmpty()
-                        && exprMap.containsKey(e.getExpression(0))
-                        && compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap))
-                .forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0)))));
-        filterEdges.stream()
-                .filter(e -> !e.getExpressions().isEmpty()
-                        && exprMap.containsKey(e.getExpression(0))
-                        && compareEdgeWithExpr(e, exprToEdge.get(exprMap.get(e.getExpression(0))), exprMap))
-                .forEach(e -> queryToView.put(e, exprToEdge.get(exprMap.get(e.getExpression(0)))));
-        return queryToView;
+    private List<Expression> processOrphanEdges(Set<Edge> edges) {
+        List<Expression> expressions = new ArrayList<>();
+        for (Edge edge : edges) {
+            if (!edge.canPullUp()) {
+                return null;
+            }
+            expressions.addAll(edge.getExpressions());
+        }
+        return expressions;
+    }
+
+    private Map<Edge, Edge> constructMapWithNode(HyperGraph viewHG, Map<Integer, Integer> nodeMap) {
+        // TODO use hash map to reduce loop
+        Map<Edge, Edge> joinEdgeMap = joinEdges.stream().map(qe -> {
+            Optional<JoinEdge> viewEdge = viewHG.joinEdges.stream()
+                    .filter(ve -> compareEdgeWithNode(qe, ve, nodeMap)).findFirst();
+            return Pair.of(qe, viewEdge);
+        }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
+        Map<Edge, Edge> filterEdgeMap = filterEdges.stream().map(qe -> {
+            Optional<FilterEdge> viewEdge = viewHG.filterEdges.stream()
+                    .filter(ve -> compareEdgeWithNode(qe, ve, nodeMap)).findFirst();
+            return Pair.of(qe, viewEdge);
+        }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
+        return ImmutableMap.<Edge, Edge>builder().putAll(joinEdgeMap).putAll(filterEdgeMap).build();
     }
 
     private boolean compareEdgeWithNode(Edge t, Edge o, Map<Integer, Integer> nodeMap) {
@@ -686,24 +712,40 @@ private boolean compareNodeMap(long bitmap1, long bitmap2, Map<Integer, Integer>
         return bitmap2 == newBitmap1;
     }
 
-    private boolean compareEdgeWithExpr(Edge t, Edge o, Map<Expression, Expression> expressionMap) {
-        if (t.getExpressions().size() != o.getExpressions().size()) {
-            return false;
-        }
-        int size = t.getExpressions().size();
-        for (int i = 0; i < size; i++) {
-            if (!Objects.equals(expressionMap.get(t.getExpression(i)), o.getExpression(i))) {
-                return false;
+    private ComparisonResult compareEdgesWithExpr(Map<Edge, Edge> queryToViewedgeMap,
+            Map<Expression, Expression> queryToView) {
+        ComparisonResult.Builder builder = new ComparisonResult.Builder();
+        for (Entry<Edge, Edge> e : queryToViewedgeMap.entrySet()) {
+            ComparisonResult res = compareEdgeWithExpr(e.getKey(), e.getValue(), queryToView);
+            if (res.isInvalid()) {
+                return ComparisonResult.INVALID;
             }
+            builder.addComparisonResult(res);
         }
-        return true;
+        return builder.build();
     }
 
-    private Map<Expression, Edge> constructExprMap(HyperGraph hyperGraph) {
-        Map<Expression, Edge> exprToEdge = new HashMap<>();
-        hyperGraph.joinEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge)));
-        hyperGraph.filterEdges.forEach(edge -> edge.getExpressions().forEach(expr -> exprToEdge.put(expr, edge)));
-        return exprToEdge;
+    private ComparisonResult compareEdgeWithExpr(Edge query, Edge view, Map<Expression, Expression> queryToView) {
+        Set<? extends Expression> queryExprSet = query.getExpressionSet();
+        Set<? extends Expression> viewExprSet = view.getExpressionSet();
+
+        Set<Expression> equalViewExpr = new HashSet<>();
+        List<Expression> residualQueryExpr = new ArrayList<>();
+        for (Expression queryExpr : queryExprSet) {
+            if (queryToView.containsKey(queryExpr) && viewExprSet.contains(queryToView.get(queryExpr))) {
+                equalViewExpr.add(queryToView.get(queryExpr));
+            } else {
+                residualQueryExpr.add(queryExpr);
+            }
+        }
+        List<Expression> residualViewExpr = ImmutableList.copyOf(Sets.difference(viewExprSet, equalViewExpr));
+        if (!residualViewExpr.isEmpty() && !view.canPullUp()) {
+            return ComparisonResult.INVALID;
+        }
+        if (!residualQueryExpr.isEmpty() && !query.canPullUp()) {
+            return ComparisonResult.INVALID;
+        }
+        return new ComparisonResult(residualQueryExpr, residualViewExpr);
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
index 4b88ce9f60dc5c..c47300922b5191 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
@@ -22,6 +22,8 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 
+import com.google.common.collect.ImmutableSet;
+
 import java.util.BitSet;
 import java.util.List;
 import java.util.Set;
@@ -51,6 +53,8 @@ public abstract class Edge {
     // record all sub nodes behind in this operator. It's T function in paper
     private final long subTreeNodes;
 
+    private long rejectNodes = 0;
+
     /**
      * Create simple edge.
      */
@@ -71,6 +75,10 @@ public boolean isSimple() {
         return LongBitmap.getCardinality(leftExtendedNodes) == 1 && LongBitmap.getCardinality(rightExtendedNodes) == 1;
     }
 
+    public void addRejectEdge(Edge edge) {
+        rejectNodes = LongBitmap.newBitmapUnion(edge.getReferenceNodes(), rejectNodes);
+    }
+
     public void addLeftExtendNode(long left) {
         this.leftExtendedNodes = LongBitmap.or(this.leftExtendedNodes, left);
     }
@@ -171,6 +179,20 @@ public double getSelectivity() {
 
     public abstract List<? extends Expression> getExpressions();
 
+    public Set<? extends Expression> getExpressionSet() {
+        return ImmutableSet.copyOf(getExpressions());
+    }
+
+    public boolean canPullUp() {
+        // Only inner join and filter with none rejectNodes can be pull up
+        return rejectNodes == 0
+                && !(this instanceof JoinEdge && !((JoinEdge) this).getJoinType().isInnerJoin());
+    }
+
+    public long getRejectNodes() {
+        return rejectNodes;
+    }
+
     public Expression getExpression(int i) {
         return getExpressions().get(i);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java
index ec037871025cf6..57c6d9660d089c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/FilterEdge.java
@@ -22,7 +22,6 @@
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 
-import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.List;
 import java.util.Set;
@@ -32,25 +31,11 @@
  */
 public class FilterEdge extends Edge {
     private final LogicalFilter<? extends Plan> filter;
-    private final List<Integer> rejectEdges;
 
     public FilterEdge(LogicalFilter<? extends Plan> filter, int index,
             BitSet childEdges, long subTreeNodes, long childRequireNodes) {
         super(index, childEdges, new BitSet(), subTreeNodes, childRequireNodes, 0L);
         this.filter = filter;
-        rejectEdges = new ArrayList<>();
-    }
-
-    public void addRejectJoin(JoinEdge joinEdge) {
-        rejectEdges.add(joinEdge.getIndex());
-    }
-
-    public List<Integer> getRejectEdges() {
-        return rejectEdges;
-    }
-
-    public boolean isTopFilter() {
-        return rejectEdges.isEmpty();
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 90ebe567c4628e..c63e2d85af35ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -136,12 +136,14 @@ protected List<Plan> rewrite(Plan queryPlan, CascadesContext cascadesContext) {
                 LogicalCompatibilityContext compatibilityContext =
                         LogicalCompatibilityContext.from(queryToViewTableMapping, queryToViewSlotMapping,
                                 queryStructInfo, viewStructInfo);
-                List<Expression> pulledUpExpressions = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
+                ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo,
                         compatibilityContext);
-                if (pulledUpExpressions == null) {
+                if (comparisonResult.isInvalid()) {
                     logger.debug(currentClassName + " graph logical is not equals so continue");
                     continue;
                 }
+                // TODO: Use set of list? And consider view expr
+                List<Expression> pulledUpExpressions = ImmutableList.copyOf(comparisonResult.getQueryExpressions());
                 // set pulled up expression to queryStructInfo predicates and update related predicates
                 if (!pulledUpExpressions.isEmpty()) {
                     queryStructInfo.addPredicates(pulledUpExpressions);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
new file mode 100644
index 00000000000000..cc8284f33e6824
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * comparison result of view and query
+ */
+public class ComparisonResult {
+    public static final ComparisonResult INVALID = new ComparisonResult(ImmutableList.of(), ImmutableList.of(), false);
+    public static final ComparisonResult EMPTY = new ComparisonResult(ImmutableList.of(), ImmutableList.of(), true);
+    private final boolean valid;
+    private final List<Expression> viewExpressions;
+    private final List<Expression> queryExpressions;
+
+    public ComparisonResult(List<Expression> queryExpressions, List<Expression> viewExpressions) {
+        this(queryExpressions, viewExpressions, true);
+    }
+
+    ComparisonResult(List<Expression> queryExpressions, List<Expression> viewExpressions, boolean valid) {
+        this.viewExpressions = ImmutableList.copyOf(viewExpressions);
+        this.queryExpressions = ImmutableList.copyOf(queryExpressions);
+        this.valid = valid;
+    }
+
+    public List<Expression> getViewExpressions() {
+        return viewExpressions;
+    }
+
+    public List<Expression> getQueryExpressions() {
+        return queryExpressions;
+    }
+
+    public boolean isInvalid() {
+        return !valid;
+    }
+
+    /**
+     * Builder
+     */
+    public static class Builder {
+        ImmutableList.Builder<Expression> queryBuilder = new ImmutableList.Builder<>();
+        ImmutableList.Builder<Expression> viewBuilder = new ImmutableList.Builder<>();
+        boolean valid = true;
+
+        /**
+         * add comparisonResult
+         */
+        public Builder addComparisonResult(ComparisonResult comparisonResult) {
+            if (comparisonResult.isInvalid()) {
+                valid = false;
+                return this;
+            }
+            queryBuilder.addAll(comparisonResult.getQueryExpressions());
+            viewBuilder.addAll(comparisonResult.getViewExpressions());
+            return this;
+        }
+
+        public Builder addQueryExpressions(Collection<Expression> expressions) {
+            queryBuilder.addAll(expressions);
+            return this;
+        }
+
+        public Builder addViewExpressions(Collection<Expression> expressions) {
+            viewBuilder.addAll(expressions);
+            return this;
+        }
+
+        public ComparisonResult build() {
+            return new ComparisonResult(queryBuilder.build(), viewBuilder.build(), valid);
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 3c9814cdce31c5..20da9ee12fdd69 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -263,7 +263,7 @@ public List<? extends Expression> getExpressions() {
      * For inner join should judge only the join tables,
      * for other join type should also judge the join direction, it's input filter that can not be pulled up etc.
      */
-    public static @Nullable List<Expression> isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo,
+    public static ComparisonResult isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo,
             LogicalCompatibilityContext compatibilityContext) {
         return queryStructInfo.hyperGraph.isLogicCompatible(viewStructInfo.hyperGraph, compatibilityContext);
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
index d0e5084a1c16cf..cfc88b2aa3cef2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
@@ -62,7 +62,7 @@ void testStarGraphWithInnerJoin() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null);
+        Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid());
     }
 
     @Test
@@ -79,7 +79,7 @@ void testRandomQuery() {
                 .getAllPlan().get(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)) != null);
+        Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid());
     }
 
     @Test
@@ -104,7 +104,7 @@ void testInnerJoinWithFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions();
         Assertions.assertEquals(1, exprList.size());
         Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
     }
@@ -132,7 +132,7 @@ void testInnerJoinWithFilter2() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions();
         Assertions.assertEquals(0, exprList.size());
     }
 
@@ -159,7 +159,7 @@ void testLeftOuterJoinWithLeftFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions();
         Assertions.assertEquals(1, exprList.size());
         Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
     }
@@ -187,8 +187,7 @@ void testLeftOuterJoinWithRightFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2));
-        Assertions.assertEquals(null, exprList);
+        Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid());
     }
 
     LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
new file mode 100644
index 00000000000000..d4818a844e80ab
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
@@ -0,0 +1,151 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.jobs.joinorder.hypergraph;
+
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.RuleSet;
+import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
+import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class PullupExpressionTest extends SqlTestBase {
+    @Test
+    void testPullUpQueryFilter() {
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id where T1.id = 1",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        Assertions.assertEquals(2, res.getQueryExpressions().size());
+        Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql());
+        Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(1).toSql());
+    }
+
+    @Test
+    void testPullUpQueryJoinCondition() {
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id and T1.score = T2.score",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        Assertions.assertEquals(1, res.getQueryExpressions().size());
+        Assertions.assertEquals("(score = score)", res.getQueryExpressions().get(0).toSql());
+    }
+
+    @Test
+    void testPullUpViewFilter() {
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id where T1.id = 1 and T2.id = 1",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        Assertions.assertEquals(2, res.getViewExpressions().size());
+        Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(0).toSql());
+        Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(1).toSql());
+    }
+
+    @Test
+    void testPullUpViewJoinCondition() {
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id ",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 join T2 on T1.id = T2.id and T1.score = T2.score",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        Assertions.assertEquals(1, res.getViewExpressions().size());
+        Assertions.assertEquals("(score = score)", res.getViewExpressions().get(0).toSql());
+    }
+
+    LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
+        StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1,
+                null).get(0);
+        StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2,
+                null).get(0);
+        RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0);
+        SlotMapping sm = SlotMapping.generate(rm);
+        return LogicalCompatibilityContext.from(rm, sm, st1, st2);
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
index 6ac41a49812f44..f68365f6708b37 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
@@ -79,7 +79,7 @@ void testFilter() {
                         .when(j -> {
                             HyperGraph structInfo = HyperGraph.toStructInfo(j).get(0);
                             Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
-                            Assertions.assertEquals(0, (int) structInfo.getFilterEdge(0).getRejectEdges().get(0));
+                            Assertions.assertEquals(3L, structInfo.getFilterEdge(0).getRejectNodes());
                             return true;
                         }));
 
@@ -92,7 +92,7 @@ void testFilter() {
                         .when(j -> {
                             HyperGraph structInfo = HyperGraph.toStructInfo(j).get(0);
                             Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
-                            Assertions.assertTrue(structInfo.getFilterEdge(0).getRejectEdges().isEmpty());
+                            Assertions.assertEquals(0, structInfo.getFilterEdge(0).getRejectNodes());
                             return true;
                         }));
     }

From c2eabbd441021732560d1e81365fb6e29fb812fd Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Mon, 25 Dec 2023 13:49:18 +0800
Subject: [PATCH 007/299] [fix](load) fix nullptr when getting memtable flush
 running count (#28942)

* [fix](load) fix nullptr when getting memtable flush running count

* style
---
 be/src/olap/delta_writer.cpp    | 3 +--
 be/src/olap/delta_writer_v2.cpp | 4 +---
 be/src/olap/memtable_writer.cpp | 4 ++++
 be/src/olap/memtable_writer.h   | 2 ++
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index 1c2f24ae6d94d3..f49a68985d8606 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -126,8 +126,7 @@ Status BaseDeltaWriter::write(const vectorized::Block* block, const std::vector<
     if (!_is_init && !_is_cancelled) {
         RETURN_IF_ERROR(init());
     }
-    while (_memtable_writer->get_flush_token_stats().flush_running_count >=
-           config::memtable_flush_running_count_limit) {
+    while (_memtable_writer->flush_running_count() >= config::memtable_flush_running_count_limit) {
         std::this_thread::sleep_for(std::chrono::milliseconds(10));
     }
     return _memtable_writer->write(block, row_idxs, is_append);
diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index cfe059f1890c89..bca1213b047480 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -37,7 +37,6 @@
 #include "gutil/strings/numbers.h"
 #include "io/fs/file_writer.h" // IWYU pragma: keep
 #include "olap/data_dir.h"
-#include "olap/memtable_flush_executor.h"
 #include "olap/olap_define.h"
 #include "olap/rowset/beta_rowset.h"
 #include "olap/rowset/beta_rowset_writer_v2.h"
@@ -153,8 +152,7 @@ Status DeltaWriterV2::write(const vectorized::Block* block, const std::vector<ui
     if (!_is_init && !_is_cancelled) {
         RETURN_IF_ERROR(init());
     }
-    while (_memtable_writer->get_flush_token_stats().flush_running_count >=
-           config::memtable_flush_running_count_limit) {
+    while (_memtable_writer->flush_running_count() >= config::memtable_flush_running_count_limit) {
         std::this_thread::sleep_for(std::chrono::milliseconds(10));
     }
     SCOPED_RAW_TIMER(&_write_memtable_time);
diff --git a/be/src/olap/memtable_writer.cpp b/be/src/olap/memtable_writer.cpp
index 1faed6c328b167..c967cdbd483968 100644
--- a/be/src/olap/memtable_writer.cpp
+++ b/be/src/olap/memtable_writer.cpp
@@ -345,6 +345,10 @@ const FlushStatistic& MemTableWriter::get_flush_token_stats() {
     return _flush_token->get_stats();
 }
 
+uint64_t MemTableWriter::flush_running_count() const {
+    return _flush_token == nullptr ? 0 : _flush_token->get_stats().flush_running_count.load();
+}
+
 int64_t MemTableWriter::mem_consumption(MemType mem) {
     if (!_is_init) {
         // This method may be called before this writer is initialized.
diff --git a/be/src/olap/memtable_writer.h b/be/src/olap/memtable_writer.h
index c5459c09065cda..a2687d9402cdca 100644
--- a/be/src/olap/memtable_writer.h
+++ b/be/src/olap/memtable_writer.h
@@ -109,6 +109,8 @@ class MemTableWriter {
 
     const FlushStatistic& get_flush_token_stats();
 
+    uint64_t flush_running_count() const;
+
 private:
     // push a full memtable to flush executor
     Status _flush_memtable_async();

From f2cdf150781c7e7b9c5c05aee47f5be2b24d7ce5 Mon Sep 17 00:00:00 2001
From: deardeng <565620795@qq.com>
Date: Mon, 25 Dec 2023 14:06:48 +0800
Subject: [PATCH 008/299] (enhance)(regression) Support
 `force_olap_table_replication_num=3` run
 test_insert_random_distribution_table case (#28903)

---
 ...st_insert_random_distribution_table.groovy | 57 ++++++++++++-------
 1 file changed, 37 insertions(+), 20 deletions(-)

diff --git a/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy b/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy
index 4630264ab41353..ae9f81400a96fb 100644
--- a/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy
+++ b/regression-test/suites/load_p0/insert/test_insert_random_distribution_table.groovy
@@ -44,11 +44,25 @@ suite("test_insert_random_distribution_table", "p0") {
     def totalCount = sql "select count() from ${tableName}"
     assertEquals(totalCount[0][0], 4)
     def res = sql "show tablets from ${tableName}"
-    def tabletId1 = res[0][0]
-    def tabletId2 = res[1][0]
-    def tabletId3 = res[2][0]
-    def tabletId4 = res[3][0]
-    def tabletId5 = res[4][0]
+    def getTablets = { result -> 
+      def map = [:]
+      def tablets = []
+      for (int i = 0; i < result.size(); i++) {
+          if (!map.get(result[i][0])) {
+              tablets.add(result[i][0])
+              map.put(result[i][0], result[i])
+          }
+      }
+      return tablets
+    }
+
+    def tablets = getTablets.call(res)
+
+    def tabletId1 = tablets[0]
+    def tabletId2 = tablets[1]
+    def tabletId3 = tablets[2] 
+    def tabletId4 = tablets[3] 
+    def tabletId5 = tablets[4] 
 
     def rowCount1 = sql "select count() from ${tableName} tablet(${tabletId1})"
     def rowCount2 = sql "select count() from ${tableName} tablet(${tabletId2})"
@@ -132,25 +146,28 @@ suite("test_insert_random_distribution_table", "p0") {
     def partition3 = "p20231013"
     assertEquals(totalCount[0][0], 5)
     res = sql "show tablets from ${tableName} partition ${partition1}"
-    def tabletId11 = res[0][0]
-    def tabletId12 = res[1][0]
-    def tabletId13 = res[2][0]
-    def tabletId14 = res[3][0]
-    def tabletId15 = res[4][0]
+    tablets = getTablets.call(res)
+    def tabletId11 = tablets[0] 
+    def tabletId12 = tablets[1] 
+    def tabletId13 = tablets[2] 
+    def tabletId14 = tablets[3] 
+    def tabletId15 = tablets[4] 
 
     res = sql "show tablets from ${tableName} partition ${partition2}"
-    def tabletId21 = res[0][0]
-    def tabletId22 = res[1][0]
-    def tabletId23 = res[2][0]
-    def tabletId24 = res[3][0]
-    def tabletId25 = res[4][0]
+    tablets = getTablets.call(res)
+    def tabletId21 = tablets[0] 
+    def tabletId22 = tablets[1] 
+    def tabletId23 = tablets[2] 
+    def tabletId24 = tablets[3] 
+    def tabletId25 = tablets[4] 
 
     res = sql "show tablets from ${tableName} partition ${partition3}"
-    def tabletId31 = res[0][0]
-    def tabletId32 = res[1][0]
-    def tabletId33 = res[2][0]
-    def tabletId34 = res[3][0]
-    def tabletId35 = res[4][0]
+    tablets = getTablets.call(res)
+    def tabletId31 = tablets[0] 
+    def tabletId32 = tablets[1] 
+    def tabletId33 = tablets[2] 
+    def tabletId34 = tablets[3] 
+    def tabletId35 = tablets[4] 
 
     def rowCount11 = sql "select count() from ${tableName} tablet(${tabletId11})"
     def rowCount12 = sql "select count() from ${tableName} tablet(${tabletId12})"

From 29d3d5e433bdb55f3692ddd3d5956cf9ba4399a4 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Mon, 25 Dec 2023 15:12:14 +0800
Subject: [PATCH 009/299] [Test](Job)Add test case (#28481)

---
 .../apache/doris/analysis/CreateJobStmt.java  |  22 ++--
 .../suites/job_p0/test_base_insert_job.groovy | 108 ++++++++++++++++--
 2 files changed, 104 insertions(+), 26 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
index dc2083debc2bfc..ececccc3169715 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
@@ -40,7 +40,6 @@
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.HashSet;
-import java.util.Set;
 
 /**
  * syntax:
@@ -86,15 +85,11 @@ public class CreateJobStmt extends DdlStmt {
     private JobExecuteType executeType;
 
     // exclude job name prefix, which is used by inner job
-    private final Set<String> excludeJobNamePrefix = new HashSet<>();
-
-    {
-        excludeJobNamePrefix.add("inner_mtmv_");
-    }
+    private final String excludeJobNamePrefix = "inner_";
 
     private static final ImmutableSet<Class<? extends DdlStmt>> supportStmtSuperClass
             = new ImmutableSet.Builder<Class<? extends DdlStmt>>().add(InsertStmt.class)
-            .add(UpdateStmt.class).build();
+            .build();
 
     private static final HashSet<String> supportStmtClassNamesCache = new HashSet<>(16);
 
@@ -164,16 +159,15 @@ public void analyze(Analyzer analyzer) throws UserException {
         String originStmt = getOrigStmt().originStmt;
         String executeSql = parseExecuteSql(originStmt);
         job.setExecuteSql(executeSql);
-
-        //job.checkJobParams();
         jobInstance = job;
     }
 
     private void checkJobName(String jobName) throws AnalysisException {
-        for (String prefix : excludeJobNamePrefix) {
-            if (jobName.startsWith(prefix)) {
-                throw new AnalysisException("job name can not start with " + prefix);
-            }
+        if (StringUtils.isBlank(jobName)) {
+            throw new AnalysisException("job name can not be null");
+        }
+        if (jobName.startsWith(excludeJobNamePrefix)) {
+            throw new AnalysisException("job name can not start with " + excludeJobNamePrefix);
         }
     }
 
@@ -193,7 +187,7 @@ private void checkStmtSupport() throws AnalysisException {
                 return;
             }
         }
-        throw new AnalysisException("Not support this stmt type");
+        throw new AnalysisException("Not support " + doStmt.getClass().getSimpleName() + " type in job");
     }
 
     private void analyzerSqlStmt() throws UserException {
diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
index 035def7a2d1424..a6f151141b921f 100644
--- a/regression-test/suites/job_p0/test_base_insert_job.groovy
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -23,10 +23,32 @@ import java.time.ZoneId;
 suite("test_base_insert_job") {
     def tableName = "t_test_BASE_inSert_job"
     def jobName = "insert_recovery_test_base_insert_job"
+    def jobMixedName = "Insert_recovery_Test_base_insert_job"
     sql """drop table if exists `${tableName}` force"""
     sql """
         DROP JOB where jobname =  '${jobName}'
     """
+    sql """
+       DROP JOB where jobname =  'JOB'
+    """
+    sql """
+       DROP JOB where jobname =  'DO'
+    """
+    sql """
+       DROP JOB where jobname =  'AT'
+    """
+    sql """
+       DROP JOB where jobname =  'SCHEDULE'
+    """
+    sql """
+       DROP JOB where jobname =  'STARTS'
+    """
+    sql """
+       DROP JOB where jobname =  'ENDS'
+    """
+    sql """
+        DROP JOB where jobname =  '${jobMixedName}'
+    """
 
     sql """
         CREATE TABLE IF NOT EXISTS `${tableName}`
@@ -48,9 +70,20 @@ suite("test_base_insert_job") {
     def jobs = sql """select * from ${tableName}"""
     println jobs
     assert 3>=jobs.size() >= (2 as Boolean) //at least 2 records, some times 3 records
+    sql """
+       CREATE JOB ${jobMixedName}  ON SCHEDULE every 1 second  DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+    """
+    def mixedNameJobs = sql """select name,comment from jobs("type"="insert") where Name='${jobMixedName}'"""
+    println mixedNameJobs
+    assert mixedNameJobs.size() == 1 && mixedNameJobs.get(0).get(0) == jobMixedName
+    assert mixedNameJobs.get(0).get(1) == ''
     sql """
         DROP JOB where jobname =  '${jobName}'
     """
+    sql """
+        DROP JOB where jobname =  '${jobMixedName}'
+    """
+    
     sql """drop table if exists `${tableName}` force """
     sql """
         CREATE TABLE IF NOT EXISTS `${tableName}`
@@ -84,7 +117,6 @@ suite("test_base_insert_job") {
     def onceJobSql= onceJob.get(0).get(1);
     println onceJobSql
     def assertSql = "insert into ${tableName}  values  (\'2023-07-19\', sleep(10000), 1001);"
-    println 'hhh'
     println assertSql
     assert onceJobSql == assertSql
     // test cancel task
@@ -108,7 +140,34 @@ suite("test_base_insert_job") {
     assert oncejob.get(0).get(0) == "FINISHED"
     //assert comment
     assert oncejob.get(0).get(1) == "test for test&68686781jbjbhj//ncsa"
- 
+    
+    // assert same job name
+    try {
+        sql """
+          CREATE JOB ${jobName}  ON SCHEDULE EVERY 10 second   comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName}  values  ('2023-07-19', sleep(10000), 1001);
+     """
+    }catch (Exception e) {
+        assert e.getMessage().contains("job name exist, jobName:insert_recovery_test_base_insert_job")
+    }
+    def errorTblName="${tableName}qwertyuioppoiuyte"
+    sql """drop table if exists `${errorTblName}` force"""
+    // assert error table name
+    try {
+        sql """
+          CREATE JOB ${jobName}  ON SCHEDULE EVERY 10 second   comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${errorTblName}  values  ('2023-07-19', sleep(10000), 1001);
+     """
+    }catch (Exception e) {
+        assert e.getMessage().contains("Unknown table 't_test_BASE_inSert_jobqwertyuioppoiuyte'")
+    }
+    // assert not support stmt
+    try{
+        sql """
+            CREATE JOB ${jobName}  ON SCHEDULE at '${startTime}'   comment 'test' DO update ${tableName} set type=2 where type=1;
+        """
+    } catch (Exception e) {
+        assert e.getMessage().contains("Not support UpdateStmt type in job")
+    }
+    // assert start time greater than current time
     try{
         sql """
             CREATE JOB ${jobName}  ON SCHEDULE at '${startTime}'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
@@ -116,9 +175,7 @@ suite("test_base_insert_job") {
     } catch (Exception e) {
         assert e.getMessage().contains("startTimeMs must be greater than current time")
     }
-    sql """
-        DROP JOB where jobname =  'test_one_time_error_starts'
-    """
+    // assert end time less than start time
     try{
         sql """
             CREATE JOB test_one_time_error_starts  ON SCHEDULE at '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
@@ -126,9 +183,14 @@ suite("test_base_insert_job") {
     } catch (Exception e) {
         assert e.getMessage().contains("startTimeMs must be greater than current time")
     }
-    sql """
-        DROP JOB where jobname =  'test_error_starts'
-    """
+    try{
+        sql """
+            CREATE JOB inner_test  ON SCHEDULE at '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+        """
+    } catch (Exception e) {
+        assert e.getMessage().contains("job name can not start with inner_")
+    }
+    // assert end time less than start time
     try{
         sql """
             CREATE JOB test_error_starts  ON SCHEDULE every 1 second ends '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
@@ -136,10 +198,7 @@ suite("test_base_insert_job") {
     } catch (Exception e) {
         assert e.getMessage().contains("end time cannot be less than start time")
     }
-
-    sql """
-        DROP JOB where jobname =  'test_error_starts'
-    """
+    // assert interval time unit can not be years
     try{
         sql """
             CREATE JOB test_error_starts  ON SCHEDULE every 1 years ends '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
@@ -147,5 +206,30 @@ suite("test_base_insert_job") {
     } catch (Exception e) {
         assert e.getMessage().contains("interval time unit can not be years")
     }
+
+    // test keyword as job name
+    sql """
+        CREATE JOB JOB  ON SCHEDULE every 20 second   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+    """
+    sql """
+        CREATE JOB SCHEDULE  ON SCHEDULE every 20 second   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+    """
+    sql """
+        CREATE JOB DO  ON SCHEDULE every 20 second   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+    """
+    sql """
+        CREATE JOB AT  ON SCHEDULE every 20 second   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+    """
+
+    sql """
+        CREATE JOB STARTS  ON SCHEDULE every 20 second   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+    """
+
+    sql """
+        CREATE JOB ENDS  ON SCHEDULE every 20 second   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
+    """
+    
+    def jobCountRsp = sql"""select count(1) from jobs("type"="insert") where name in ('JOB','DO','SCHEDULE','AT','STARTS','ENDS')"""
+    assert jobCountRsp.get(0).get(0) == 6
     
 }

From 8e47fd28e2fad52abb90fe0dd6319f1c90958565 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Mon, 25 Dec 2023 14:49:18 +0700
Subject: [PATCH 010/299] [fix](doc) typo fix in auto-partition page (#28512)

---
 docs/en/docs/advanced/partition/auto-partition.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/docs/advanced/partition/auto-partition.md b/docs/en/docs/advanced/partition/auto-partition.md
index 4117a0d90a29ba..398135ecd91dec 100644
--- a/docs/en/docs/advanced/partition/auto-partition.md
+++ b/docs/en/docs/advanced/partition/auto-partition.md
@@ -81,7 +81,7 @@ PROPERTIES (
 
 The table stores a large amount of business history data, partitioned based on the date the transaction occurred. As you can see when building the table, we need to manually create the partitions in advance. If the data range of the partitioned columns changes, for example, 2022 is added to the above table, we need to create a partition by [ALTER-TABLE-PARTITION](../../sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-TABLE-PARTITION) to make changes to the table partition. If such partitions need to be changed, or subdivided at a finer level of granularity, it is very tedious to modify them. At this point we can rewrite the table DDL using AUTO PARTITION.
 
-## Grammer
+## Grammar
 
 When building a table, use the following syntax to populate [CREATE-TABLE](../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE) with the `partition_info` section:
 
@@ -143,7 +143,7 @@ When building a table, use the following syntax to populate [CREATE-TABLE](../..
 
 1. Currently the AUTO RANGE PARTITION function supports only one partition column;
 2. In AUTO RANGE PARTITION, the partition function supports only `date_trunc` and the partition column supports only `DATEV2` or `DATETIMEV2` format;
-3. In AUTO LIST PARTITION, function calls are not supported. Partitioned columns support `BOOLEAN`, `TINYINT`, `SMALLINT`, `INT`, `BIGINT`, `LARGEINT`, `DATE`, `DATETIME`, `CHAR`, `VARCHAR` datatypes, and partitioned values are enum values.
+3. In AUTO LIST PARTITION, function calls are not supported. Partitioned columns support `BOOLEAN`, `TINYINT`, `SMALLINT`, `INT`, `BIGINT`, `LARGEINT`, `DATE`, `DATETIME`, `CHAR`, `VARCHAR` data-types, and partitioned values are enum values.
 4. In AUTO LIST PARTITION, a separate new PARTITION is created for each fetch of a partition column for which the corresponding partition does not currently exist.
 
 ## Sample Scenarios

From 17917a0adb57ac337dfd41e8853b1aa87c9d5acd Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Mon, 25 Dec 2023 16:27:16 +0800
Subject: [PATCH 011/299] [feature](load) enable memtable on sink node by
 default (#28963)

---
 docs/en/docs/advanced/variables.md                              | 2 +-
 docs/zh-CN/docs/advanced/variables.md                           | 2 +-
 .../src/main/java/org/apache/doris/qe/SessionVariable.java      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/docs/advanced/variables.md b/docs/en/docs/advanced/variables.md
index 030e228f5234aa..e339b1893dc789 100644
--- a/docs/en/docs/advanced/variables.md
+++ b/docs/en/docs/advanced/variables.md
@@ -689,7 +689,7 @@ Note that the comment must start with /*+ and can only follow the SELECT.
 * `enable_memtable_on_sink_node`
 
   <version since="2.1.0">
-  Whether to enable MemTable on DataSink node when loading data, default is false.
+  Whether to enable MemTable on DataSink node when loading data, default is true.
   </version>
 
   Build MemTable on DataSink node, and send segments to other backends through brpc streaming.
diff --git a/docs/zh-CN/docs/advanced/variables.md b/docs/zh-CN/docs/advanced/variables.md
index 2a697b01bf7de4..510ad85c36e9ff 100644
--- a/docs/zh-CN/docs/advanced/variables.md
+++ b/docs/zh-CN/docs/advanced/variables.md
@@ -677,7 +677,7 @@ try (Connection conn = DriverManager.getConnection("jdbc:mysql://127.0.0.1:9030/
 * `enable_memtable_on_sink_node`
 
   <version since="2.1.0">
-  是否在数据导入中启用 MemTable 前移，默认为 false
+  是否在数据导入中启用 MemTable 前移，默认为 true
   </version>
 
   在 DataSink 节点上构建 MemTable，并通过 brpc streaming 发送 segment 到其他 BE。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index b9658bf34a5802..332e0f5bfaa8f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1322,7 +1322,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
     public boolean truncateCharOrVarcharColumns = false;
 
     @VariableMgr.VarAttr(name = ENABLE_MEMTABLE_ON_SINK_NODE, needForward = true)
-    public boolean enableMemtableOnSinkNode = false;
+    public boolean enableMemtableOnSinkNode = true;
 
     @VariableMgr.VarAttr(name = LOAD_STREAM_PER_NODE)
     public int loadStreamPerNode = 20;

From 2dc7d82affa6a4ca47f749cfdcba8b2db6ef8c9a Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Mon, 25 Dec 2023 16:55:59 +0800
Subject: [PATCH 012/299] [Chore](Job)print log before task execute (#28962)

---
 .../apache/doris/job/executor/DefaultTaskExecutorHandler.java  | 3 ++-
 .../org/apache/doris/job/executor/DispatchTaskHandler.java     | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java
index a07e248af6d593..befa8cc35fcbcc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DefaultTaskExecutorHandler.java
@@ -42,9 +42,10 @@ public void onEvent(ExecuteTaskEvent<T> executeTaskEvent) {
             return;
         }
         if (task.isCancelled()) {
-            log.info("task is canceled, ignore");
+            log.info("task is canceled, ignore. task id is {}", task.getTaskId());
             return;
         }
+        log.info("start to execute task, task id is {}", task.getTaskId());
         try {
             task.runTask();
         } catch (Exception e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java
index cb0a393d9175c3..e5933d133cbd61 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/executor/DispatchTaskHandler.java
@@ -67,6 +67,8 @@ public void onEvent(TimerJobEvent<T> event) {
                 JobType jobType = event.getJob().getJobType();
                 for (AbstractTask task : tasks) {
                     disruptorMap.get(jobType).publishEvent(task, event.getJob().getJobConfig());
+                    log.info("dispatch timer job success, job id is {},  task id is {}",
+                            event.getJob().getJobId(), task.getTaskId());
                 }
             }
         } catch (Exception e) {

From 91e5b47439692068a74f84ad80dc543d34fd561e Mon Sep 17 00:00:00 2001
From: walter <w41ter.l@gmail.com>
Date: Mon, 25 Dec 2023 19:18:01 +0800
Subject: [PATCH 013/299] [fix](hdfs) Fix HdfsFileSystem::exists_impl crash
 (#28952)

Calling hdfsGetLastExceptionRootCause without initializing ThreadLocalState
will crash. This PR modifies the condition for determining the existence of
a hdfs file, because hdfsExists will set errno to ENOENT when the file does
not exist, we can use this condition to check whether a file existence rather
than check the existence of the root cause.
---
 be/src/io/fs/hdfs_file_system.cpp | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp
index 7e0062d2430c7f..be508aa23a922f 100644
--- a/be/src/io/fs/hdfs_file_system.cpp
+++ b/be/src/io/fs/hdfs_file_system.cpp
@@ -17,6 +17,7 @@
 
 #include "io/fs/hdfs_file_system.h"
 
+#include <errno.h>
 #include <fcntl.h>
 #include <gen_cpp/PlanNodes_types.h>
 #include <limits.h>
@@ -227,28 +228,20 @@ Status HdfsFileSystem::delete_internal(const Path& path, int is_recursive) {
 Status HdfsFileSystem::exists_impl(const Path& path, bool* res) const {
     CHECK_HDFS_HANDLE(_fs_handle);
     Path real_path = convert_path(path, _fs_name);
-#ifdef USE_HADOOP_HDFS
-    // HACK: the HDFS native client won't clear the last exception as expected so
-    // `hdfsGetLastExceptionRootCause` might return a staled root cause. Save the
-    // last root cause here and verify after hdfsExists returns a non-zero code.
-    //
-    // See details:
-    //  https://github.com/apache/hadoop/blob/5cda162a804fb0cfc2a5ac0058ab407662c5fb00/
-    //  hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c#L795
-    char* former_root_cause = hdfsGetLastExceptionRootCause();
-#endif
     int is_exists = hdfsExists(_fs_handle->hdfs_fs, real_path.string().c_str());
 #ifdef USE_HADOOP_HDFS
     // when calling hdfsExists() and return non-zero code,
-    // if root_cause is nullptr, which means the file does not exist.
-    // if root_cause is not nullptr, which means it encounter other error, should return.
+    // if errno is ENOENT, which means the file does not exist.
+    // if errno is not ENOENT, which means it encounter other error, should return.
     // NOTE: not for libhdfs3 since it only runs on MaxOS, don't have to support it.
-    if (is_exists != 0) {
+    //
+    // See details:
+    //  https://github.com/apache/hadoop/blob/5cda162a804fb0cfc2a5ac0058ab407662c5fb00/
+    //  hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c#L1923-L1924
+    if (is_exists != 0 && errno != ENOENT) {
         char* root_cause = hdfsGetLastExceptionRootCause();
-        if (root_cause != nullptr && root_cause != former_root_cause) {
-            return Status::IOError("failed to check path existence {}: {}", path.native(),
-                                   root_cause);
-        }
+        return Status::IOError("failed to check path existence {}: {}", path.native(),
+                               (root_cause ? root_cause : "unknown"));
     }
 #endif
     *res = (is_exists == 0);

From 7081139bdce53e3536c8e25e45a9b4cb92e4f43d Mon Sep 17 00:00:00 2001
From: caiconghui <55968745+caiconghui@users.noreply.github.com>
Date: Mon, 25 Dec 2023 20:35:22 +0800
Subject: [PATCH 014/299] [fix](block) fix be core while mutable block merge
 may cause different row size between columns in origin block (#27943)

---
 be/src/exec/exec_node.cpp                     |  1 +
 .../olap/rowset/segment_v2/segment_writer.cpp |  4 +-
 be/src/olap/tablet.cpp                        |  2 +
 be/src/pipeline/exec/join_probe_operator.cpp  |  5 +-
 .../exec/nested_loop_join_probe_operator.cpp  | 32 ++++++-----
 .../exec/nested_loop_join_probe_operator.h    |  6 +--
 be/src/pipeline/pipeline_x/operator.cpp       |  1 +
 be/src/vec/common/sort/sorter.cpp             |  1 +
 be/src/vec/core/block.cpp                     |  9 ++--
 be/src/vec/exec/format/csv/csv_reader.cpp     |  6 ++-
 be/src/vec/exec/format/orc/vorc_reader.cpp    |  6 +--
 .../exec/format/parquet/vparquet_reader.cpp   |  5 +-
 .../vec/exec/format/table/iceberg_reader.cpp  |  4 +-
 be/src/vec/exec/join/vjoin_node_base.cpp      |  5 +-
 .../vec/exec/join/vnested_loop_join_node.cpp  | 15 +++---
 be/src/vec/exec/join/vnested_loop_join_node.h | 22 ++++----
 be/src/vec/exec/scan/pip_scanner_context.h    |  1 +
 be/src/vec/exec/scan/scanner_context.cpp      |  1 +
 be/src/vec/exec/scan/scanner_scheduler.cpp    |  4 +-
 be/src/vec/exec/vpartition_sort_node.h        |  1 +
 be/src/vec/exec/vrepeat_node.cpp              |  2 +-
 be/src/vec/exec/vtable_function_node.cpp      |  2 +-
 be/src/vec/exec/vunion_node.cpp               |  4 ++
 be/src/vec/olap/block_reader.cpp              |  8 ++-
 be/src/vec/olap/vcollect_iterator.cpp         |  4 +-
 be/src/vec/olap/vertical_block_reader.cpp     |  2 +
 be/src/vec/olap/vgeneric_iterators.cpp        |  5 +-
 be/src/vec/runtime/vsorted_run_merger.cpp     |  1 +
 be/test/vec/core/block_test.cpp               | 53 +++++++++++++++++++
 29 files changed, 146 insertions(+), 66 deletions(-)

diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp
index 4681218dcae824..7d38ee5e651611 100644
--- a/be/src/exec/exec_node.cpp
+++ b/be/src/exec/exec_node.cpp
@@ -561,6 +561,7 @@ Status ExecNode::do_projections(vectorized::Block* origin_block, vectorized::Blo
             }
         }
         DCHECK(mutable_block.rows() == rows);
+        output_block->set_columns(std::move(mutable_columns));
     }
 
     return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 713448e1aaee11..a36b83f23b9f1d 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -519,6 +519,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block*
     auto mutable_full_columns = full_block.mutate_columns();
     RETURN_IF_ERROR(fill_missing_columns(mutable_full_columns, use_default_or_null_flag,
                                          has_default_or_nullable, segment_start_pos));
+    full_block.set_columns(std::move(mutable_full_columns));
     // row column should be filled here
     if (_tablet_schema->store_row_column()) {
         // convert block to row store format
@@ -586,7 +587,6 @@ Status SegmentWriter::fill_missing_columns(vectorized::MutableColumns& mutable_f
     const auto& cids_missing = _opts.rowset_ctx->partial_update_info->missing_cids;
     auto old_value_block = _tablet_schema->create_block_by_cids(cids_missing);
     CHECK(cids_missing.size() == old_value_block.columns());
-    auto mutable_old_columns = old_value_block.mutate_columns();
     bool has_row_column = _tablet_schema->store_row_column();
     // record real pos, key is input line num, value is old_block line num
     std::map<uint32_t, uint32_t> read_index;
@@ -609,6 +609,7 @@ Status SegmentWriter::fill_missing_columns(vectorized::MutableColumns& mutable_f
                 }
                 continue;
             }
+            auto mutable_old_columns = old_value_block.mutate_columns();
             for (size_t cid = 0; cid < mutable_old_columns.size(); ++cid) {
                 TabletColumn tablet_column = _tablet_schema->column(cids_missing[cid]);
                 auto st = tablet->fetch_value_by_rowids(rowset, seg_it.first, rids, tablet_column,
@@ -619,6 +620,7 @@ Status SegmentWriter::fill_missing_columns(vectorized::MutableColumns& mutable_f
                     return st;
                 }
             }
+            old_value_block.set_columns(std::move(mutable_old_columns));
         }
     }
     // build default value columns
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index a5f34633d79240..5f915ad1de16c9 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -3199,6 +3199,7 @@ Status Tablet::generate_new_block_for_partial_update(
                     read_index_update[idx]);
         }
     }
+    output_block->set_columns(std::move(full_mutable_columns));
     VLOG_DEBUG << "full block when publish: " << output_block->dump_data();
     return Status::OK();
 }
@@ -3244,6 +3245,7 @@ Status Tablet::read_columns_by_plan(TabletSchemaSPtr tablet_schema,
             }
         }
     }
+    block.set_columns(std::move(mutable_columns));
     return Status::OK();
 }
 
diff --git a/be/src/pipeline/exec/join_probe_operator.cpp b/be/src/pipeline/exec/join_probe_operator.cpp
index 6b6796767e9505..4e94b4a206be51 100644
--- a/be/src/pipeline/exec/join_probe_operator.cpp
+++ b/be/src/pipeline/exec/join_probe_operator.cpp
@@ -134,9 +134,8 @@ Status JoinProbeLocalState<DependencyType, Derived>::_build_output_block(
             }
         }
 
-        if (!is_mem_reuse || !keep_origin) {
-            output_block->swap(mutable_block.to_block());
-        }
+        output_block->swap(mutable_block.to_block());
+
         DCHECK(output_block->rows() == rows);
     }
 
diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
index b7477d0b4f8755..1e3de36e92062f 100644
--- a/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
+++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.cpp
@@ -130,7 +130,6 @@ Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* sta
     _left_side_process_count = 0;
     DCHECK(!_need_more_input_data || !_matched_rows_done);
 
-    vectorized::MutableBlock mutable_join_block(&_join_block);
     if (!_matched_rows_done && !_need_more_input_data) {
         // We should try to join rows if there still are some rows from probe side.
         while (_join_block.rows() < state->batch_size()) {
@@ -144,7 +143,7 @@ Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* sta
                 _reset_with_next_probe_row();
                 if (_left_block_pos < _child_block->rows()) {
                     if constexpr (set_probe_side_flag) {
-                        _probe_offset_stack.push(mutable_join_block.rows());
+                        _probe_offset_stack.push(_join_block.rows());
                     }
                 } else {
                     if (_shared_state->left_side_eos) {
@@ -163,9 +162,9 @@ Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* sta
 
             const auto& now_process_build_block = _shared_state->build_blocks[_current_build_pos++];
             if constexpr (set_build_side_flag) {
-                _build_offset_stack.push(mutable_join_block.rows());
+                _build_offset_stack.push(_join_block.rows());
             }
-            _process_left_child_block(mutable_join_block, now_process_build_block);
+            _process_left_child_block(_join_block, now_process_build_block);
         }
 
         if constexpr (set_probe_side_flag) {
@@ -178,21 +177,20 @@ Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* sta
             if (!status.ok()) {
                 return status;
             }
-            mutable_join_block = vectorized::MutableBlock(&_join_block);
             // If this join operation is left outer join or full outer join, when
             // `_left_side_process_count`, means all rows from build
             // side have been joined with _left_side_process_count, we should output current
             // probe row with null from build side.
             if (_left_side_process_count) {
                 _finalize_current_phase<false, JoinOpType::value == TJoinOp::LEFT_SEMI_JOIN>(
-                        mutable_join_block, state->batch_size());
+                        _join_block, state->batch_size());
             }
         }
 
         if (_left_side_process_count) {
             if (p._is_mark_join && _shared_state->build_blocks.empty()) {
                 DCHECK_EQ(JoinOpType::value, TJoinOp::CROSS_JOIN);
-                _append_left_data_with_null(mutable_join_block);
+                _append_left_data_with_null(_join_block);
             }
         }
     }
@@ -204,7 +202,6 @@ Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* sta
                                                                  set_probe_side_flag, ignore_null>(
                          &_join_block, !p._is_right_semi_anti)));
         _update_additional_flags(&_join_block);
-        mutable_join_block = vectorized::MutableBlock(&_join_block);
         if (!status.ok()) {
             return status;
         }
@@ -214,7 +211,7 @@ Status NestedLoopJoinProbeLocalState::generate_join_block_data(RuntimeState* sta
         if (_matched_rows_done &&
             _output_null_idx_build_side < _shared_state->build_blocks.size()) {
             _finalize_current_phase<true, JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN>(
-                    mutable_join_block, state->batch_size());
+                    _join_block, state->batch_size());
         }
     }
     return Status::OK();
@@ -235,10 +232,10 @@ void NestedLoopJoinProbeLocalState::_resize_fill_tuple_is_null_column(size_t new
 }
 
 template <bool BuildSide, bool IsSemi>
-void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::MutableBlock& mutable_block,
+void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::Block& block,
                                                             size_t batch_size) {
     auto& p = _parent->cast<NestedLoopJoinProbeOperatorX>();
-    auto& dst_columns = mutable_block.mutable_columns();
+    auto dst_columns = block.mutate_columns();
     DCHECK_GT(dst_columns.size(), 0);
     auto column_size = dst_columns[0]->size();
     if constexpr (BuildSide) {
@@ -355,12 +352,12 @@ void NestedLoopJoinProbeLocalState::_finalize_current_phase(vectorized::MutableB
             _resize_fill_tuple_is_null_column(_left_side_process_count, 0, 1);
         }
     }
+    block.set_columns(std::move(dst_columns));
 }
 
-void NestedLoopJoinProbeLocalState::_append_left_data_with_null(
-        vectorized::MutableBlock& mutable_block) const {
+void NestedLoopJoinProbeLocalState::_append_left_data_with_null(vectorized::Block& block) const {
     auto& p = _parent->cast<NestedLoopJoinProbeOperatorX>();
-    auto& dst_columns = mutable_block.mutable_columns();
+    auto dst_columns = block.mutate_columns();
     DCHECK(p._is_mark_join);
     for (size_t i = 0; i < p._num_probe_side_columns; ++i) {
         const vectorized::ColumnWithTypeAndName& src_column = _child_block->get_by_position(i);
@@ -387,13 +384,13 @@ void NestedLoopJoinProbeLocalState::_append_left_data_with_null(
     auto& mark_column = *dst_columns[dst_columns.size() - 1];
     vectorized::ColumnFilterHelper(mark_column)
             .resize_fill(mark_column.size() + _left_side_process_count, 0);
+    block.set_columns(std::move(dst_columns));
 }
 
 void NestedLoopJoinProbeLocalState::_process_left_child_block(
-        vectorized::MutableBlock& mutable_block,
-        const vectorized::Block& now_process_build_block) const {
+        vectorized::Block& block, const vectorized::Block& now_process_build_block) const {
     auto& p = _parent->cast<NestedLoopJoinProbeOperatorX>();
-    auto& dst_columns = mutable_block.mutable_columns();
+    auto dst_columns = block.mutate_columns();
     const int max_added_rows = now_process_build_block.rows();
     for (size_t i = 0; i < p._num_probe_side_columns; ++i) {
         const vectorized::ColumnWithTypeAndName& src_column = _child_block->get_by_position(i);
@@ -434,6 +431,7 @@ void NestedLoopJoinProbeLocalState::_process_left_child_block(
                                                                           0, max_added_rows);
         }
     }
+    block.set_columns(std::move(dst_columns));
 }
 
 NestedLoopJoinProbeOperatorX::NestedLoopJoinProbeOperatorX(ObjectPool* pool, const TPlanNode& tnode,
diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.h b/be/src/pipeline/exec/nested_loop_join_probe_operator.h
index bc8913f5d08c7e..525640beb48519 100644
--- a/be/src/pipeline/exec/nested_loop_join_probe_operator.h
+++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.h
@@ -82,11 +82,11 @@ class NestedLoopJoinProbeLocalState final
     friend class NestedLoopJoinProbeOperatorX;
     void _update_additional_flags(vectorized::Block* block);
     template <bool BuildSide, bool IsSemi>
-    void _finalize_current_phase(vectorized::MutableBlock& mutable_block, size_t batch_size);
+    void _finalize_current_phase(vectorized::Block& block, size_t batch_size);
     void _resize_fill_tuple_is_null_column(size_t new_size, int left_flag, int right_flag);
     void _reset_with_next_probe_row();
-    void _append_left_data_with_null(vectorized::MutableBlock& mutable_block) const;
-    void _process_left_child_block(vectorized::MutableBlock& mutable_block,
+    void _append_left_data_with_null(vectorized::Block& block) const;
+    void _process_left_child_block(vectorized::Block& block,
                                    const vectorized::Block& now_process_build_block) const;
     template <typename Filter, bool SetBuildSideFlag, bool SetProbeSideFlag>
     void _do_filtering_and_update_visited_flags_impl(vectorized::Block* block, int column_to_keep,
diff --git a/be/src/pipeline/pipeline_x/operator.cpp b/be/src/pipeline/pipeline_x/operator.cpp
index 39598dadbd8325..b90c9551426f64 100644
--- a/be/src/pipeline/pipeline_x/operator.cpp
+++ b/be/src/pipeline/pipeline_x/operator.cpp
@@ -187,6 +187,7 @@ Status OperatorXBase::do_projections(RuntimeState* state, vectorized::Block* ori
             }
         }
         DCHECK(mutable_block.rows() == rows);
+        output_block->set_columns(std::move(mutable_columns));
     }
 
     return Status::OK();
diff --git a/be/src/vec/common/sort/sorter.cpp b/be/src/vec/common/sort/sorter.cpp
index 40e51ce20ce357..bd095e1afc561c 100644
--- a/be/src/vec/common/sort/sorter.cpp
+++ b/be/src/vec/common/sort/sorter.cpp
@@ -186,6 +186,7 @@ Status MergeSorterState::_merge_sort_read_not_spilled(int batch_size,
 
         if (merged_rows == batch_size) break;
     }
+    block->set_columns(std::move(merged_columns));
 
     if (merged_rows == 0) {
         *eos = true;
diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp
index 544a558afc01ba..c2594a2e35a3e2 100644
--- a/be/src/vec/core/block.cpp
+++ b/be/src/vec/core/block.cpp
@@ -577,7 +577,7 @@ MutableColumns Block::mutate_columns() {
     size_t num_columns = data.size();
     MutableColumns columns(num_columns);
     for (size_t i = 0; i < num_columns; ++i) {
-        columns[i] = data[i].column ? (*std::move(data[i].column)).assume_mutable()
+        columns[i] = data[i].column ? (*std::move(data[i].column)).mutate()
                                     : data[i].type->create_column();
     }
     return columns;
@@ -716,7 +716,7 @@ void Block::swap(Block& other) noexcept {
 void Block::swap(Block&& other) noexcept {
     clear();
     data = std::move(other.data);
-    initialize_index_by_name();
+    index_by_name = std::move(other.index_by_name);
     row_same_bit = std::move(other.row_same_bit);
 }
 
@@ -936,7 +936,7 @@ void MutableBlock::swap(MutableBlock& another) noexcept {
     _columns.swap(another._columns);
     _data_types.swap(another._data_types);
     _names.swap(another._names);
-    initialize_index_by_name();
+    index_by_name.swap(another.index_by_name);
 }
 
 void MutableBlock::swap(MutableBlock&& another) noexcept {
@@ -944,7 +944,7 @@ void MutableBlock::swap(MutableBlock&& another) noexcept {
     _columns = std::move(another._columns);
     _data_types = std::move(another._data_types);
     _names = std::move(another._names);
-    initialize_index_by_name();
+    index_by_name = std::move(another.index_by_name);
 }
 
 void MutableBlock::add_row(const Block* block, int row) {
@@ -1027,6 +1027,7 @@ Block MutableBlock::to_block(int start_column) {
 
 Block MutableBlock::to_block(int start_column, int end_column) {
     ColumnsWithTypeAndName columns_with_schema;
+    columns_with_schema.reserve(end_column - start_column);
     for (size_t i = start_column; i < end_column; ++i) {
         columns_with_schema.emplace_back(std::move(_columns[i]), _data_types[i], _names[i]);
     }
diff --git a/be/src/vec/exec/format/csv/csv_reader.cpp b/be/src/vec/exec/format/csv/csv_reader.cpp
index 0d93fcd648c1db..9f8c01b10dbac9 100644
--- a/be/src/vec/exec/format/csv/csv_reader.cpp
+++ b/be/src/vec/exec/format/csv/csv_reader.cpp
@@ -479,10 +479,11 @@ Status CsvReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
             RETURN_IF_ERROR(_validate_line(Slice(ptr, size), &success));
             ++rows;
         }
-        for (auto& col : block->mutate_columns()) {
+        auto mutate_columns = block->mutate_columns();
+        for (auto& col : mutate_columns) {
             col->resize(rows);
         }
-
+        block->set_columns(std::move(mutate_columns));
     } else {
         auto columns = block->mutate_columns();
         while (rows < batch_size && !_line_reader_eof) {
@@ -504,6 +505,7 @@ Status CsvReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
             }
             RETURN_IF_ERROR(_fill_dest_columns(Slice(ptr, size), block, columns, &rows));
         }
+        block->set_columns(std::move(columns));
     }
 
     *eof = (rows == 0);
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index c57d38076241c2..12a889ec23ed2f 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -1412,11 +1412,11 @@ Status OrcReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
         auto rows = std::min(get_remaining_rows(), (int64_t)_batch_size);
 
         set_remaining_rows(get_remaining_rows() - rows);
-
-        for (auto& col : block->mutate_columns()) {
+        auto mutate_columns = block->mutate_columns();
+        for (auto& col : mutate_columns) {
             col->resize(rows);
         }
-
+        block->set_columns(std::move(mutate_columns));
         *read_rows = rows;
         if (get_remaining_rows() == 0) {
             *eof = true;
diff --git a/be/src/vec/exec/format/parquet/vparquet_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
index 87555f6ee86cda..6ae4ea2f5bc69e 100644
--- a/be/src/vec/exec/format/parquet/vparquet_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_reader.cpp
@@ -528,10 +528,11 @@ Status ParquetReader::get_next_block(Block* block, size_t* read_rows, bool* eof)
 
         _current_group_reader->set_remaining_rows(_current_group_reader->get_remaining_rows() -
                                                   rows);
-
-        for (auto& col : block->mutate_columns()) {
+        auto mutate_columns = block->mutate_columns();
+        for (auto& col : mutate_columns) {
             col->resize(rows);
         }
+        block->set_columns(std::move(mutate_columns));
 
         *read_rows = rows;
         if (_current_group_reader->get_remaining_rows() == 0) {
diff --git a/be/src/vec/exec/format/table/iceberg_reader.cpp b/be/src/vec/exec/format/table/iceberg_reader.cpp
index 5ad1bdd315df56..8c05b8c2a08e70 100644
--- a/be/src/vec/exec/format/table/iceberg_reader.cpp
+++ b/be/src/vec/exec/format/table/iceberg_reader.cpp
@@ -143,9 +143,11 @@ Status IcebergTableReader::get_next_block(Block* block, size_t* read_rows, bool*
         auto rows =
                 std::min(_remaining_push_down_count, (int64_t)_state->query_options().batch_size);
         _remaining_push_down_count -= rows;
-        for (auto& col : block->mutate_columns()) {
+        auto mutate_columns = block->mutate_columns();
+        for (auto& col : mutate_columns) {
             col->resize(rows);
         }
+        block->set_columns(std::move(mutate_columns));
         *read_rows = rows;
         if (_remaining_push_down_count == 0) {
             *eof = true;
diff --git a/be/src/vec/exec/join/vjoin_node_base.cpp b/be/src/vec/exec/join/vjoin_node_base.cpp
index 0077fe2a7bfbaa..656810ba7bd39d 100644
--- a/be/src/vec/exec/join/vjoin_node_base.cpp
+++ b/be/src/vec/exec/join/vjoin_node_base.cpp
@@ -177,6 +177,7 @@ Status VJoinNodeBase::_build_output_block(Block* origin_block, Block* output_blo
             }
         }
     };
+
     if (rows != 0) {
         auto& mutable_columns = mutable_block.mutable_columns();
         if (_output_expr_ctxs.empty()) {
@@ -207,9 +208,7 @@ Status VJoinNodeBase::_build_output_block(Block* origin_block, Block* output_blo
             }
         }
 
-        if (!is_mem_reuse || !keep_origin) {
-            output_block->swap(mutable_block.to_block());
-        }
+        output_block->swap(mutable_block.to_block());
         DCHECK(output_block->rows() == rows);
     }
 
diff --git a/be/src/vec/exec/join/vnested_loop_join_node.cpp b/be/src/vec/exec/join/vnested_loop_join_node.cpp
index 7d8100aa6caa18..08d4c7d4487c37 100644
--- a/be/src/vec/exec/join/vnested_loop_join_node.cpp
+++ b/be/src/vec/exec/join/vnested_loop_join_node.cpp
@@ -278,8 +278,8 @@ Status VNestedLoopJoinNode::get_next(RuntimeState* state, Block* block, bool* eo
     return pull(state, block, eos);
 }
 
-void VNestedLoopJoinNode::_append_left_data_with_null(MutableBlock& mutable_block) const {
-    auto& dst_columns = mutable_block.mutable_columns();
+void VNestedLoopJoinNode::_append_left_data_with_null(Block& block) const {
+    auto dst_columns = block.mutate_columns();
     DCHECK(_is_mark_join);
     for (size_t i = 0; i < _num_probe_side_columns; ++i) {
         const ColumnWithTypeAndName& src_column = _left_block->get_by_position(i);
@@ -305,11 +305,12 @@ void VNestedLoopJoinNode::_append_left_data_with_null(MutableBlock& mutable_bloc
 
     auto& mark_column = *dst_columns[dst_columns.size() - 1];
     ColumnFilterHelper(mark_column).resize_fill(mark_column.size() + _left_side_process_count, 0);
+    block.set_columns(std::move(dst_columns));
 }
 
-void VNestedLoopJoinNode::_process_left_child_block(MutableBlock& mutable_block,
+void VNestedLoopJoinNode::_process_left_child_block(Block& block,
                                                     const Block& now_process_build_block) const {
-    auto& dst_columns = mutable_block.mutable_columns();
+    auto dst_columns = block.mutate_columns();
     const int max_added_rows = now_process_build_block.rows();
     for (size_t i = 0; i < _num_probe_side_columns; ++i) {
         const ColumnWithTypeAndName& src_column = _left_block->get_by_position(i);
@@ -345,6 +346,7 @@ void VNestedLoopJoinNode::_process_left_child_block(MutableBlock& mutable_block,
                                                                         max_added_rows);
         }
     }
+    block.set_columns(std::move(dst_columns));
 }
 
 void VNestedLoopJoinNode::_update_additional_flags(Block* block) {
@@ -395,8 +397,8 @@ void VNestedLoopJoinNode::_add_tuple_is_null_column(Block* block) {
 }
 
 template <bool BuildSide, bool IsSemi>
-void VNestedLoopJoinNode::_finalize_current_phase(MutableBlock& mutable_block, size_t batch_size) {
-    auto& dst_columns = mutable_block.mutable_columns();
+void VNestedLoopJoinNode::_finalize_current_phase(Block& block, size_t batch_size) {
+    auto dst_columns = block.mutate_columns();
     DCHECK_GT(dst_columns.size(), 0);
     auto column_size = dst_columns[0]->size();
     if constexpr (BuildSide) {
@@ -508,6 +510,7 @@ void VNestedLoopJoinNode::_finalize_current_phase(MutableBlock& mutable_block, s
             _resize_fill_tuple_is_null_column(_left_side_process_count, 0, 1);
         }
     }
+    block.set_columns(std::move(dst_columns));
 }
 
 void VNestedLoopJoinNode::_reset_with_next_probe_row() {
diff --git a/be/src/vec/exec/join/vnested_loop_join_node.h b/be/src/vec/exec/join/vnested_loop_join_node.h
index 810bf57e7f5e8b..732648592745c9 100644
--- a/be/src/vec/exec/join/vnested_loop_join_node.h
+++ b/be/src/vec/exec/join/vnested_loop_join_node.h
@@ -115,7 +115,6 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
         _left_side_process_count = 0;
         DCHECK(!_need_more_input_data || !_matched_rows_done);
 
-        MutableBlock mutable_join_block(&_join_block);
         if (!_matched_rows_done && !_need_more_input_data) {
             // We should try to join rows if there still are some rows from probe side.
             while (_join_block.rows() < state->batch_size()) {
@@ -129,7 +128,7 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
                     _reset_with_next_probe_row();
                     if (_left_block_pos < _left_block->rows()) {
                         if constexpr (set_probe_side_flag) {
-                            _probe_offset_stack.push(mutable_join_block.rows());
+                            _probe_offset_stack.push(_join_block.rows());
                         }
                     } else {
                         if (_left_side_eos) {
@@ -148,9 +147,9 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
 
                 const auto& now_process_build_block = _build_blocks[_current_build_pos++];
                 if constexpr (set_build_side_flag) {
-                    _build_offset_stack.push(mutable_join_block.rows());
+                    _build_offset_stack.push(_join_block.rows());
                 }
-                _process_left_child_block(mutable_join_block, now_process_build_block);
+                _process_left_child_block(_join_block, now_process_build_block);
             }
 
             if constexpr (set_probe_side_flag) {
@@ -163,21 +162,20 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
                 if (!status.ok()) {
                     return status;
                 }
-                mutable_join_block = MutableBlock(&_join_block);
                 // If this join operation is left outer join or full outer join, when
                 // `_left_side_process_count`, means all rows from build
                 // side have been joined with _left_side_process_count, we should output current
                 // probe row with null from build side.
                 if (_left_side_process_count) {
                     _finalize_current_phase<false, JoinOpType::value == TJoinOp::LEFT_SEMI_JOIN>(
-                            mutable_join_block, state->batch_size());
+                            _join_block, state->batch_size());
                 }
             }
 
             if (_left_side_process_count) {
                 if (_is_mark_join && _build_blocks.empty()) {
                     DCHECK_EQ(JoinOpType::value, TJoinOp::CROSS_JOIN);
-                    _append_left_data_with_null(mutable_join_block);
+                    _append_left_data_with_null(_join_block);
                 }
             }
         }
@@ -189,7 +187,6 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
                              set_build_side_flag, set_probe_side_flag, ignore_null>(
                              &_join_block, !_is_right_semi_anti)));
             _update_additional_flags(&_join_block);
-            mutable_join_block = MutableBlock(&_join_block);
             if (!status.ok()) {
                 return status;
             }
@@ -198,7 +195,7 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
         if constexpr (set_build_side_flag) {
             if (_matched_rows_done && _output_null_idx_build_side < _build_blocks.size()) {
                 _finalize_current_phase<true, JoinOpType::value == TJoinOp::RIGHT_SEMI_JOIN>(
-                        mutable_join_block, state->batch_size());
+                        _join_block, state->batch_size());
             }
         }
         return Status::OK();
@@ -208,8 +205,7 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
     // Processes a block from the left child.
     //  dst_columns: left_child_row and now_process_build_block to construct a bundle column of new block
     //  now_process_build_block: right child block now to process
-    void _process_left_child_block(MutableBlock& mutable_block,
-                                   const Block& now_process_build_block) const;
+    void _process_left_child_block(Block& block, const Block& now_process_build_block) const;
 
     template <bool SetBuildSideFlag, bool SetProbeSideFlag, bool IgnoreNull>
     Status _do_filtering_and_update_visited_flags(Block* block, bool materialize);
@@ -221,7 +217,7 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
                                                      bool materialize, Filter& filter);
 
     template <bool BuildSide, bool IsSemi>
-    void _finalize_current_phase(MutableBlock& mutable_block, size_t batch_size);
+    void _finalize_current_phase(Block& block, size_t batch_size);
 
     void _reset_with_next_probe_row();
 
@@ -238,7 +234,7 @@ class VNestedLoopJoinNode final : public VJoinNodeBase {
 
     // For mark join, if the relation from right side is empty, we should construct intermediate
     // block with data from left side and filled with null for right side
-    void _append_left_data_with_null(MutableBlock& mutable_block) const;
+    void _append_left_data_with_null(Block& block) const;
 
     // List of build blocks, constructed in prepare()
     Blocks _build_blocks;
diff --git a/be/src/vec/exec/scan/pip_scanner_context.h b/be/src/vec/exec/scan/pip_scanner_context.h
index 309aed96a8cf7f..42726cb17fe0f0 100644
--- a/be/src/vec/exec/scan/pip_scanner_context.h
+++ b/be/src/vec/exec/scan/pip_scanner_context.h
@@ -106,6 +106,7 @@ class PipScannerContext : public vectorized::ScannerContext {
                 static_cast<void>(m.merge(*merge_block));
                 return_free_block(std::move(merge_block));
             }
+            (*block)->set_columns(std::move(m.mutable_columns()));
         }
 
         return Status::OK();
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index 5ad2dbec5b69fa..4d3820d7634074 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -339,6 +339,7 @@ Status ScannerContext::get_block_from_queue(RuntimeState* state, vectorized::Blo
             static_cast<void>(m.merge(*merge_block));
             return_free_block(std::move(merge_block));
         }
+        (*block)->set_columns(std::move(m.mutable_columns()));
     }
 
     return Status::OK();
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp
index e8d7f8a7139a6d..42285f1d22a854 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -374,7 +374,9 @@ void ScannerScheduler::_scanner_scan(ScannerScheduler* scheduler,
             ctx->return_free_block(std::move(block));
         } else {
             if (!blocks.empty() && blocks.back()->rows() + block->rows() <= state->batch_size()) {
-                static_cast<void>(vectorized::MutableBlock(blocks.back().get()).merge(*block));
+                vectorized::MutableBlock mutable_block(blocks.back().get());
+                static_cast<void>(mutable_block.merge(*block));
+                blocks.back().get()->set_columns(std::move(mutable_block.mutable_columns()));
                 ctx->return_free_block(std::move(block));
             } else {
                 blocks.push_back(std::move(block));
diff --git a/be/src/vec/exec/vpartition_sort_node.h b/be/src/vec/exec/vpartition_sort_node.h
index acf18bad7b4ba0..d369846df5926c 100644
--- a/be/src/vec/exec/vpartition_sort_node.h
+++ b/be/src/vec/exec/vpartition_sort_node.h
@@ -58,6 +58,7 @@ struct PartitionBlocks {
         for (int i = 0; i < mutable_columns.size(); ++i) {
             columns[i]->append_data_by_selector(mutable_columns[i], selector);
         }
+        blocks.back()->set_columns(std::move(mutable_columns));
         init_rows = init_rows - selector.size();
         total_rows = total_rows + selector.size();
         selector.clear();
diff --git a/be/src/vec/exec/vrepeat_node.cpp b/be/src/vec/exec/vrepeat_node.cpp
index 1765c1dcf7f44c..717c0c28d08b67 100644
--- a/be/src/vec/exec/vrepeat_node.cpp
+++ b/be/src/vec/exec/vrepeat_node.cpp
@@ -167,7 +167,7 @@ Status VRepeatNode::get_repeated_block(Block* child_block, int repeat_id_idx, Bl
         }
         cur_col++;
     }
-
+    output_block->set_columns(std::move(columns));
     DCHECK_EQ(cur_col, column_size);
 
     return Status::OK();
diff --git a/be/src/vec/exec/vtable_function_node.cpp b/be/src/vec/exec/vtable_function_node.cpp
index b7302aedcdc747..8affd4cbe7e265 100644
--- a/be/src/vec/exec/vtable_function_node.cpp
+++ b/be/src/vec/exec/vtable_function_node.cpp
@@ -218,7 +218,7 @@ Status VTableFunctionNode::_get_expanded_block(RuntimeState* state, Block* outpu
     for (auto index : _useless_slot_indexs) {
         columns[index]->insert_many_defaults(row_size - columns[index]->size());
     }
-
+    output_block->set_columns(std::move(columns));
     // 3. eval conjuncts
     RETURN_IF_ERROR(VExprContext::filter_block(_conjuncts, output_block, output_block->columns()));
 
diff --git a/be/src/vec/exec/vunion_node.cpp b/be/src/vec/exec/vunion_node.cpp
index 884b66347b5c13..8ee258ca05678e 100644
--- a/be/src/vec/exec/vunion_node.cpp
+++ b/be/src/vec/exec/vunion_node.cpp
@@ -205,6 +205,7 @@ Status VUnionNode::get_next_materialized(RuntimeState* state, Block* block) {
             ++_child_idx;
         }
     }
+    block->set_columns(std::move(mblock.mutable_columns()));
 
     DCHECK_LE(_child_idx, _children.size());
     return Status::OK();
@@ -233,6 +234,7 @@ Status VUnionNode::get_next_const(RuntimeState* state, Block* block) {
             tmp_block.clear();
         }
     }
+    block->set_columns(std::move(mblock.mutable_columns()));
 
     // some insert query like "insert into string_test select 1, repeat('a', 1024 * 1024);"
     // the const expr will be in output expr cause the union node return a empty block. so here we
@@ -256,6 +258,8 @@ Status VUnionNode::materialize_child_block(RuntimeState* state, int child_id,
         Block res;
         RETURN_IF_ERROR(materialize_block(input_block, child_id, &res));
         RETURN_IF_ERROR(mblock.merge(res));
+
+        output_block->set_columns(std::move(mblock.mutable_columns()));
     }
     return Status::OK();
 }
diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp
index 9c15302af6c0b4..82604ac4083307 100644
--- a/be/src/vec/olap/block_reader.cpp
+++ b/be/src/vec/olap/block_reader.cpp
@@ -337,6 +337,7 @@ Status BlockReader::_agg_key_next_block(Block* block, bool* eof) {
     _agg_data_counters.push_back(_last_agg_data_counter);
     _last_agg_data_counter = 0;
     _update_agg_data(target_columns);
+    block->set_columns(std::move(target_columns));
 
     _merged_rows += merged_row;
     return Status::OK();
@@ -410,17 +411,20 @@ Status BlockReader::_unique_key_next_block(Block* block, bool* eof) {
                 }
             }
         }
-
+        auto target_columns_size = target_columns.size();
         ColumnWithTypeAndName column_with_type_and_name {_delete_filter_column,
                                                          std::make_shared<DataTypeUInt8>(),
                                                          "__DORIS_COMPACTION_FILTER__"};
+        block->set_columns(std::move(target_columns));
         block->insert(column_with_type_and_name);
-        RETURN_IF_ERROR(Block::filter_block(block, target_columns.size(), target_columns.size()));
+        RETURN_IF_ERROR(Block::filter_block(block, target_columns_size, target_columns_size));
         _stats.rows_del_filtered += target_block_row - block->rows();
         DCHECK(block->try_get_by_name("__DORIS_COMPACTION_FILTER__") == nullptr);
         if (UNLIKELY(_reader_context.record_rowids)) {
             DCHECK_EQ(_block_row_locations.size(), block->rows() + delete_count);
         }
+    } else {
+        block->set_columns(std::move(target_columns));
     }
     return Status::OK();
 }
diff --git a/be/src/vec/olap/vcollect_iterator.cpp b/be/src/vec/olap/vcollect_iterator.cpp
index 921e89c3c8a4e2..ca9e2b729671d9 100644
--- a/be/src/vec/olap/vcollect_iterator.cpp
+++ b/be/src/vec/olap/vcollect_iterator.cpp
@@ -770,7 +770,7 @@ Status VCollectIterator::Level1Iterator::_normal_next(IteratorRowRef* ref) {
 Status VCollectIterator::Level1Iterator::_merge_next(Block* block) {
     int target_block_row = 0;
     auto target_columns = block->mutate_columns();
-    size_t column_count = block->columns();
+    size_t column_count = target_columns.size();
     IteratorRowRef cur_row = _ref;
     IteratorRowRef pre_row_ref = _ref;
 
@@ -809,6 +809,7 @@ Status VCollectIterator::Level1Iterator::_merge_next(Block* block) {
             if (UNLIKELY(_reader->_reader_context.record_rowids)) {
                 _block_row_locations.resize(target_block_row);
             }
+            block->set_columns(std::move(target_columns));
             return res;
         }
 
@@ -825,6 +826,7 @@ Status VCollectIterator::Level1Iterator::_merge_next(Block* block) {
                                                          continuous_row_in_block);
                 }
             }
+            block->set_columns(std::move(target_columns));
             return Status::OK();
         }
         if (continuous_row_in_block == 0) {
diff --git a/be/src/vec/olap/vertical_block_reader.cpp b/be/src/vec/olap/vertical_block_reader.cpp
index 3f4a0cd3a1299b..5f2d856fe794bd 100644
--- a/be/src/vec/olap/vertical_block_reader.cpp
+++ b/be/src/vec/olap/vertical_block_reader.cpp
@@ -405,6 +405,7 @@ Status VerticalBlockReader::_agg_key_next_block(Block* block, bool* eof) {
     _agg_data_counters.push_back(_last_agg_data_counter);
     _last_agg_data_counter = 0;
     _update_agg_data(target_columns);
+    block->set_columns(std::move(target_columns));
 
     return Status::OK();
 }
@@ -545,6 +546,7 @@ Status VerticalBlockReader::_unique_key_next_block(Block* block, bool* eof) {
         });
         ++target_block_row;
     } while (target_block_row < _reader_context.batch_size);
+    block->set_columns(std::move(target_columns));
     return Status::OK();
 }
 
diff --git a/be/src/vec/olap/vgeneric_iterators.cpp b/be/src/vec/olap/vgeneric_iterators.cpp
index 65dc8e3e0dc6f3..155bb4ee076be8 100644
--- a/be/src/vec/olap/vgeneric_iterators.cpp
+++ b/be/src/vec/olap/vgeneric_iterators.cpp
@@ -75,14 +75,15 @@ Status VStatisticsIterator::next_batch(Block* block) {
                               : std::min(_target_rows - _output_rows, MAX_ROW_SIZE_IN_COUNT);
         if (_push_down_agg_type_opt == TPushAggOp::COUNT) {
             size = std::min(_target_rows - _output_rows, MAX_ROW_SIZE_IN_COUNT);
-            for (int i = 0; i < block->columns(); ++i) {
+            for (int i = 0; i < columns.size(); ++i) {
                 columns[i]->resize(size);
             }
         } else {
-            for (int i = 0; i < block->columns(); ++i) {
+            for (int i = 0; i < columns.size(); ++i) {
                 RETURN_IF_ERROR(_column_iterators[i]->next_batch_of_zone_map(&size, columns[i]));
             }
         }
+        block->set_columns(std::move(columns));
         _output_rows += size;
         return Status::OK();
     }
diff --git a/be/src/vec/runtime/vsorted_run_merger.cpp b/be/src/vec/runtime/vsorted_run_merger.cpp
index 1bdd82ba8e4f68..6d2f68db6287df 100644
--- a/be/src/vec/runtime/vsorted_run_merger.cpp
+++ b/be/src/vec/runtime/vsorted_run_merger.cpp
@@ -194,6 +194,7 @@ Status VSortedRunMerger::get_next(Block* output_block, bool* eos) {
                 break;
             }
         }
+        output_block->set_columns(std::move(merged_columns));
 
         if (merged_rows == 0) {
             *eos = true;
diff --git a/be/test/vec/core/block_test.cpp b/be/test/vec/core/block_test.cpp
index 020c4f2e923cd6..7e8a26ce438a2c 100644
--- a/be/test/vec/core/block_test.cpp
+++ b/be/test/vec/core/block_test.cpp
@@ -390,4 +390,57 @@ TEST(BlockTest, dump_data) {
     vectorized::Block::filter_block_internal(&block1, filter, block1.columns());
     EXPECT_EQ(size, block1.rows());
 }
+
+TEST(BlockTest, merge_with_shared_columns) {
+    auto vec = vectorized::ColumnVector<Int32>::create();
+    auto& int32_data = vec->get_data();
+    for (int i = 0; i < 1024; ++i) {
+        int32_data.push_back(i);
+    }
+    vectorized::DataTypePtr int32_type(std::make_shared<vectorized::DataTypeInt32>());
+    vectorized::ColumnWithTypeAndName test_k1(vec->get_ptr(), int32_type, "k1");
+
+    auto strcol = vectorized::ColumnString::create();
+    for (int i = 0; i < 1024; ++i) {
+        std::string is = std::to_string(i);
+        strcol->insert_data(is.c_str(), is.size());
+    }
+    vectorized::DataTypePtr string_type(std::make_shared<vectorized::DataTypeString>());
+    vectorized::ColumnWithTypeAndName test_v1(strcol->get_ptr(), string_type, "v1");
+
+    vectorized::ColumnWithTypeAndName test_v2(strcol->get_ptr(), string_type, "v2");
+
+    vectorized::Block src_block({test_k1, test_v1, test_v2});
+
+    auto vec_temp = vectorized::ColumnVector<Int32>::create();
+    auto& int32_data_temp = vec_temp->get_data();
+    for (int i = 0; i < 10; ++i) {
+        int32_data_temp.push_back(i);
+    }
+
+    vectorized::ColumnWithTypeAndName test_k1_temp(vec_temp->get_ptr(), int32_type, "k1");
+
+    auto strcol_temp = vectorized::ColumnString::create();
+    for (int i = 0; i < 10; ++i) {
+        std::string is = std::to_string(i);
+        strcol_temp->insert_data(is.c_str(), is.size());
+    }
+
+    vectorized::ColumnWithTypeAndName test_v1_temp(strcol_temp->get_ptr(), string_type, "v1");
+    vectorized::ColumnWithTypeAndName test_v2_temp(strcol_temp->get_ptr(), string_type, "v2");
+
+    vectorized::Block temp_block({test_k1_temp, test_v1_temp, test_v2_temp});
+
+    vectorized::MutableBlock mutable_block(&src_block);
+    auto status = mutable_block.merge(temp_block);
+    ASSERT_TRUE(status.ok());
+
+    src_block.set_columns(std::move(mutable_block.mutable_columns()));
+
+    for (auto& column : src_block.get_columns()) {
+        EXPECT_EQ(1034, column->size());
+    }
+    EXPECT_EQ(1034, src_block.rows());
+}
+
 } // namespace doris

From 8921b313b43172c002821ef14650df8ae8bc2891 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Mon, 25 Dec 2023 20:02:37 +0700
Subject: [PATCH 015/299] [fix](doc) typo fix in dynamic-partition page
 (#28511)

---
 docs/en/docs/advanced/partition/dynamic-partition.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/docs/advanced/partition/dynamic-partition.md b/docs/en/docs/advanced/partition/dynamic-partition.md
index 99ab594cdd809a..037caa28805b22 100644
--- a/docs/en/docs/advanced/partition/dynamic-partition.md
+++ b/docs/en/docs/advanced/partition/dynamic-partition.md
@@ -206,7 +206,7 @@ Assuming the number of history partitions to be created is `expect_create_partit
         `expect_create_partition_num` = `end` - `start`; 
 
    - `dynamic_partition.history_partition_num` is set   
-        `expect_create_partition_num` = `end` - max(`start`, `-histoty_partition_num`);
+        `expect_create_partition_num` = `end` - max(`start`, `-history_partition_num`);
 
 2. `create_history_partition` = `false`  
     No history partition will be created, `expect_create_partition_num` = `end` - 0;

From 0af9371a96dccd7ddbfcad13cffc25a46b858158 Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Mon, 25 Dec 2023 22:19:01 +0800
Subject: [PATCH 016/299] [fix](hash join) fix column ref DCHECK failure of
 hash join node block mem reuse (#28991)

Introduced by #28851, after evaluating build side expr, some columns in resulting block may be referenced more than once in the same block.

e.g. coalesce(col_a, 'string') if col_a is nullable but actually contains no null values, in this case funcition coalesce will insert a new nullable column which references the original col_a.
---
 be/src/vec/exec/join/vhash_join_node.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp
index 30f2450f458c11..60391716e28af6 100644
--- a/be/src/vec/exec/join/vhash_join_node.cpp
+++ b/be/src/vec/exec/join/vhash_join_node.cpp
@@ -692,7 +692,7 @@ Status HashJoinNode::_materialize_build_side(RuntimeState* state) {
         // data from data.
         while (!eos && (!_short_circuit_for_null_in_build_side || !_has_null_in_build_side) &&
                (!_probe_open_finish || !_is_hash_join_early_start_probe_eos(state))) {
-            block.clear_column_data();
+            release_block_memory(block, 1);
             RETURN_IF_CANCELLED(state);
             {
                 SCOPED_TIMER(_build_get_next_timer);

From c2c5df9341a310c442bf0b3f489ef30a702a2982 Mon Sep 17 00:00:00 2001
From: zhiqiang <seuhezhiqiang@163.com>
Date: Mon, 25 Dec 2023 22:47:23 +0800
Subject: [PATCH 017/299] [opt](assert_num_rows) support filter in
 AssertNumRows operator and fix some explain (#28935)

* NEED

* Update pipeline x

* fix pipelinex compile
---
 .../exec/assert_num_rows_operator.cpp         |  3 +
 be/src/vec/exec/vassert_num_rows_node.cpp     |  2 +
 .../doris/planner/AssertNumRowsNode.java      |  5 +
 .../apache/doris/planner/DataGenScanNode.java | 20 ++++
 .../doris/planner/PartitionSortNode.java      |  4 +
 .../correctness_p0/test_assert_row_num.out    | 17 ++++
 .../correctness_p0/test_assert_row_num.groovy | 91 +++++++++++++++++++
 7 files changed, 142 insertions(+)
 create mode 100644 regression-test/data/correctness_p0/test_assert_row_num.out
 create mode 100644 regression-test/suites/correctness_p0/test_assert_row_num.groovy

diff --git a/be/src/pipeline/exec/assert_num_rows_operator.cpp b/be/src/pipeline/exec/assert_num_rows_operator.cpp
index baddce5a94d538..d08f60cce74218 100644
--- a/be/src/pipeline/exec/assert_num_rows_operator.cpp
+++ b/be/src/pipeline/exec/assert_num_rows_operator.cpp
@@ -17,6 +17,8 @@
 
 #include "assert_num_rows_operator.h"
 
+#include "vec/exprs/vexpr_context.h"
+
 namespace doris::pipeline {
 
 OperatorPtr AssertNumRowsOperatorBuilder::build_operator() {
@@ -83,6 +85,7 @@ Status AssertNumRowsOperatorX::pull(doris::RuntimeState* state, vectorized::Bloc
     }
     COUNTER_SET(local_state.rows_returned_counter(), local_state.num_rows_returned());
     COUNTER_UPDATE(local_state.blocks_returned_counter(), 1);
+    RETURN_IF_ERROR(vectorized::VExprContext::filter_block(_conjuncts, block, block->columns()));
     return Status::OK();
 }
 
diff --git a/be/src/vec/exec/vassert_num_rows_node.cpp b/be/src/vec/exec/vassert_num_rows_node.cpp
index f01c4a254b0802..9ef1166e2fb29e 100644
--- a/be/src/vec/exec/vassert_num_rows_node.cpp
+++ b/be/src/vec/exec/vassert_num_rows_node.cpp
@@ -29,6 +29,7 @@
 
 #include "runtime/runtime_state.h"
 #include "vec/core/block.h"
+#include "vec/exprs/vexpr_context.h"
 
 namespace doris {
 class DescriptorTbl;
@@ -100,6 +101,7 @@ Status VAssertNumRowsNode::pull(doris::RuntimeState* state, vectorized::Block* b
                                  to_string_lambda(_assertion), _desired_num_rows, _subquery_string);
     }
     COUNTER_SET(_rows_returned_counter, _num_rows_returned);
+    RETURN_IF_ERROR(VExprContext::filter_block(_conjuncts, block, block->columns()));
     return Status::OK();
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
index 84ac5520ee294f..0ae50ce0e4c756 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
@@ -79,6 +79,11 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
         StringBuilder output = new StringBuilder()
                 .append(prefix).append("assert number of rows: ")
                 .append(assertion).append(" ").append(desiredNumOfRows).append("\n");
+
+        if (!conjuncts.isEmpty()) {
+            output.append(prefix).append("predicates: ").append(getExplainString(conjuncts)).append("\n");
+        }
+
         return output.toString();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
index d00641d135c86f..457c643182ece5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java
@@ -26,6 +26,7 @@
 import org.apache.doris.tablefunction.DataGenTableValuedFunction;
 import org.apache.doris.tablefunction.TableValuedFunctionTask;
 import org.apache.doris.thrift.TDataGenScanNode;
+import org.apache.doris.thrift.TExplainLevel;
 import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TPlanNode;
 import org.apache.doris.thrift.TPlanNodeType;
@@ -117,4 +118,23 @@ public boolean needToCheckColumnPriv() {
     public int getNumInstances() {
         return 1;
     }
+
+    @Override
+    public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
+        if (detailLevel == TExplainLevel.BRIEF) {
+            return "";
+        }
+
+        StringBuilder output = new StringBuilder();
+
+        if (!conjuncts.isEmpty()) {
+            output.append(prefix).append("predicates: ").append(getExplainString(conjuncts)).append("\n");
+        }
+
+        output.append(prefix).append("table value function: ").append(tvf.getDataGenFunctionName()).append("\n");
+
+
+
+        return output.toString();
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
index fc63af3b6c1cce..20142e380ce710 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PartitionSortNode.java
@@ -120,6 +120,10 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
         }
         output.append("\n");
 
+        if (!conjuncts.isEmpty()) {
+            output.append(prefix).append("predicates: ").append(getExplainString(conjuncts)).append("\n");
+        }
+
         // Add the limit information;
         output.append(prefix).append("has global limit: ").append(hasGlobalLimit).append("\n");
         output.append(prefix).append("partition limit: ").append(partitionLimit).append("\n");
diff --git a/regression-test/data/correctness_p0/test_assert_row_num.out b/regression-test/data/correctness_p0/test_assert_row_num.out
new file mode 100644
index 00000000000000..0ac1221fb698a7
--- /dev/null
+++ b/regression-test/data/correctness_p0/test_assert_row_num.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql_1 --
+
+-- !sql_2 --
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+
+-- !sql_3 --
+
diff --git a/regression-test/suites/correctness_p0/test_assert_row_num.groovy b/regression-test/suites/correctness_p0/test_assert_row_num.groovy
new file mode 100644
index 00000000000000..818213f56fee89
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_assert_row_num.groovy
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_assert_num_rows") {
+    qt_sql_1 """
+        SELECT * from numbers("number"="10") WHERE  ( SELECT * FROM (SELECT 3) __DORIS_DUAL__ ) IS  NULL
+    """
+
+    qt_sql_2 """
+        SELECT * from numbers("number"="10") WHERE  ( SELECT * FROM (SELECT 3) __DORIS_DUAL__ ) IS NOT NULL
+    """
+    sql """
+        DROP TABLE IF EXISTS table_9_undef_undef;
+    """
+    sql """
+        DROP TABLE IF EXISTS table_10_undef_undef;
+    """
+    sql """
+    CREATE TABLE table_10_undef_undef (
+        `pk` int, `col_int_undef_signed` int ,
+        `col_varchar_10__undef_signed` varchar(10),
+        `col_varchar_1024__undef_signed` varchar(1024))
+    ENGINE=olap distributed BY hash(pk) buckets 10 properties('replication_num'='1');
+    """
+
+    sql """
+    CREATE TABLE table_9_undef_undef (
+        `pk` int,`col_int_undef_signed` int ,
+        `col_varchar_10__undef_signed` varchar(10) ,
+        `col_varchar_1024__undef_signed` varchar(1024))
+    ENGINE=olap distributed BY hash(pk) buckets 10 properties('replication_num' = '1');
+    """
+
+    sql """
+        INSERT INTO table_9_undef_undef
+        VALUES (0,NULL,"get",'r'),
+            (1,NULL,'q','i'),
+            (2,2,"about","yes"),
+            (3,NULL,"see","see"),
+            (4,NULL,"was","been"),
+            (5,NULL,"yes",'p'),
+            (6,6,"you",'u'),
+            (7,0,"me",'v'),
+            (8,5,"something",'f');
+    """
+    sql """
+        INSERT INTO table_10_undef_undef
+        VALUES (0,NULL,"it's","time"),
+            (1,NULL,"right",'o'),
+            (2,5,'y','k'),
+            (3,1,'r',"I'll"),
+            (4,2,'e',"time"),
+            (5,8,'v',"from"),
+            (6,NULL,"you",'v'),
+            (7,NULL,'r','a'),
+            (8,1,'d',"didn't"),
+            (9,NULL,'r',"go");
+    """
+
+    qt_sql_3 """
+        SELECT alias1 . `pk` AS field1,
+            alias1 . `col_int_undef_signed` AS field2
+        FROM table_10_undef_undef AS alias1,
+            table_9_undef_undef AS alias2
+        WHERE
+            (SELECT *
+            FROM
+            (SELECT 3) __DORIS_DUAL__) IS NULL
+        HAVING field2 < 2
+        ORDER BY alias1 . `pk`,
+                alias1 .`pk` ASC,
+                field1,
+                field2
+        LIMIT 2
+        OFFSET 6;
+    """
+}
\ No newline at end of file

From 137f785698023640eddf598014c67579b43826af Mon Sep 17 00:00:00 2001
From: py023 <137761033+py023@users.noreply.github.com>
Date: Mon, 25 Dec 2023 22:58:08 +0800
Subject: [PATCH 018/299] [fix](parquet_reader) misused bool pointer (#28986)

Signed-off-by: pengyu <pengyu@selectdb.com>
---
 be/src/vec/exec/format/parquet/vparquet_column_reader.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
index 99d6f6b4e85849..a675046e1fd92b 100644
--- a/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
+++ b/be/src/vec/exec/format/parquet/vparquet_column_reader.cpp
@@ -447,7 +447,7 @@ Status ScalarColumnReader::read_dict_values_to_column(MutableColumnPtr& doris_co
                                                       bool* has_dict) {
     bool loaded;
     RETURN_IF_ERROR(_try_load_dict_page(&loaded, has_dict));
-    if (loaded && has_dict) {
+    if (loaded && *has_dict) {
         return _chunk_reader->read_dict_values_to_column(doris_column);
     }
     return Status::OK();

From 2411dbe0e6d9e78931535472d8ff0d8ab0af5b8a Mon Sep 17 00:00:00 2001
From: slothever <18522955+wsjz@users.noreply.github.com>
Date: Mon, 25 Dec 2023 23:05:44 +0800
Subject: [PATCH 019/299] [doc](multi-catalog)add krb and some ertificates FAQs
 (#28858)

add some security docs
---
 docs/en/docs/lakehouse/faq.md                 | 18 ++++++++++++++++++
 docs/en/docs/lakehouse/multi-catalog/hive.md  |  2 ++
 .../docs/lakehouse/multi-catalog/iceberg.md   |  2 ++
 docs/zh-CN/docs/lakehouse/faq.md              | 19 +++++++++++++++++++
 .../docs/lakehouse/multi-catalog/hive.md      |  2 ++
 .../docs/lakehouse/multi-catalog/iceberg.md   |  2 ++
 6 files changed, 45 insertions(+)

diff --git a/docs/en/docs/lakehouse/faq.md b/docs/en/docs/lakehouse/faq.md
index ee441961c963cd..3b20fa878c2005 100644
--- a/docs/en/docs/lakehouse/faq.md
+++ b/docs/en/docs/lakehouse/faq.md
@@ -27,6 +27,19 @@ under the License.
 
 # FAQ
 
+## Certificates
+
+1. If an error is reported: `curl 77: Problem with the SSL CA cert.`, need update your certificate.
+   - Download the latest certificate from `https://curl.haxx.se/docs/caextract.html`.
+   - Place the downloaded cacert-xxx.pem in the `/etc/ssl/certs/` directory. For example: `sudo cp cacert-xxx.pem  /etc/ssl/certs/ca-certificates.crt`.
+
+2. If an error is reported: `ERROR 1105 (HY000): errCode = 2, detailMessage = (x.x.x.x)[CANCELLED][INTERNAL_ERROR]error setting certificate verify locations:  CAfile: /etc/ssl/certs/ca-certificates.crt CApath: none`.
+
+```
+yum install -y ca-certificates
+ln -s /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt /etc/ssl/certs/ca-certificates.crt
+```
+
 ## Kerberos
 
 
@@ -58,6 +71,11 @@ under the License.
     - The principal used must exist in the klist, use `klist -kt your.keytab` to check.
     - Ensure the catalog configuration correct, such as missing the `yarn.resourcemanager.principal`.
     - If the preceding checks are correct, the JDK version installed by yum or other package-management utility in the current system maybe have an unsupported encryption algorithm. It is recommended to install JDK by yourself and set `JAVA_HOME` environment variable.
+    - Kerberos uses AES-256 by default for encryption. If you use Oracle JDK, you must install JCE. In the case of OpenJDK, some distributions of OpenJDK automatically provide the JCE Unlimited Strength Jurisdiction Policy Files, so it's not need to install JCE.
+    - The JCE version corresponds to the JDK version. You need to select the JCE according to the JDK version. Download the JCE zip package and decompress it into `$JAVA_HOME/jre/lib/security`:
+       - JDK6：[JCE6](http://www.oracle.com/technetwork/java/javase/downloads/jce-6-download-429243.html)
+       - JDK7：[JCE7](http://www.oracle.com/technetwork/java/embedded/embedded-se/downloads/jce-7-download-432124.html)
+       - JDK8：[JCE8](http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
 
 5. An error is reported when using KMS to access HDFS: `java.security.InvalidKeyException: Illegal key size`
    
diff --git a/docs/en/docs/lakehouse/multi-catalog/hive.md b/docs/en/docs/lakehouse/multi-catalog/hive.md
index 754c3954f5268e..25fddea125064d 100644
--- a/docs/en/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/en/docs/lakehouse/multi-catalog/hive.md
@@ -168,6 +168,8 @@ CREATE CATALOG hive PROPERTIES (
 
 ### Hive With Glue
 
+> When connecting Glue, if it's not on the EC2 environment, need copy the `~/.aws` from the EC2 environment to the current environment. And can also download and configure the [AWS Cli tools](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html), which also creates the `.aws` directory under the current user directory.
+
 ```sql
 CREATE CATALOG hive PROPERTIES (
     "type"="hms",
diff --git a/docs/en/docs/lakehouse/multi-catalog/iceberg.md b/docs/en/docs/lakehouse/multi-catalog/iceberg.md
index 2baa05770ff135..07325114ca1d56 100644
--- a/docs/en/docs/lakehouse/multi-catalog/iceberg.md
+++ b/docs/en/docs/lakehouse/multi-catalog/iceberg.md
@@ -96,6 +96,8 @@ CREATE CATALOG iceberg PROPERTIES (
 
 #### AWS Glue
 
+> When connecting Glue, if it's not on the EC2 environment, need copy the `~/.aws` from the EC2 environment to the current environment. And can also download and configure the [AWS Cli tools](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html), which also creates the `.aws` directory under the current user directory.
+
 ```sql
 CREATE CATALOG glue PROPERTIES (
     "type"="iceberg",
diff --git a/docs/zh-CN/docs/lakehouse/faq.md b/docs/zh-CN/docs/lakehouse/faq.md
index 8651784e2e81b0..7c608eb61ae234 100644
--- a/docs/zh-CN/docs/lakehouse/faq.md
+++ b/docs/zh-CN/docs/lakehouse/faq.md
@@ -27,6 +27,19 @@ under the License.
 
 # 常见问题
 
+## 证书问题
+
+1. 查询时报错 `curl 77: Problem with the SSL CA cert.`。说明当前系统证书过旧，需要更新本地证书。
+   - 可以从 `https://curl.haxx.se/docs/caextract.html` 下载最新的CA证书。
+   - 将下载后的cacert-xxx.pem放到`/etc/ssl/certs/`目录，例如：`sudo cp cacert-xxx.pem  /etc/ssl/certs/ca-certificates.crt`。
+
+2. 查询时报错：`ERROR 1105 (HY000): errCode = 2, detailMessage = (x.x.x.x)[CANCELLED][INTERNAL_ERROR]error setting certificate verify locations:  CAfile: /etc/ssl/certs/ca-certificates.crt CApath: none`.
+
+```
+yum install -y ca-certificates
+ln -s /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt /etc/ssl/certs/ca-certificates.crt
+```
+
 ## Kerberos
 
 1. 连接 Kerberos 认证的 Hive Metastore 报错：`GSS initiate failed`
@@ -58,6 +71,12 @@ under the License.
     - 用到的principal必须在klist中存在，使用`klist -kt your.keytab`检查。
     - 检查catalog配置是否正确，比如漏配`yarn.resourcemanager.principal`。
     - 若上述检查没问题，则当前系统yum或者其他包管理软件安装的JDK版本存在不支持的加密算法，建议自行安装JDK并设置`JAVA_HOME`环境变量。
+    - Kerberos默认使用AES-256来进行加密。如果使用Oracle JDK，则必须安装JCE。如果是OpenJDK，OpenJDK的某些发行版会自动提供无限强度的JCE，因此不需要安装JCE。
+    - JCE与JDK版本是对应的，需要根据JDK的版本来选择JCE版本，下载JCE的zip包并解压到`$JAVA_HOME/jre/lib/security`目录下：
+      - JDK6：[JCE6](http://www.oracle.com/technetwork/java/javase/downloads/jce-6-download-429243.html)
+      - JDK7：[JCE7](http://www.oracle.com/technetwork/java/embedded/embedded-se/downloads/jce-7-download-432124.html)
+      - JDK8：[JCE8](http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html)
+
 
 5. 使用 KMS 访问 HDFS 时报错：`java.security.InvalidKeyException: Illegal key size`
 
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
index 16f4565522a300..e75977c25f4766 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
@@ -168,6 +168,8 @@ CREATE CATALOG hive PROPERTIES (
 
 ### Hive With Glue
 
+> 连接Glue时，如果是在非EC2环境，需要将EC2环境里的 `~/.aws` 目录拷贝到当前环境里。也可以下载[AWS Cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)工具进行配置，这种方式也会在当前用户目录下创建`.aws`目录。
+
 ```sql
 CREATE CATALOG hive PROPERTIES (
     "type"="hms",
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
index 3e6a4826d02049..9bf61f6d9f7646 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
@@ -96,6 +96,8 @@ CREATE CATALOG iceberg PROPERTIES (
 
 #### AWS Glue
 
+> 连接Glue时，如果是在非EC2环境，需要将EC2环境里的 `~/.aws` 目录拷贝到当前环境里。也可以下载[AWS Cli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html)工具进行配置，这种方式也会在当前用户目录下创建`.aws`目录。
+
 ```sql
 CREATE CATALOG glue PROPERTIES (
     "type"="iceberg",

From 17f3ca7349d05e10c851d35838836a2ac2235da4 Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Mon, 25 Dec 2023 23:06:45 +0800
Subject: [PATCH 020/299] [fix](planner)should save original select list item
 before analyze (#28187)

* [fix](planner)should save original select list item before analyze

* fix test case

* fix failed case
---
 .../org/apache/doris/catalog/ScalarType.java  |  2 +-
 .../org/apache/doris/analysis/SelectStmt.java | 30 ++++++++++++++-----
 .../data/ddl_p0/test_create_view.out          |  2 +-
 regression-test/suites/view_p0/view_p0.groovy | 28 +++++++++++++++++
 4 files changed, 53 insertions(+), 9 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
index a4972d9ced7002..57876d08d721af 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ScalarType.java
@@ -616,7 +616,7 @@ public String toSql(int depth) {
         StringBuilder stringBuilder = new StringBuilder();
         switch (type) {
             case CHAR:
-                if (isWildcardVarchar()) {
+                if (isWildcardChar()) {
                     stringBuilder.append("CHARACTER");
                 } else if (Strings.isNullOrEmpty(lenStr)) {
                     stringBuilder.append("CHAR").append("(").append(len).append(")");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index dc3c2f52d812ca..d316a07f093069 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -561,8 +561,14 @@ public void analyze(Analyzer analyzer) throws UserException {
             // remove excepted columns
             resultExprs.removeIf(expr -> exceptCols.contains(expr.toColumnLabel()));
             colLabels.removeIf(exceptCols::contains);
+            if (needToSql) {
+                originalExpr = Expr.cloneList(resultExprs);
+            }
 
         } else {
+            if (needToSql) {
+                originalExpr = new ArrayList<>();
+            }
             List<SelectListItem> items = selectList.getItems();
             for (int i = 0; i < items.size(); i++) {
                 SelectListItem item = items.get(i);
@@ -574,6 +580,11 @@ public void analyze(Analyzer analyzer) throws UserException {
                         expandStar(analyzer, tblName);
                     }
                 } else {
+                    // save originalExpr before being analyzed
+                    // because analyze may change the expr by adding cast or some other stuff
+                    if (needToSql) {
+                        originalExpr.add(item.getExpr().clone());
+                    }
                     // Analyze the resultExpr before generating a label to ensure enforcement
                     // of expr child and depth limits (toColumn() label may call toSql()).
                     item.getExpr().analyze(analyzer);
@@ -647,10 +658,6 @@ public void analyze(Analyzer analyzer) throws UserException {
                 subColPath.add(expr.toSubColumnLabel());
             }
         }
-        // analyze valueList if exists
-        if (needToSql) {
-            originalExpr = Expr.cloneList(resultExprs);
-        }
 
         // analyze selectListExprs
         Expr.analyze(resultExprs, analyzer);
@@ -1252,6 +1259,9 @@ private void expandStar(TableName tblName, TupleDescriptor desc) throws Analysis
             slot.setTable(desc.getTable());
             slot.setTupleId(desc.getId());
             resultExprs.add(rewriteQueryExprByMvColumnExpr(slot, analyzer));
+            if (needToSql) {
+                originalExpr.add(slot);
+            }
             colLabels.add(col.getName());
             // empty sub lables
             subColPath.add(Lists.newArrayList());
@@ -2601,6 +2611,9 @@ public void substituteSelectList(Analyzer analyzer, List<String> newColLabels)
             tblRef.analyze(analyzer);
             leftTblRef = tblRef;
         }
+        if (needToSql) {
+            originalExpr = new ArrayList<>();
+        }
         // populate selectListExprs, aliasSMap, and colNames
         for (SelectListItem item : selectList.getItems()) {
             if (item.isStar()) {
@@ -2611,6 +2624,11 @@ public void substituteSelectList(Analyzer analyzer, List<String> newColLabels)
                     expandStar(analyzer, tblName);
                 }
             } else {
+                if (needToSql) {
+                    // save originalExpr before being analyzed
+                    // because analyze may change the expr by adding cast or some other stuff
+                    originalExpr.add(item.getExpr().clone());
+                }
                 // to make sure the sortinfo's AnalyticExpr and resultExprs's AnalyticExpr analytic once
                 if (item.getExpr() instanceof AnalyticExpr) {
                     item.getExpr().analyze(analyzer);
@@ -2624,9 +2642,7 @@ public void substituteSelectList(Analyzer analyzer, List<String> newColLabels)
                 resultExprs.add(rewriteQueryExprByMvColumnExpr(item.getExpr(), analyzer));
             }
         }
-        if (needToSql) {
-            originalExpr = Expr.cloneList(resultExprs);
-        }
+
         // substitute group by
         if (groupByClause != null) {
             boolean aliasFirst = false;
diff --git a/regression-test/data/ddl_p0/test_create_view.out b/regression-test/data/ddl_p0/test_create_view.out
index a84d667d0acd14..239fafc025ef3a 100644
--- a/regression-test/data/ddl_p0/test_create_view.out
+++ b/regression-test/data/ddl_p0/test_create_view.out
@@ -25,5 +25,5 @@
 3	[-1, 20, 0]	[0, 1, 0]
 
 -- !test_view_6 --
-v1	CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT `error_code` AS `error_code`, 1 AS `__literal_1`, 'string' AS `__literal_2`, now() AS `__now_3`, dayofyear(`op_time`) AS `__dayofyear_4`, CAST(`source` AS BIGINT) AS `__cast_expr_5`, min(`timestamp`) OVER (ORDER BY `op_time` DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) AS `__min_6`, 1 > 2 AS `__binary_predicate_7`, (2 + 3) AS `__arithmetic_expr_8`, 1 IN (1, 2, 3, 4) AS `__in_predicate_9`, `remark` LIKE '%like' AS `__like_predicate_10`, CASE WHEN `remark` = 's' THEN 1 ELSE 2 END AS `__case_expr_11`, (CAST(TRUE AS BIGINT) | CAST(FALSE AS BIGINT)) AS `__arithmetic_expr_12` FROM `regression_test_ddl_p0`.`view_column_name_test`;
+v1	CREATE VIEW `v1` COMMENT 'VIEW' AS SELECT `error_code` AS `error_code`, 1 AS `__literal_1`, 'string' AS `__literal_2`, now() AS `__now_3`, dayofyear(`op_time`) AS `__dayofyear_4`, CAST(`source` AS BIGINT) AS `__cast_expr_5`, min(`timestamp`) OVER (ORDER BY `op_time` DESC NULLS LAST ROWS BETWEEN UNBOUNDED PRECEDING AND 1 FOLLOWING) AS `__min_6`, 1 > 2 AS `__binary_predicate_7`, (2 + 3) AS `__arithmetic_expr_8`, 1 IN (1, 2, 3, 4) AS `__in_predicate_9`, `remark` LIKE '%like' AS `__like_predicate_10`, CASE WHEN `remark` = 's' THEN 1 ELSE 2 END AS `__case_expr_11`, (TRUE | FALSE) AS `__arithmetic_expr_12` FROM `regression_test_ddl_p0`.`view_column_name_test`;
 
diff --git a/regression-test/suites/view_p0/view_p0.groovy b/regression-test/suites/view_p0/view_p0.groovy
index a1df7e07cf4d20..82c9e3987ce03e 100644
--- a/regression-test/suites/view_p0/view_p0.groovy
+++ b/regression-test/suites/view_p0/view_p0.groovy
@@ -136,4 +136,32 @@ suite("view_p0") {
     sql """CREATE VIEW IF NOT EXISTS `test_view_abc`(`a`) AS WITH T1 AS (SELECT 1 AS 'a'), T2 AS (SELECT 2 AS 'a') SELECT T1.a FROM T1 UNION ALL SELECT T2.a FROM T2;"""
 
     sql "drop view if exists test_view_abc;" 
+
+    sql """DROP TABLE IF EXISTS test_view_table2"""
+    
+    sql """ 
+        CREATE TABLE test_view_table2 (
+            c_date varchar(50)
+        ) 
+        ENGINE=OLAP
+        UNIQUE KEY(`c_date`)
+        distributed by hash(c_date) properties('replication_num'='1');
+    """
+
+    sql """ drop view if exists test_view_table2_view;"""
+    sql """CREATE VIEW `test_view_table2_view` 
+            AS
+            SELECT 
+                date_format(c_date,'%Y-%m-%d') AS `CREATE_DATE`
+            FROM 
+                test_view_table2
+            GROUP BY  
+                date_format(c_date, '%Y-%m-%d');
+    """
+    sql """set enable_nereids_planner=false;"""
+    sql """select * from test_view_table2_view;"""
+    sql """set enable_nereids_planner=true;"""
+    sql """select * from test_view_table2_view;"""
+    sql """ drop view if exists test_view_table2_view;"""
+    sql """DROP TABLE IF EXISTS test_view_table2"""
 }

From cefae3dc90a4d59bba7b4ca94219e98e6062402a Mon Sep 17 00:00:00 2001
From: plat1ko <platonekosama@gmail.com>
Date: Tue, 26 Dec 2023 00:29:03 +0800
Subject: [PATCH 021/299] [bug](storage) Fix gc rowset bug (#28979)

---
 be/src/olap/data_dir.cpp                | 76 ++++++++++++++++---------
 be/test/agent/task_worker_pool_test.cpp |  2 +
 2 files changed, 50 insertions(+), 28 deletions(-)

diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 8aa07791ebc837..80a0f7131fdad1 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -681,14 +681,17 @@ void DataDir::_perform_path_gc_by_tablet(std::vector<std::string>& tablet_paths)
 }
 
 void DataDir::_perform_path_gc_by_rowset(const std::vector<std::string>& tablet_paths) {
-    if (_stop_bg_worker) return;
-    if (tablet_paths.empty()) {
+    if (_stop_bg_worker || tablet_paths.empty()) {
         return;
     }
+
     LOG(INFO) << "start to path gc by rowset";
     int counter = 0;
     for (const auto& path : tablet_paths) {
-        if (_stop_bg_worker) break;
+        if (_stop_bg_worker) {
+            break;
+        }
+
         ++counter;
         if (config::path_gc_check_step > 0 && counter % config::path_gc_check_step == 0) {
             std::this_thread::sleep_for(
@@ -696,31 +699,50 @@ void DataDir::_perform_path_gc_by_rowset(const std::vector<std::string>& tablet_
         }
         TTabletId tablet_id = -1;
         TSchemaHash schema_hash = -1;
-        bool is_valid = _tablet_manager->get_tablet_id_and_schema_hash_from_path(path, &tablet_id,
-                                                                                 &schema_hash);
+        bool is_valid = doris::TabletManager::get_tablet_id_and_schema_hash_from_path(
+                path, &tablet_id, &schema_hash);
         if (!is_valid || tablet_id < 1 || schema_hash < 1) [[unlikely]] {
             LOG(WARNING) << "unknown path:" << path;
             continue;
         }
+
+        auto tablet = _tablet_manager->get_tablet(tablet_id);
+        if (!tablet) {
+            // Could not found the tablet, maybe it's a dropped tablet, will be reclaimed
+            // in the next time `_perform_path_gc_by_tablet`
+            continue;
+        }
+
         bool exists;
         std::vector<io::FileInfo> files;
-        if (_stop_bg_worker) break;
         auto st = io::global_local_filesystem()->list(path, true, &files, &exists);
         if (!st.ok()) [[unlikely]] {
             LOG(WARNING) << "fail to list tablet path " << path << " : " << st;
             continue;
         }
-        RowsetIdUnorderedSet useful_rowsets;
-        auto tablet = _tablet_manager->get_tablet(tablet_id);
-        if (!tablet) {
-            // Could not found the tablet, maybe it's a dropped tablet, will be reclaimed
-            // in the next time `_perform_path_gc_by_tablet`
-            continue;
+
+        // Rowset files excluding pending rowsets
+        std::vector<std::pair<RowsetId, std::string /* filename */>> rowsets_not_pending;
+        for (auto&& file : files) {
+            auto rowset_id = extract_rowset_id(file.file_name);
+            if (rowset_id.hi == 0) {
+                continue; // Not a rowset
+            }
+
+            if (StorageEngine::instance()->pending_local_rowsets().contains(rowset_id)) {
+                continue; // Pending rowset file
+            }
+
+            rowsets_not_pending.emplace_back(rowset_id, std::move(file.file_name));
         }
+
+        RowsetIdUnorderedSet rowsets_in_version_map;
         tablet->traverse_rowsets(
-                [&useful_rowsets](auto& rs) { useful_rowsets.insert(rs->rowset_id()); }, true);
-        // rowset_id -> is_garbage
-        std::unordered_map<RowsetId, bool, HashOfRowsetId> checked_rowsets;
+                [&rowsets_in_version_map](auto& rs) {
+                    rowsets_in_version_map.insert(rs->rowset_id());
+                },
+                true);
+
         auto reclaim_rowset_file = [](const std::string& path) {
             auto st = io::global_local_filesystem()->delete_file(path);
             if (!st.ok()) [[unlikely]] {
@@ -729,28 +751,26 @@ void DataDir::_perform_path_gc_by_rowset(const std::vector<std::string>& tablet_
             }
             LOG(INFO) << "delete garbage path: " << path; // Audit log
         };
+
         auto should_reclaim = [&, this](const RowsetId& rowset_id) {
-            return !useful_rowsets.contains(rowset_id) &&
-                   !StorageEngine::instance()->pending_local_rowsets().contains(rowset_id) &&
+            return !rowsets_in_version_map.contains(rowset_id) &&
                    !StorageEngine::instance()->check_rowset_id_in_unused_rowsets(rowset_id) &&
                    RowsetMetaManager::exists(get_meta(), tablet->tablet_uid(), rowset_id)
                            .is<META_KEY_NOT_FOUND>();
         };
-        for (auto&& file : files) {
-            auto rowset_id = extract_rowset_id(file.file_name);
-            if (rowset_id.hi == 0) {
-                continue; // Not a rowset
-            }
-            auto find_it = checked_rowsets.find(rowset_id);
-            if (find_it != checked_rowsets.end()) {
-                if (find_it->second) {
-                    reclaim_rowset_file(path + '/' + file.file_name);
+
+        // rowset_id -> is_garbage
+        std::unordered_map<RowsetId, bool, HashOfRowsetId> checked_rowsets;
+        for (auto&& [rowset_id, filename] : rowsets_not_pending) {
+            if (auto it = checked_rowsets.find(rowset_id); it != checked_rowsets.end()) {
+                if (it->second) { // Is checked garbage rowset
+                    reclaim_rowset_file(path + '/' + filename);
                 }
                 continue;
             }
-            // Check rowset useful
+
             if (should_reclaim(rowset_id)) {
-                reclaim_rowset_file(path + '/' + file.file_name);
+                reclaim_rowset_file(path + '/' + filename);
                 checked_rowsets.emplace(rowset_id, true);
             } else {
                 checked_rowsets.emplace(rowset_id, false);
diff --git a/be/test/agent/task_worker_pool_test.cpp b/be/test/agent/task_worker_pool_test.cpp
index 3ea591c805840f..8fd78cc3fd225a 100644
--- a/be/test/agent/task_worker_pool_test.cpp
+++ b/be/test/agent/task_worker_pool_test.cpp
@@ -44,6 +44,7 @@ TEST(TaskWorkerPoolTest, TaskWorkerPool) {
     workers.submit_task(task);
     workers.submit_task(task); // Pending and ignored when stop
 
+    std::this_thread::sleep_for(200ms);
     workers.stop();
 
     workers.submit_task(task); // Ignore
@@ -92,6 +93,7 @@ TEST(TaskWorkerPoolTest, PriorTaskWorkerPool) {
     workers.submit_task(task);
     workers.submit_task(task);
     workers.submit_task(task); // Pending and ignored when stop
+    std::this_thread::sleep_for(100ms);
     workers.stop();
 
     EXPECT_EQ(normal_count.load(), 2);

From 2aea47c0a9383c1c981cbf2cb7dd72bce2bfb3ec Mon Sep 17 00:00:00 2001
From: zhangdong <493738387@qq.com>
Date: Tue, 26 Dec 2023 00:29:28 +0800
Subject: [PATCH 022/299] [fix](mtmv)add log for resolve pending task (#28999)

* add lock for resolve pending task

* add lock for resolve pending task
---
 .../org/apache/doris/job/extensions/mtmv/MTMVTask.java   | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index ab9b6f9fa94211..7c11f0549ad859 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -135,6 +135,7 @@ public MTMVTask(long dbId, long mtmvId, MTMVTaskContext taskContext) {
 
     @Override
     public void run() throws JobException {
+        LOG.info("mtmv task run, taskId: {}", super.getTaskId());
         try {
             ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv);
             // Every time a task is run, the relation is regenerated because baseTables and baseViews may change,
@@ -185,18 +186,21 @@ private void exec(ConnectContext ctx, Set<Long> refreshPartitionIds, Map<OlapTab
 
     @Override
     public synchronized void onFail() throws JobException {
+        LOG.info("mtmv task onFail, taskId: {}", super.getTaskId());
         super.onFail();
         after();
     }
 
     @Override
     public synchronized void onSuccess() throws JobException {
+        LOG.info("mtmv task onSuccess, taskId: {}", super.getTaskId());
         super.onSuccess();
         after();
     }
 
     @Override
     public synchronized void cancel() throws JobException {
+        LOG.info("mtmv task cancel, taskId: {}", super.getTaskId());
         super.cancel();
         if (executor != null) {
             executor.cancel();
@@ -206,6 +210,7 @@ public synchronized void cancel() throws JobException {
 
     @Override
     public void before() throws JobException {
+        LOG.info("mtmv task before, taskId: {}", super.getTaskId());
         super.before();
         try {
             Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId);
@@ -222,12 +227,16 @@ public void before() throws JobException {
 
     @Override
     public void runTask() throws JobException {
+        LOG.info("mtmv task runTask, taskId: {}", super.getTaskId());
         MTMVJob job = (MTMVJob) getJobOrJobException();
         try {
+            LOG.info("mtmv task get writeLock start, taskId: {}", super.getTaskId());
             job.writeLock();
+            LOG.info("mtmv task get writeLock end, taskId: {}", super.getTaskId());
             super.runTask();
         } finally {
             job.writeUnlock();
+            LOG.info("mtmv task release writeLock, taskId: {}", super.getTaskId());
         }
     }
 

From 7434de9ed817cc3c3464f782f75cae254b9f6b3a Mon Sep 17 00:00:00 2001
From: seawinde <149132972+seawinde@users.noreply.github.com>
Date: Tue, 26 Dec 2023 00:29:42 +0800
Subject: [PATCH 023/299] [improvement](nereids) Get partition related table
 disable nullable field and complete agg matched pattern mv rules. (#28973)

* [improvement] (nereids) Get partition related table disable nullable field and modify regression test, complete agg mv rules.

* make filed not null to create partition mv
---
 .../apache/doris/nereids/rules/RuleSet.java   |  6 +++
 .../MaterializedViewFilterAggregateRule.java  | 45 +++++++++++++++++
 ...ializedViewFilterProjectAggregateRule.java | 47 ++++++++++++++++++
 ...ializedViewProjectFilterAggregateRule.java | 47 ++++++++++++++++++
 .../exploration/mv/MaterializedViewUtils.java |  1 +
 .../mv/MaterializedViewUtilsTest.java         | 48 +++++++++++++++++++
 .../aggregate_with_roll_up.out                |  2 +
 .../aggregate_without_roll_up.out             | 14 ++++++
 .../mv/join/inner/inner_join.out              | 20 ++++++++
 .../mv/join/left_outer/outer_join.out         | 16 +++++++
 .../test_partition_refresh_mtmv.groovy        | 12 ++---
 .../aggregate_with_roll_up.groovy             |  2 +-
 .../aggregate_without_roll_up.groovy          |  8 ++--
 .../mv/join/inner/inner_join.groovy           | 10 ++--
 .../mv/join/left_outer/outer_join.groovy      |  2 +-
 15 files changed, 263 insertions(+), 17 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index a9ce2cce17412d..b7d07aa134df74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -40,10 +40,13 @@
 import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose;
 import org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewAggregateRule;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterAggregateRule;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterJoinRule;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectAggregateRule;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterJoinRule;
 import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectJoinRule;
 import org.apache.doris.nereids.rules.implementation.AggregateStrategies;
@@ -233,6 +236,9 @@ public class RuleSet {
             .add(MaterializedViewProjectFilterJoinRule.INSTANCE)
             .add(MaterializedViewAggregateRule.INSTANCE)
             .add(MaterializedViewProjectAggregateRule.INSTANCE)
+            .add(MaterializedViewFilterAggregateRule.INSTANCE)
+            .add(MaterializedViewProjectFilterAggregateRule.INSTANCE)
+            .add(MaterializedViewFilterProjectAggregateRule.INSTANCE)
             .build();
 
     public List<Rule> getDPHypReorderRules() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java
new file mode 100644
index 00000000000000..40c8b6d681b3ea
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterAggregateRule.java
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * MaterializedViewFilterAggregateRule
+ */
+public class MaterializedViewFilterAggregateRule extends AbstractMaterializedViewAggregateRule {
+
+    public static final MaterializedViewFilterAggregateRule INSTANCE = new MaterializedViewFilterAggregateRule();
+
+    @Override
+    public List<Rule> buildRules() {
+        return ImmutableList.of(
+                logicalFilter(logicalAggregate(any())).thenApplyMultiNoThrow(ctx -> {
+                    LogicalFilter<LogicalAggregate<Plan>> root = ctx.root;
+                    return rewrite(root, ctx.cascadesContext);
+                }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
new file mode 100644
index 00000000000000..40655e97394463
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectAggregateRule.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * MaterializedViewFilterProjectAggregateRule
+ */
+public class MaterializedViewFilterProjectAggregateRule extends AbstractMaterializedViewAggregateRule {
+
+    public static final MaterializedViewFilterProjectAggregateRule
+            INSTANCE = new MaterializedViewFilterProjectAggregateRule();
+
+    @Override
+    public List<Rule> buildRules() {
+        return ImmutableList.of(
+                logicalFilter(logicalProject(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx -> {
+                    LogicalFilter<LogicalProject<LogicalAggregate<Plan>>> root = ctx.root;
+                    return rewrite(root, ctx.cascadesContext);
+                }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java
new file mode 100644
index 00000000000000..b841862d5b2505
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterAggregateRule.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * MaterializedViewProjectFilterAggregateRule
+ */
+public class MaterializedViewProjectFilterAggregateRule extends AbstractMaterializedViewAggregateRule {
+
+    public static final MaterializedViewProjectFilterAggregateRule
+            INSTANCE = new MaterializedViewProjectFilterAggregateRule();
+
+    @Override
+    public List<Rule> buildRules() {
+        return ImmutableList.of(
+                logicalProject(logicalFilter(logicalAggregate(any()))).thenApplyMultiNoThrow(ctx -> {
+                    LogicalProject<LogicalFilter<LogicalAggregate<Plan>>> root = ctx.root;
+                    return rewrite(root, ctx.cascadesContext);
+                }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_AGGREGATE));
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
index ca54c3768318cc..3a8da2c752bc49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java
@@ -222,6 +222,7 @@ public Void visitLogicalRelation(LogicalRelation relation, IncrementCheckerConte
             if (partitionColumnSet.contains(mvReferenceColumn)) {
                 context.setRelatedTable(table);
                 context.setRelatedTableColumn(mvReferenceColumn);
+                context.setPctPossible(!mvReferenceColumn.isAllowNull());
             }
             return visit(relation, context);
         }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
index c2af1dbdf4aa17..3b78d88c39b05b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java
@@ -93,6 +93,31 @@ protected void runBeforeAll() throws Exception {
                 + "PROPERTIES (\n"
                 + "  \"replication_num\" = \"1\"\n"
                 + ")");
+
+        createTable("CREATE TABLE IF NOT EXISTS lineitem_null (\n"
+                + "  L_ORDERKEY    INTEGER NOT NULL,\n"
+                + "  L_PARTKEY     INTEGER NOT NULL,\n"
+                + "  L_SUPPKEY     INTEGER NOT NULL,\n"
+                + "  L_LINENUMBER  INTEGER NOT NULL,\n"
+                + "  L_QUANTITY    DECIMALV3(15,2) NOT NULL,\n"
+                + "  L_EXTENDEDPRICE  DECIMALV3(15,2) NOT NULL,\n"
+                + "  L_DISCOUNT    DECIMALV3(15,2) NOT NULL,\n"
+                + "  L_TAX         DECIMALV3(15,2) NOT NULL,\n"
+                + "  L_RETURNFLAG  CHAR(1) NOT NULL,\n"
+                + "  L_LINESTATUS  CHAR(1) NOT NULL,\n"
+                + "  L_SHIPDATE    DATE NULL,\n"
+                + "  L_COMMITDATE  DATE NULL,\n"
+                + "  L_RECEIPTDATE DATE NULL,\n"
+                + "  L_SHIPINSTRUCT CHAR(25) NOT NULL,\n"
+                + "  L_SHIPMODE     CHAR(10) NOT NULL,\n"
+                + "  L_COMMENT      VARCHAR(44) NOT NULL\n"
+                + ")\n"
+                + "DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)\n"
+                + "PARTITION BY RANGE(L_SHIPDATE) (PARTITION `day_1` VALUES LESS THAN ('2017-02-01'))\n"
+                + "DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3\n"
+                + "PROPERTIES (\n"
+                + "  \"replication_num\" = \"1\"\n"
+                + ")");
     }
 
     @Test
@@ -121,6 +146,29 @@ public void getRelatedTableInfoTestWithoutGroupTest() {
                         });
     }
 
+    @Test
+    public void getRelatedTableInfoTestWithoutGroupNullTest() {
+        PlanChecker.from(connectContext)
+                .checkExplain("SELECT (o.c1_abs + ps.c2_abs) as add_alias, l.L_SHIPDATE, l.L_ORDERKEY, o.O_ORDERDATE, "
+                                + "ps.PS_AVAILQTY "
+                                + "FROM "
+                                + "lineitem_null as l "
+                                + "LEFT JOIN "
+                                + "(SELECT abs(O_TOTALPRICE + 10) as c1_abs, O_CUSTKEY, O_ORDERDATE, O_ORDERKEY "
+                                + "FROM orders) as o "
+                                + "ON l.L_ORDERKEY = o.O_ORDERKEY "
+                                + "JOIN "
+                                + "(SELECT abs(sqrt(PS_SUPPLYCOST)) as c2_abs, PS_AVAILQTY, PS_PARTKEY, PS_SUPPKEY "
+                                + "FROM partsupp) as ps "
+                                + "ON l.L_PARTKEY = ps.PS_PARTKEY and l.L_SUPPKEY = ps.PS_SUPPKEY",
+                        nereidsPlanner -> {
+                            Plan rewrittenPlan = nereidsPlanner.getRewrittenPlan();
+                            Optional<RelatedTableInfo> relatedTableInfo =
+                                    MaterializedViewUtils.getRelatedTableInfo("l_shipdate", rewrittenPlan);
+                            Assertions.assertFalse(relatedTableInfo.isPresent());
+                        });
+    }
+
     @Test
     public void getRelatedTableInfoTestWithSubqueryTest() {
         PlanChecker.from(connectContext)
diff --git a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
index e6facb7b89026f..fb223bc661b9b1 100644
--- a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
+++ b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
@@ -38,8 +38,10 @@
 3	3	2023-12-11	43.20	43.20	43.20	1	0
 
 -- !query18_0_before --
+3	2023-12-11	43.20	43.20	43.20	1	0
 
 -- !query18_0_after --
+3	2023-12-11	43.20	43.20	43.20	1	0
 
 -- !query19_0_before --
 2	3	2023-12-08	20.00
diff --git a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
index cb924d69bbbfb7..8bf8d7d4f9fa76 100644
--- a/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
+++ b/regression-test/data/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.out
@@ -1,7 +1,13 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !query1_0_before --
+1	yy	0	0	77.50	33.50	9.50	5
+2	mi	0	0	57.40	56.20	1.20	2
+2	mm	0	0	43.20	43.20	43.20	1
 
 -- !query1_0_after --
+1	yy	0	0	77.50	33.50	9.50	5
+2	mi	0	0	57.40	56.20	1.20	2
+2	mm	0	0	43.20	43.20	43.20	1
 
 -- !query1_1_before --
 2023-12-08	1	yy	0	0	0	0	0	0	0	0	0	0	0	0	0	0	0
@@ -114,16 +120,24 @@
 5	2	mi	0	0	0	0	0	0	0	0	0	0	0	0	0
 
 -- !query18_0_before --
+3	2023-12-11	43.20	43.20	43.20	1	0
 
 -- !query18_0_after --
+3	2023-12-11	43.20	43.20	43.20	1	0
 
 -- !query18_1_before --
+4	43.20
 
 -- !query18_1_after --
+4	43.20
 
 -- !query18_2_before --
+4	43.20
+6	57.40
 
 -- !query18_2_after --
+4	43.20
+6	57.40
 
 -- !query19_0_before --
 2	3	2023-12-08	20.00	10.50	9.50	2
diff --git a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
index df3e20dc64ce0b..abf0eb188e6b25 100644
--- a/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
+++ b/regression-test/data/nereids_rules_p0/mv/join/inner/inner_join.out
@@ -100,8 +100,24 @@
 6
 
 -- !query2_0_before --
+4
+4
+4
+4
+4
+4
+6
+6
 
 -- !query2_0_after --
+4
+4
+4
+4
+4
+4
+6
+6
 
 -- !query2_1_before --
 4
@@ -268,8 +284,12 @@
 2	3	2023-12-08
 
 -- !query7_0_before --
+2	3	2023-12-08
+2	3	2023-12-08
 
 -- !query7_0_after --
+2	3	2023-12-08
+2	3	2023-12-08
 
 -- !query10_0_before --
 
diff --git a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
index 94143e800ada97..e8665ccb9484fc 100644
--- a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
+++ b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
@@ -88,8 +88,24 @@
 2	2
 
 -- !query2_0_before --
+4
+4
+4
+4
+4
+4
+6
+6
 
 -- !query2_0_after --
+4
+4
+4
+4
+4
+4
+6
+6
 
 -- !query2_1_before --
 4
diff --git a/regression-test/suites/mtmv_p0/test_partition_refresh_mtmv.groovy b/regression-test/suites/mtmv_p0/test_partition_refresh_mtmv.groovy
index 8688cb00b6857c..f2a55d43472d38 100644
--- a/regression-test/suites/mtmv_p0/test_partition_refresh_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_partition_refresh_mtmv.groovy
@@ -30,7 +30,7 @@ suite("test_partition_refresh_mtmv") {
         CREATE TABLE `${tableNameNum}` (
           `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
           `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
-          `num` SMALLINT NULL COMMENT '\"数量\"'
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
         ) ENGINE=OLAP
         DUPLICATE KEY(`user_id`, `date`, `num`)
         COMMENT 'OLAP'
@@ -64,7 +64,7 @@ suite("test_partition_refresh_mtmv") {
         CREATE TABLE `${tableNameNum}` (
           `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
           `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
-          `num` SMALLINT NULL COMMENT '\"数量\"'
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
         ) ENGINE=OLAP
         DUPLICATE KEY(`user_id`, `date`, `num`)
         COMMENT 'OLAP'
@@ -98,7 +98,7 @@ suite("test_partition_refresh_mtmv") {
         CREATE TABLE `${tableNameNum}` (
           `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
           `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
-          `num` SMALLINT NULL COMMENT '\"数量\"'
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
         ) ENGINE=OLAP
         DUPLICATE KEY(`user_id`, `date`, `num`)
         COMMENT 'OLAP'
@@ -144,7 +144,7 @@ suite("test_partition_refresh_mtmv") {
         CREATE TABLE `${tableNameNum}` (
           `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
           `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
-          `num` SMALLINT NULL COMMENT '\"数量\"'
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
         ) ENGINE=OLAP
         DUPLICATE KEY(`user_id`, `date`, `num`)
         COMMENT 'OLAP'
@@ -236,7 +236,7 @@ suite("test_partition_refresh_mtmv") {
         CREATE TABLE `${tableNameNum}` (
           `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
           `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
-          `num` SMALLINT NULL COMMENT '\"数量\"'
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
         ) ENGINE=OLAP
         DUPLICATE KEY(`user_id`, `date`, `num`)
         COMMENT 'OLAP'
@@ -337,7 +337,7 @@ suite("test_partition_refresh_mtmv") {
         CREATE TABLE `${tableNameNum}` (
           `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
           `date` DATE NOT NULL COMMENT '\"数据灌入日期时间\"',
-          `num` SMALLINT NULL COMMENT '\"数量\"'
+          `num` SMALLINT NOT NULL COMMENT '\"数量\"'
         ) ENGINE=OLAP
         DUPLICATE KEY(`user_id`, `date`, `num`)
         COMMENT 'OLAP'
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
index b56922d5db6470..d6b5c37e59c095 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
@@ -402,7 +402,7 @@ suite("aggregate_with_roll_up") {
             "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) " +
             "from lineitem t1 " +
             "left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate " +
-            "where o_orderdate = '2023-12-11' and l_partkey = 2 " +
+            "where o_orderdate = '2023-12-11' and l_partkey = 3 " +
             "group by " +
             "l_shipdate, " +
             "l_suppkey"
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 72e80eadbf93f0..5d40aa03138a6d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -194,7 +194,7 @@ suite("aggregate_without_roll_up") {
             "min(o_totalprice), " +
             "count(*) " +
             "from orders " +
-            "where o_shippriority in (9.5, 10.5)" +
+            "where o_shippriority in (1, 2)" +
             "group by " +
             "o_shippriority, " +
             "o_comment "
@@ -645,7 +645,7 @@ suite("aggregate_without_roll_up") {
             "count(distinct case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end) " +
             "from lineitem t1 " +
             "left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate " +
-            "where l_shipdate = '2023-12-11' and l_suppkey = 2 " +
+            "where l_shipdate = '2023-12-11' and l_suppkey = 3 " +
             "group by " +
             "l_shipdate, " +
             "l_suppkey"
@@ -662,7 +662,7 @@ suite("aggregate_without_roll_up") {
     def query18_1 = "select l_linenumber, sum(o_totalprice) as sum_alias " +
             "from lineitem " +
             "inner join orders on l_orderkey = o_orderkey " +
-            "where o_custkey = 2 and l_linenumber = 3 " +
+            "where o_custkey = 2 and l_linenumber = 4 " +
             "group by l_linenumber, o_custkey "
     order_qt_query18_1_before "${query18_1}"
     check_rewrite(mv18_1, query18_1, "mv18_1")
@@ -677,7 +677,7 @@ suite("aggregate_without_roll_up") {
     def query18_2 = "select lineitem.l_linenumber, sum(o_totalprice) as sum_alias " +
             "from lineitem " +
             "inner join orders on lineitem.l_orderkey = orders.o_orderkey " +
-            "where o_custkey = 2 and l_suppkey= 4 " +
+            "where o_custkey = 2 and l_suppkey= 3 " +
             "group by lineitem.l_linenumber, orders.o_custkey "
     order_qt_query18_2_before "${query18_2}"
     check_not_match(mv18_2, query18_2, "mv18_2")
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
index 0dcf33a9d87eb2..ccf6a83f2c610d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
@@ -239,7 +239,7 @@ suite("inner_join") {
     def query2_0 = "select lineitem.L_LINENUMBER " +
             "from lineitem " +
             "inner join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " +
-            "where lineitem.L_LINENUMBER > 10"
+            "where lineitem.L_LINENUMBER > 0"
     order_qt_query2_0_before "${query2_0}"
     check_rewrite(mv2_0, query2_0, "mv2_0")
     order_qt_query2_0_after "${query2_0}"
@@ -400,10 +400,10 @@ suite("inner_join") {
             "on lineitem.l_orderkey = o_orderkey and l_shipdate = o_orderdate "
     def query7_0 = "select l_partkey, l_suppkey, l_shipdate " +
             "from (select l_shipdate, l_orderkey, l_partkey, l_suppkey " +
-            "from lineitem where l_partkey in (3, 4)) t1  " +
+            "from lineitem where l_partkey in (2, 3, 4)) t1  " +
             "inner join (select * from orders where o_orderdate = '2023-12-08') t2 " +
             "on t1.l_orderkey = o_orderkey and t1.l_shipdate = o_orderdate " +
-            "where l_partkey = 3"
+            "where l_partkey = 2"
     order_qt_query7_0_before "${query7_0}"
     check_rewrite(mv7_0, query7_0, "mv7_0")
     order_qt_query7_0_after "${query7_0}"
@@ -417,8 +417,8 @@ suite("inner_join") {
     def query10_0 = "select orders.O_CUSTKEY " +
             "from orders " +
             "inner join lineitem on orders.O_ORDERKEY = lineitem.L_ORDERKEY " +
-            "WHERE lineitem.L_LINENUMBER > 10 AND orders.O_CUSTKEY = 5 AND " +
-            "orders.O_SHIPPRIORITY = 1"
+            "WHERE lineitem.L_LINENUMBER > 0 AND orders.O_CUSTKEY = 1 AND " +
+            "orders.O_SHIPPRIORITY = 2"
     order_qt_query10_0_before "${query10_0}"
     check_not_match(mv10_0, query10_0, "mv10_0")
     order_qt_query10_0_after "${query10_0}"
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
index d7ec67189ac871..e8b1da2572a22d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
@@ -224,7 +224,7 @@ suite("outer_join") {
     def query2_0 = "select lineitem.L_LINENUMBER " +
             "from lineitem " +
             "left join orders on lineitem.L_ORDERKEY = orders.O_ORDERKEY " +
-            "where lineitem.L_LINENUMBER > 10"
+            "where lineitem.L_LINENUMBER > 0"
     order_qt_query2_0_before "${query2_0}"
     check_not_match(mv2_0, query2_0, "mv2_0")
     order_qt_query2_0_after "${query2_0}"

From 75a45484b6d7864ee3415b3d2351d326db48bf86 Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Tue, 26 Dec 2023 00:34:58 +0800
Subject: [PATCH 024/299] [chore](config) modify
 `tablet_schema_cache_recycle_interval` from 24h to 1h (#28980)

To prevent from too many tablet schema cache in memory and lead to performance issue when hold lock
to erase item
---
 be/src/common/config.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 8153e5aca5f86b..2f12cea1fa3470 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1100,7 +1100,7 @@ DEFINE_Int32(group_commit_memory_rows_for_max_filter_ratio, "10000");
 DEFINE_Bool(wait_internal_group_commit_finish, "false");
 
 DEFINE_mInt32(scan_thread_nice_value, "0");
-DEFINE_mInt32(tablet_schema_cache_recycle_interval, "86400");
+DEFINE_mInt32(tablet_schema_cache_recycle_interval, "3600");
 
 DEFINE_Bool(exit_on_exception, "false");
 // This config controls whether the s3 file writer would flush cache asynchronously

From f30e50676ec93aa647ef96f78996616686f92c12 Mon Sep 17 00:00:00 2001
From: Ashin Gau <AshinGau@users.noreply.github.com>
Date: Tue, 26 Dec 2023 10:24:12 +0800
Subject: [PATCH 025/299] [opt](scanner) optimize the number of threads of
 scanners (#28640)

1. Remove `doris_max_remote_scanner_thread_pool_thread_num`, use `doris_scanner_thread_pool_thread_num` only.
2. Set the default value `doris_scanner_thread_pool_thread_num` as `std::max(48, CpuInfo::num_cores() * 4)`
---
 be/src/common/config.cpp                             | 12 +++++++++++-
 be/src/common/config.h                               |  6 +++++-
 be/src/io/fs/buffered_reader.h                       |  7 +++----
 be/src/vec/exec/scan/scanner_scheduler.cpp           |  2 ++
 .../java/org/apache/doris/hudi/HudiJniScanner.java   |  4 ++--
 5 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 2f12cea1fa3470..919b83f0c7d706 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -41,6 +41,7 @@
 #include "common/status.h"
 #include "io/fs/file_writer.h"
 #include "io/fs/local_file_system.h"
+#include "util/cpu_info.h"
 
 namespace doris::config {
 
@@ -228,7 +229,14 @@ DEFINE_Bool(doris_enable_scanner_thread_pool_per_disk, "true");
 DEFINE_mInt64(doris_blocking_priority_queue_wait_timeout_ms, "500");
 // number of scanner thread pool size for olap table
 // and the min thread num of remote scanner thread pool
-DEFINE_Int32(doris_scanner_thread_pool_thread_num, "48");
+DEFINE_Int32(doris_scanner_thread_pool_thread_num, "-1");
+DEFINE_Validator(doris_scanner_thread_pool_thread_num, [](const int config) -> bool {
+    if (config == -1) {
+        CpuInfo::init();
+        doris_scanner_thread_pool_thread_num = std::max(48, CpuInfo::num_cores() * 4);
+    }
+    return true;
+});
 DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "-1");
 // number of olap scanner thread pool queue size
 DEFINE_Int32(doris_scanner_thread_pool_queue_size, "102400");
@@ -864,6 +872,8 @@ DEFINE_mInt32(parquet_rowgroup_max_buffer_mb, "128");
 // Max buffer size for parquet chunk column
 DEFINE_mInt32(parquet_column_max_buffer_mb, "8");
 DEFINE_mDouble(max_amplified_read_ratio, "0.8");
+DEFINE_mInt32(merged_oss_min_io_size, "1048576");
+DEFINE_mInt32(merged_hdfs_min_io_size, "8192");
 
 // OrcReader
 DEFINE_mInt32(orc_natural_read_size_mb, "8");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index abdf1c67a2e06f..affe47851fb97a 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -276,7 +276,7 @@ DECLARE_Bool(doris_enable_scanner_thread_pool_per_disk);
 DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms);
 // number of scanner thread pool size for olap table
 // and the min thread num of remote scanner thread pool
-DECLARE_Int32(doris_scanner_thread_pool_thread_num);
+DECLARE_mInt32(doris_scanner_thread_pool_thread_num);
 // max number of remote scanner thread pool size
 // if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10)
 DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num);
@@ -921,6 +921,10 @@ DECLARE_mInt32(parquet_rowgroup_max_buffer_mb);
 DECLARE_mInt32(parquet_column_max_buffer_mb);
 // Merge small IO, the max amplified read ratio
 DECLARE_mDouble(max_amplified_read_ratio);
+// Equivalent min size of each IO that can reach the maximum storage speed limit
+// 1MB for oss, 8KB for hdfs
+DECLARE_mInt32(merged_oss_min_io_size);
+DECLARE_mInt32(merged_hdfs_min_io_size);
 
 // OrcReader
 DECLARE_mInt32(orc_natural_read_size_mb);
diff --git a/be/src/io/fs/buffered_reader.h b/be/src/io/fs/buffered_reader.h
index e78c1c79251146..2f8eb465cdfc54 100644
--- a/be/src/io/fs/buffered_reader.h
+++ b/be/src/io/fs/buffered_reader.h
@@ -131,8 +131,6 @@ class MergeRangeFileReader : public io::FileReader {
     static constexpr size_t READ_SLICE_SIZE = 8 * 1024 * 1024;      // 8MB
     static constexpr size_t BOX_SIZE = 1 * 1024 * 1024;             // 1MB
     static constexpr size_t SMALL_IO = 2 * 1024 * 1024;             // 2MB
-    static constexpr size_t HDFS_MIN_IO_SIZE = 4 * 1024;            // 4KB
-    static constexpr size_t OSS_MIN_IO_SIZE = 512 * 1024;           // 512KB
     static constexpr size_t NUM_BOX = TOTAL_BUFFER_SIZE / BOX_SIZE; // 128
 
     MergeRangeFileReader(RuntimeProfile* profile, io::FileReaderSPtr reader,
@@ -146,8 +144,9 @@ class MergeRangeFileReader : public io::FileReader {
         _is_oss = typeid_cast<io::S3FileReader*>(_reader.get()) != nullptr;
         _max_amplified_ratio = config::max_amplified_read_ratio;
         // Equivalent min size of each IO that can reach the maximum storage speed limit:
-        // 512KB for oss, 4KB for hdfs
-        _equivalent_io_size = _is_oss ? OSS_MIN_IO_SIZE : HDFS_MIN_IO_SIZE;
+        // 1MB for oss, 8KB for hdfs
+        _equivalent_io_size =
+                _is_oss ? config::merged_oss_min_io_size : config::merged_hdfs_min_io_size;
         for (const PrefetchRange& range : _random_access_ranges) {
             _statistics.apply_bytes += range.end_offset - range.start_offset;
         }
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 42285f1d22a854..84f56813c34a9e 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -122,6 +122,8 @@ Status ScannerScheduler::init(ExecEnv* env) {
     _remote_thread_pool_max_size = config::doris_max_remote_scanner_thread_pool_thread_num != -1
                                            ? config::doris_max_remote_scanner_thread_pool_thread_num
                                            : std::max(512, CpuInfo::num_cores() * 10);
+    _remote_thread_pool_max_size =
+            std::max(_remote_thread_pool_max_size, config::doris_scanner_thread_pool_thread_num);
     _remote_scan_thread_pool = std::make_unique<PriorityThreadPool>(
             _remote_thread_pool_max_size, config::doris_remote_scanner_thread_pool_queue_size,
             "RemoteScanThreadPool");
diff --git a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
index daf8d4a21fa87b..17ca650675de3c 100644
--- a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
+++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
@@ -85,8 +85,8 @@ public class HudiJniScanner extends JniScanner {
     private static final ScheduledExecutorService cleanResolverService = Executors.newScheduledThreadPool(1);
 
     static {
-        int numThreads = Math.max(Runtime.getRuntime().availableProcessors() * 2 + 1, 4);
-        if (numThreads > 32) {
+        int numThreads = Math.max(Runtime.getRuntime().availableProcessors() * 2, 4);
+        if (numThreads > 48) {
             numThreads = Runtime.getRuntime().availableProcessors();
         }
         avroReadPool = Executors.newFixedThreadPool(numThreads,

From 92660bb1b273ab562bb8ebc4885686ba055534b9 Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Tue, 26 Dec 2023 10:24:43 +0800
Subject: [PATCH 026/299] [chore](config) modify
 `variant_ratio_of_defaults_as_sparse_column` from 0.95 to 1 (#28984)

since sparse column is not stable at present
---
 be/src/common/config.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 919b83f0c7d706..1d8178ceec2e86 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -969,7 +969,7 @@ DEFINE_mInt64(workload_group_scan_task_wait_timeout_ms, "10000");
 DEFINE_Bool(enable_index_apply_preds_except_leafnode_of_andnode, "true");
 
 DEFINE_mBool(variant_enable_flatten_nested, "false");
-DEFINE_mDouble(variant_ratio_of_defaults_as_sparse_column, "0.95");
+DEFINE_mDouble(variant_ratio_of_defaults_as_sparse_column, "1");
 DEFINE_mInt64(variant_threshold_rows_to_estimate_sparse_column, "1000");
 
 // block file cache

From 065eb9a72b599d5ca0058fa80135ea37240de96f Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Tue, 26 Dec 2023 11:19:04 +0800
Subject: [PATCH 027/299] [feature](nereids)support partition property in
 nereids (#28982)

---
 .../org/apache/doris/nereids/DorisParser.g4   |   2 +-
 fe/fe-core/src/main/cup/sql_parser.cup        |  26 --
 .../doris/analysis/SinglePartitionDesc.java   |  19 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  27 +-
 .../plans/commands/info/ColumnDefinition.java | 260 +++++++++++++++++-
 .../commands/info/FixedRangePartition.java    |  21 +-
 .../plans/commands/info/InPartition.java      |  19 +-
 .../commands/info/LessThanPartition.java      |  31 ++-
 .../commands/info/PartitionDefinition.java    |  88 +++++-
 .../plans/commands/info/StepPartition.java    |  11 +-
 .../rules/exploration/mv/MappingTest.java     |   2 +-
 .../test_agg_materialize.groovy               |   2 +-
 .../datatype_p0/date/test_date_key.groovy     |   2 +-
 .../datatype_p0/date/test_datetime_key.groovy |   2 +-
 .../datetimev1/test_datetimev1_calc.groovy    |   2 +-
 .../datev1/test_datev1_calc.groovy            |   2 +-
 .../decimalv2/test_decimalv2_calc.groovy      |   2 +-
 ...test_decimalv2_calc_with_conversion.groovy |   2 +-
 .../decimalv3/test_decimalv3_overflow.groovy  |   2 +-
 .../decimalv3/test_uniq_tab_decimalv2.groovy  |   2 +-
 ...te_nestedtypes_with_unsupport_types.groovy |   8 +-
 .../ddl/dup_tbl_basic_multi_table_create.sql  |   2 +-
 ...te_or_datetime_computation_negative.groovy |   2 +-
 .../test_default_limit.groovy                 |   4 +-
 ...count_distinct_with_collection_type.groovy |   2 +-
 .../test_array_functions.groovy               |   2 +-
 .../size_funciton/test_size_function.groovy   |   2 +-
 27 files changed, 444 insertions(+), 102 deletions(-)

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index abef9ffa6d7555..a2f90ea21667c8 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -502,7 +502,7 @@ partitionsDef
     ;
     
 partitionDef
-    : (lessThanPartitionDef | fixedPartitionDef | stepPartitionDef | inPartitionDef) properties=propertyClause?
+    : (lessThanPartitionDef | fixedPartitionDef | stepPartitionDef | inPartitionDef) (LEFT_PAREN partitionProperties=propertyItemList RIGHT_PAREN)?
     ;
     
 lessThanPartitionDef
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 26b221c128c6e3..7e8f4dd96b569a 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -1800,19 +1800,6 @@ create_stmt ::=
         RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, engineName, keys, partition,
         distribution, tblProperties, extProperties, tableComment, index);
     :}
-    | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
-            LPAREN column_definition_list:columns COMMA RPAREN opt_engine:engineName
-            opt_keys:keys
-            opt_comment:tableComment
-            opt_partition:partition
-            opt_distribution:distribution
-            opt_rollup:index
-            opt_properties:tblProperties
-            opt_ext_properties:extProperties
-    {:
-        RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, null, engineName, keys, partition,
-        distribution, tblProperties, extProperties, tableComment, index);
-    :}
     | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
             LPAREN column_definition_list:columns COMMA index_definition_list:indexes RPAREN opt_engine:engineName
             opt_keys:keys
@@ -1826,19 +1813,6 @@ create_stmt ::=
         RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, keys, partition,
         distribution, tblProperties, extProperties, tableComment, index);
     :}
-    | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
-            LPAREN column_definition_list:columns COMMA index_definition_list:indexes COMMA RPAREN opt_engine:engineName
-            opt_keys:keys
-            opt_comment:tableComment
-            opt_partition:partition
-            opt_distribution:distribution
-            opt_rollup:index
-            opt_properties:tblProperties
-            opt_ext_properties:extProperties
-    {:
-        RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, keys, partition,
-        distribution, tblProperties, extProperties, tableComment, index);
-    :}
     | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
             opt_col_list:columns
             opt_engine:engineName
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
index ac6cad493ef73c..e25adbaba0ffba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SinglePartitionDesc.java
@@ -68,19 +68,22 @@ public SinglePartitionDesc(boolean ifNotExists, String partName, PartitionKeyDes
     /**
      * for Nereids
      */
-    public SinglePartitionDesc(boolean ifNotExists, String partName, PartitionKeyDesc partitionKeyDesc,
-            ReplicaAllocation replicaAlloc, Map<String, String> properties) {
-        this.ifNotExists = ifNotExists;
-
+    public SinglePartitionDesc(boolean ifNotExists, String partName,
+            PartitionKeyDesc partitionKeyDesc, ReplicaAllocation replicaAlloc,
+            Map<String, String> properties, DataProperty partitionDataProperty, boolean isInMemory,
+            TTabletType tabletType, Long versionInfo, String storagePolicy, boolean isMutable) {
         this.isAnalyzed = true;
-
+        this.ifNotExists = ifNotExists;
         this.partName = partName;
         this.partitionKeyDesc = partitionKeyDesc;
         this.properties = properties;
-
-        this.partitionDataProperty = new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
+        this.partitionDataProperty = partitionDataProperty;
         this.replicaAlloc = replicaAlloc;
-        this.storagePolicy = "";
+        this.isInMemory = isInMemory;
+        this.tabletType = tabletType;
+        this.versionInfo = versionInfo;
+        this.storagePolicy = storagePolicy;
+        this.isMutable = isMutable;
     }
 
     public boolean isSetIfNotExists() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 37b42c7123f75c..0d24a8b9377048 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2430,7 +2430,7 @@ public IndexDefinition visitIndexDef(IndexDefContext ctx) {
         String indexName = ctx.indexName.getText();
         List<String> indexCols = visitIdentifierList(ctx.cols);
         Map<String, String> properties = visitPropertyItemList(ctx.properties);
-        String indexType = ctx.indexType.getText();
+        String indexType = ctx.indexType != null ? ctx.indexType.getText() : null;
         String comment = ctx.comment != null ? ctx.comment.getText() : "";
         return new IndexDefinition(indexName, indexCols, indexType, properties, comment);
     }
@@ -2438,8 +2438,16 @@ public IndexDefinition visitIndexDef(IndexDefContext ctx) {
     @Override
     public List<PartitionDefinition> visitPartitionsDef(PartitionsDefContext ctx) {
         return ctx.partitions.stream()
-                .map(p -> ((PartitionDefinition) visit(p)).withProperties(visitPropertyClause(p.properties)))
-                .collect(Collectors.toList());
+                .map(p -> ((PartitionDefinition) visit(p))).collect(Collectors.toList());
+    }
+
+    @Override
+    public PartitionDefinition visitPartitionDef(DorisParser.PartitionDefContext ctx) {
+        PartitionDefinition partitionDefinition = (PartitionDefinition) visit(ctx.getChild(0));
+        if (ctx.partitionProperties != null) {
+            partitionDefinition.withProperties(visitPropertyItemList(ctx.partitionProperties));
+        }
+        return partitionDefinition;
     }
 
     @Override
@@ -2447,9 +2455,10 @@ public PartitionDefinition visitLessThanPartitionDef(LessThanPartitionDefContext
         String partitionName = ctx.partitionName.getText();
         if (ctx.MAXVALUE() == null) {
             List<Expression> lessThanValues = visitConstantSeq(ctx.constantSeq());
-            return new LessThanPartition(partitionName, lessThanValues);
+            return new LessThanPartition(ctx.EXISTS() != null, partitionName, lessThanValues);
         } else {
-            return new LessThanPartition(partitionName, ImmutableList.of(MaxValue.INSTANCE));
+            return new LessThanPartition(ctx.EXISTS() != null, partitionName,
+                    ImmutableList.of(MaxValue.INSTANCE));
         }
     }
 
@@ -2458,15 +2467,15 @@ public PartitionDefinition visitFixedPartitionDef(FixedPartitionDefContext ctx)
         String partitionName = ctx.partitionName.getText();
         List<Expression> lowerBounds = visitConstantSeq(ctx.lower);
         List<Expression> upperBounds = visitConstantSeq(ctx.upper);
-        return new FixedRangePartition(partitionName, lowerBounds, upperBounds);
+        return new FixedRangePartition(ctx.EXISTS() != null, partitionName, lowerBounds, upperBounds);
     }
 
     @Override
     public PartitionDefinition visitStepPartitionDef(StepPartitionDefContext ctx) {
         List<Expression> fromExpression = visitConstantSeq(ctx.from);
         List<Expression> toExpression = visitConstantSeq(ctx.to);
-        return new StepPartition(fromExpression, toExpression, Long.parseLong(ctx.unitsAmount.getText()),
-                ctx.unit != null ? ctx.unit.getText() : null);
+        return new StepPartition(false, null, fromExpression, toExpression,
+                Long.parseLong(ctx.unitsAmount.getText()), ctx.unit != null ? ctx.unit.getText() : null);
     }
 
     @Override
@@ -2479,7 +2488,7 @@ public PartitionDefinition visitInPartitionDef(InPartitionDefContext ctx) {
             values = visitConstantSeq(ctx.constants).stream().map(ImmutableList::of)
                     .collect(Collectors.toList());
         }
-        return new InPartition(ctx.partitionName.getText(), values);
+        return new InPartition(ctx.EXISTS() != null, ctx.partitionName.getText(), values);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
index 2736d71a50c0bc..5832a0534c0b8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ColumnDefinition.java
@@ -21,6 +21,9 @@
 import org.apache.doris.catalog.AggregateType;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.KeysType;
+import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.catalog.Type;
 import org.apache.doris.common.Config;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.types.ArrayType;
@@ -33,10 +36,13 @@
 import org.apache.doris.nereids.types.StructType;
 import org.apache.doris.nereids.types.TinyIntType;
 import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.qe.SessionVariable;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
@@ -176,9 +182,7 @@ private DataType updateCharacterTypeLength(DataType dataType) {
      * validate column definition and analyze
      */
     public void validate(boolean isOlap, Set<String> keysSet, boolean isEnableMergeOnWrite, KeysType keysType) {
-        if (Config.disable_nested_complex_type && isNestedComplexType(type)) {
-            throw new AnalysisException("Unsupported data type: " + type.toSql());
-        }
+        validateDataType(type.toCatalogDataType());
         type = updateCharacterTypeLength(type);
         if (type.isArrayType()) {
             int depth = 0;
@@ -369,6 +373,256 @@ private boolean isNestedComplexType(DataType dataType) {
         return false;
     }
 
+    // from TypeDef.java analyze()
+    private void validateDataType(Type catalogType) {
+        if (catalogType.exceedsMaxNestingDepth()) {
+            throw new AnalysisException(
+                    String.format("Type exceeds the maximum nesting depth of %s:\n%s",
+                            Type.MAX_NESTING_DEPTH, catalogType.toSql()));
+        }
+        if (!catalogType.isSupported()) {
+            throw new AnalysisException("Unsupported data type: " + catalogType.toSql());
+        }
+
+        if (catalogType.isScalarType()) {
+            validateScalarType((ScalarType) catalogType);
+        } else if (catalogType.isComplexType()) {
+            // now we not support array / map / struct nesting complex type
+            if (catalogType.isArrayType()) {
+                Type itemType = ((org.apache.doris.catalog.ArrayType) catalogType).getItemType();
+                if (itemType instanceof ScalarType) {
+                    validateNestedType(catalogType, (ScalarType) itemType);
+                } else if (Config.disable_nested_complex_type
+                        && !(itemType instanceof org.apache.doris.catalog.ArrayType)) {
+                    // now we can array nesting array
+                    throw new AnalysisException(
+                            "Unsupported data type: ARRAY<" + itemType.toSql() + ">");
+                }
+            }
+            if (catalogType.isMapType()) {
+                org.apache.doris.catalog.MapType mt =
+                        (org.apache.doris.catalog.MapType) catalogType;
+                if (Config.disable_nested_complex_type && (!(mt.getKeyType() instanceof ScalarType)
+                        || !(mt.getValueType() instanceof ScalarType))) {
+                    throw new AnalysisException("Unsupported data type: MAP<"
+                            + mt.getKeyType().toSql() + "," + mt.getValueType().toSql() + ">");
+                }
+                if (mt.getKeyType() instanceof ScalarType) {
+                    validateNestedType(catalogType, (ScalarType) mt.getKeyType());
+                }
+                if (mt.getValueType() instanceof ScalarType) {
+                    validateNestedType(catalogType, (ScalarType) mt.getValueType());
+                }
+            }
+            if (catalogType.isStructType()) {
+                ArrayList<org.apache.doris.catalog.StructField> fields =
+                        ((org.apache.doris.catalog.StructType) catalogType).getFields();
+                Set<String> fieldNames = new HashSet<>();
+                for (org.apache.doris.catalog.StructField field : fields) {
+                    Type fieldType = field.getType();
+                    if (fieldType instanceof ScalarType) {
+                        validateNestedType(catalogType, (ScalarType) fieldType);
+                        if (!fieldNames.add(field.getName())) {
+                            throw new AnalysisException("Duplicate field name " + field.getName()
+                                    + " in struct " + catalogType.toSql());
+                        }
+                    } else if (Config.disable_nested_complex_type) {
+                        throw new AnalysisException(
+                                "Unsupported field type: " + fieldType.toSql() + " for STRUCT");
+                    }
+                }
+            }
+        }
+    }
+
+    private void validateScalarType(ScalarType scalarType) {
+        PrimitiveType type = scalarType.getPrimitiveType();
+        // When string type length is not assigned, it needs to be assigned to 1.
+        if (scalarType.getPrimitiveType().isStringType() && !scalarType.isLengthSet()) {
+            if (scalarType.getPrimitiveType() == PrimitiveType.VARCHAR) {
+                // always set varchar length MAX_VARCHAR_LENGTH
+                scalarType.setLength(ScalarType.MAX_VARCHAR_LENGTH);
+            } else if (scalarType.getPrimitiveType() == PrimitiveType.STRING) {
+                // always set text length MAX_STRING_LENGTH
+                scalarType.setLength(ScalarType.MAX_STRING_LENGTH);
+            } else {
+                scalarType.setLength(1);
+            }
+        }
+        switch (type) {
+            case CHAR:
+            case VARCHAR: {
+                String name;
+                int maxLen;
+                if (type == PrimitiveType.VARCHAR) {
+                    name = "VARCHAR";
+                    maxLen = ScalarType.MAX_VARCHAR_LENGTH;
+                } else {
+                    name = "CHAR";
+                    maxLen = ScalarType.MAX_CHAR_LENGTH;
+                    return;
+                }
+                int len = scalarType.getLength();
+                // len is decided by child, when it is -1.
+
+                if (len <= 0) {
+                    throw new AnalysisException(name + " size must be > 0: " + len);
+                }
+                if (scalarType.getLength() > maxLen) {
+                    throw new AnalysisException(name + " size must be <= " + maxLen + ": " + len);
+                }
+                break;
+            }
+            case DECIMALV2: {
+                int precision = scalarType.decimalPrecision();
+                int scale = scalarType.decimalScale();
+                // precision: [1, 27]
+                if (precision < 1 || precision > ScalarType.MAX_DECIMALV2_PRECISION) {
+                    throw new AnalysisException("Precision of decimal must between 1 and 27."
+                            + " Precision was set to: " + precision + ".");
+                }
+                // scale: [0, 9]
+                if (scale < 0 || scale > ScalarType.MAX_DECIMALV2_SCALE) {
+                    throw new AnalysisException("Scale of decimal must between 0 and 9."
+                            + " Scale was set to: " + scale + ".");
+                }
+                if (precision - scale > ScalarType.MAX_DECIMALV2_PRECISION
+                        - ScalarType.MAX_DECIMALV2_SCALE) {
+                    throw new AnalysisException("Invalid decimal type with precision = " + precision
+                            + ", scale = " + scale);
+                }
+                // scale < precision
+                if (scale > precision) {
+                    throw new AnalysisException("Scale of decimal must be smaller than precision."
+                            + " Scale is " + scale + " and precision is " + precision);
+                }
+                break;
+            }
+            case DECIMAL32: {
+                int decimal32Precision = scalarType.decimalPrecision();
+                int decimal32Scale = scalarType.decimalScale();
+                if (decimal32Precision < 1
+                        || decimal32Precision > ScalarType.MAX_DECIMAL32_PRECISION) {
+                    throw new AnalysisException("Precision of decimal must between 1 and 9."
+                            + " Precision was set to: " + decimal32Precision + ".");
+                }
+                // scale >= 0
+                if (decimal32Scale < 0) {
+                    throw new AnalysisException("Scale of decimal must not be less than 0."
+                            + " Scale was set to: " + decimal32Scale + ".");
+                }
+                // scale < precision
+                if (decimal32Scale > decimal32Precision) {
+                    throw new AnalysisException(
+                            "Scale of decimal must be smaller than precision." + " Scale is "
+                                    + decimal32Scale + " and precision is " + decimal32Precision);
+                }
+                break;
+            }
+            case DECIMAL64: {
+                int decimal64Precision = scalarType.decimalPrecision();
+                int decimal64Scale = scalarType.decimalScale();
+                if (decimal64Precision < 1
+                        || decimal64Precision > ScalarType.MAX_DECIMAL64_PRECISION) {
+                    throw new AnalysisException("Precision of decimal64 must between 1 and 18."
+                            + " Precision was set to: " + decimal64Precision + ".");
+                }
+                // scale >= 0
+                if (decimal64Scale < 0) {
+                    throw new AnalysisException("Scale of decimal must not be less than 0."
+                            + " Scale was set to: " + decimal64Scale + ".");
+                }
+                // scale < precision
+                if (decimal64Scale > decimal64Precision) {
+                    throw new AnalysisException(
+                            "Scale of decimal must be smaller than precision." + " Scale is "
+                                    + decimal64Scale + " and precision is " + decimal64Precision);
+                }
+                break;
+            }
+            case DECIMAL128: {
+                int decimal128Precision = scalarType.decimalPrecision();
+                int decimal128Scale = scalarType.decimalScale();
+                if (decimal128Precision < 1
+                        || decimal128Precision > ScalarType.MAX_DECIMAL128_PRECISION) {
+                    throw new AnalysisException("Precision of decimal128 must between 1 and 38."
+                            + " Precision was set to: " + decimal128Precision + ".");
+                }
+                // scale >= 0
+                if (decimal128Scale < 0) {
+                    throw new AnalysisException("Scale of decimal must not be less than 0."
+                            + " Scale was set to: " + decimal128Scale + ".");
+                }
+                // scale < precision
+                if (decimal128Scale > decimal128Precision) {
+                    throw new AnalysisException(
+                            "Scale of decimal must be smaller than precision." + " Scale is "
+                                    + decimal128Scale + " and precision is " + decimal128Precision);
+                }
+                break;
+            }
+            case DECIMAL256: {
+                if (SessionVariable.getEnableDecimal256()) {
+                    int precision = scalarType.decimalPrecision();
+                    int scale = scalarType.decimalScale();
+                    if (precision < 1 || precision > ScalarType.MAX_DECIMAL256_PRECISION) {
+                        throw new AnalysisException("Precision of decimal256 must between 1 and 76."
+                                + " Precision was set to: " + precision + ".");
+                    }
+                    // scale >= 0
+                    if (scale < 0) {
+                        throw new AnalysisException("Scale of decimal must not be less than 0."
+                                + " Scale was set to: " + scale + ".");
+                    }
+                    // scale < precision
+                    if (scale > precision) {
+                        throw new AnalysisException(
+                                "Scale of decimal must be smaller than precision." + " Scale is "
+                                        + scale + " and precision is " + precision);
+                    }
+                    break;
+                } else {
+                    int precision = scalarType.decimalPrecision();
+                    throw new AnalysisException("Column of type Decimal256 with precision "
+                            + precision + " in not supported.");
+                }
+            }
+            case TIMEV2:
+            case DATETIMEV2: {
+                int precision = scalarType.decimalPrecision();
+                int scale = scalarType.decimalScale();
+                // precision: [1, 27]
+                if (precision != ScalarType.DATETIME_PRECISION) {
+                    throw new AnalysisException(
+                            "Precision of Datetime/Time must be " + ScalarType.DATETIME_PRECISION
+                                    + "." + " Precision was set to: " + precision + ".");
+                }
+                // scale: [0, 9]
+                if (scale < 0 || scale > 6) {
+                    throw new AnalysisException("Scale of Datetime/Time must between 0 and 6."
+                            + " Scale was set to: " + scale + ".");
+                }
+                break;
+            }
+            case INVALID_TYPE:
+                throw new AnalysisException("Invalid type.");
+            default:
+                break;
+        }
+    }
+
+    private void validateNestedType(Type parent, Type child) throws AnalysisException {
+        if (child.isNull()) {
+            throw new AnalysisException("Unsupported data type: " + child.toSql());
+        }
+        // check whether the sub-type is supported
+        if (!parent.supportSubType(child)) {
+            throw new AnalysisException(
+                    parent.getPrimitiveType() + " unsupported sub-type: " + child.toSql());
+        }
+        validateDataType(child);
+    }
+
     /**
      * translate to catalog create table stmt
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/FixedRangePartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/FixedRangePartition.java
index 7655c6b32857c0..104ea0a3eba845 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/FixedRangePartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/FixedRangePartition.java
@@ -20,10 +20,10 @@
 import org.apache.doris.analysis.PartitionKeyDesc;
 import org.apache.doris.analysis.PartitionValue;
 import org.apache.doris.analysis.SinglePartitionDesc;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
 
-import com.google.common.collect.Maps;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -33,12 +33,12 @@
  * represent fixed range partition
  */
 public class FixedRangePartition extends PartitionDefinition {
-    private final String partitionName;
     private List<Expression> lowerBounds;
     private List<Expression> upperBounds;
 
-    public FixedRangePartition(String partitionName, List<Expression> lowerBounds, List<Expression> upperBounds) {
-        this.partitionName = partitionName;
+    public FixedRangePartition(boolean ifNotExists, String partitionName,
+            List<Expression> lowerBounds, List<Expression> upperBounds) {
+        super(ifNotExists, partitionName);
         this.lowerBounds = lowerBounds;
         this.upperBounds = upperBounds;
     }
@@ -46,6 +46,11 @@ public FixedRangePartition(String partitionName, List<Expression> lowerBounds, L
     @Override
     public void validate(Map<String, String> properties) {
         super.validate(properties);
+        try {
+            FeNameFormat.checkPartitionName(partitionName);
+        } catch (Exception e) {
+            throw new AnalysisException(e.getMessage(), e.getCause());
+        }
         List<Expression> newLowerBounds = new ArrayList<>();
         List<Expression> newUpperBounds = new ArrayList<>();
         for (int i = 0; i < partitionTypes.size(); ++i) {
@@ -72,7 +77,9 @@ public SinglePartitionDesc translateToCatalogStyle() {
                 .collect(Collectors.toList());
         List<PartitionValue> upperValues = upperBounds.stream().map(this::toLegacyPartitionValueStmt)
                 .collect(Collectors.toList());
-        return new SinglePartitionDesc(false, partitionName,
-                PartitionKeyDesc.createFixed(lowerValues, upperValues), replicaAllocation, Maps.newHashMap());
+        return new SinglePartitionDesc(ifNotExists, partitionName,
+                PartitionKeyDesc.createFixed(lowerValues, upperValues), replicaAllocation,
+                properties, partitionDataProperty, isInMemory, tabletType, versionInfo,
+                storagePolicy, isMutable);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java
index 30dfcb9efcecbb..611da907e15eed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/InPartition.java
@@ -21,11 +21,10 @@
 import org.apache.doris.analysis.PartitionKeyDesc;
 import org.apache.doris.analysis.PartitionValue;
 import org.apache.doris.analysis.SinglePartitionDesc;
+import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
 
-import com.google.common.collect.Maps;
-
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -35,17 +34,21 @@
  * represent in partition
  */
 public class InPartition extends PartitionDefinition {
-    private final String partitionName;
     private final List<List<Expression>> values;
 
-    public InPartition(String partitionName, List<List<Expression>> values) {
-        this.partitionName = partitionName;
+    public InPartition(boolean ifNotExists, String partitionName, List<List<Expression>> values) {
+        super(ifNotExists, partitionName);
         this.values = values;
     }
 
     @Override
     public void validate(Map<String, String> properties) {
         super.validate(properties);
+        try {
+            FeNameFormat.checkPartitionName(partitionName);
+        } catch (Exception e) {
+            throw new AnalysisException(e.getMessage(), e.getCause());
+        }
         if (values.stream().anyMatch(l -> l.stream().anyMatch(MaxValue.class::isInstance))) {
             throw new AnalysisException("MAXVALUE cannot be used in 'in partition'");
         }
@@ -65,7 +68,9 @@ public AllPartitionDesc translateToCatalogStyle() {
         List<List<PartitionValue>> catalogValues = values.stream().map(l -> l.stream()
                 .map(this::toLegacyPartitionValueStmt)
                 .collect(Collectors.toList())).collect(Collectors.toList());
-        return new SinglePartitionDesc(false, partitionName, PartitionKeyDesc.createIn(catalogValues),
-                replicaAllocation, Maps.newHashMap());
+        return new SinglePartitionDesc(ifNotExists, partitionName,
+                PartitionKeyDesc.createIn(catalogValues), replicaAllocation, properties,
+                partitionDataProperty, isInMemory, tabletType, versionInfo, storagePolicy,
+                isMutable);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LessThanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LessThanPartition.java
index 11a6f4f50da7f2..627c514a550b3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LessThanPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/LessThanPartition.java
@@ -20,10 +20,10 @@
 import org.apache.doris.analysis.PartitionKeyDesc;
 import org.apache.doris.analysis.PartitionValue;
 import org.apache.doris.analysis.SinglePartitionDesc;
+import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
 
-import com.google.common.collect.Maps;
-
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -32,17 +32,21 @@
  * represent less than partition
  */
 public class LessThanPartition extends PartitionDefinition {
-    private final String partitionName;
     private final List<Expression> values;
 
-    public LessThanPartition(String partitionName, List<Expression> values) {
-        this.partitionName = partitionName;
+    public LessThanPartition(boolean ifNotExists, String partitionName, List<Expression> values) {
+        super(ifNotExists, partitionName);
         this.values = values;
     }
 
     @Override
     public void validate(Map<String, String> properties) {
         super.validate(properties);
+        try {
+            FeNameFormat.checkPartitionName(partitionName);
+        } catch (Exception e) {
+            throw new AnalysisException(e.getMessage(), e.getCause());
+        }
     }
 
     public String getPartitionName() {
@@ -54,13 +58,16 @@ public String getPartitionName() {
      */
     public SinglePartitionDesc translateToCatalogStyle() {
         if (values.get(0) instanceof MaxValue) {
-            return new SinglePartitionDesc(false, partitionName, PartitionKeyDesc.createMaxKeyDesc(),
-                    replicaAllocation, Maps.newHashMap());
+            return new SinglePartitionDesc(ifNotExists, partitionName,
+                    PartitionKeyDesc.createMaxKeyDesc(), replicaAllocation, properties,
+                    partitionDataProperty, isInMemory, tabletType, versionInfo, storagePolicy,
+                    isMutable);
         }
-        List<PartitionValue> partitionValues = values.stream()
-                .map(this::toLegacyPartitionValueStmt)
-                .collect(Collectors.toList());
-        return new SinglePartitionDesc(false, partitionName,
-                PartitionKeyDesc.createLessThan(partitionValues), replicaAllocation, Maps.newHashMap());
+        List<PartitionValue> partitionValues =
+                values.stream().map(this::toLegacyPartitionValueStmt).collect(Collectors.toList());
+        return new SinglePartitionDesc(ifNotExists, partitionName,
+                PartitionKeyDesc.createLessThan(partitionValues), replicaAllocation, properties,
+                partitionDataProperty, isInMemory, tabletType, versionInfo, storagePolicy,
+                isMutable);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
index 2d89ed94909267..81ddd7b2ac2199 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.analysis.AllPartitionDesc;
 import org.apache.doris.analysis.PartitionValue;
+import org.apache.doris.catalog.DataProperty;
 import org.apache.doris.catalog.ReplicaAllocation;
 import org.apache.doris.common.util.PropertyAnalyzer;
 import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -27,6 +28,11 @@
 import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.thrift.TTabletType;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
 
 import java.util.List;
 import java.util.Map;
@@ -35,9 +41,25 @@
  * abstract class for partition definition
  */
 public abstract class PartitionDefinition {
-    protected List<DataType> partitionTypes;
+
+    protected boolean ifNotExists;
+    protected String partitionName;
     protected Map<String, String> properties;
+
+    protected List<DataType> partitionTypes;
+    protected DataProperty partitionDataProperty =
+            new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM);
     protected ReplicaAllocation replicaAllocation = ReplicaAllocation.DEFAULT_ALLOCATION;
+    protected boolean isInMemory = false;
+    protected TTabletType tabletType = TTabletType.TABLET_TYPE_DISK;
+    protected Long versionInfo;
+    protected String storagePolicy = "";
+    protected boolean isMutable;
+
+    public PartitionDefinition(boolean ifNotExists, String partName) {
+        this.ifNotExists = ifNotExists;
+        this.partitionName = partName;
+    }
 
     public PartitionDefinition withProperties(Map<String, String> properties) {
         this.properties = properties;
@@ -50,12 +72,74 @@ public PartitionDefinition withProperties(Map<String, String> properties) {
      * Validate the properties.
      * Derived class can override this method to do more validation.
      */
-    public void validate(Map<String, String> properties) {
+    public void validate(Map<String, String> otherProperties) {
         try {
+            if (PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(otherProperties)) {
+                String storagePolicy = PropertyAnalyzer.analyzeStoragePolicy(properties);
+                if (!storagePolicy.isEmpty()) {
+                    throw new AnalysisException(
+                            "Can not create UNIQUE KEY table that enables Merge-On-write"
+                                    + " with storage policy(" + storagePolicy + ")");
+                }
+            }
+            boolean hasStoragePolicy = false;
+            if (properties != null) {
+                hasStoragePolicy = properties.keySet().stream().anyMatch(iter -> {
+                    boolean equal = iter
+                            .compareToIgnoreCase(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY) == 0;
+                    // when find has storage policy properties, here will set it in partition
+                    if (equal) {
+                        storagePolicy = properties.get(iter);
+                    }
+                    return equal;
+                });
+            }
+
+            Map<String, String> mergedMap = Maps.newHashMap();
+            // Should putAll `otherProperties` before `this.properties`,
+            // because the priority of partition is higher than table
+            if (otherProperties != null) {
+                mergedMap.putAll(otherProperties);
+            }
+            if (this.properties != null) {
+                mergedMap.putAll(this.properties);
+            }
+            this.properties = mergedMap;
+
+            // analyze data property
+            partitionDataProperty = PropertyAnalyzer.analyzeDataProperty(properties,
+                    new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM));
+            Preconditions.checkNotNull(partitionDataProperty);
             replicaAllocation = PropertyAnalyzer.analyzeReplicaAllocation(properties, "");
             if (replicaAllocation.isNotSet()) {
                 replicaAllocation = ReplicaAllocation.DEFAULT_ALLOCATION;
             }
+            // analyze version info
+            versionInfo = PropertyAnalyzer.analyzeVersionInfo(properties);
+
+            // analyze in memory
+            isInMemory = PropertyAnalyzer.analyzeBooleanProp(properties,
+                    PropertyAnalyzer.PROPERTIES_INMEMORY, false);
+            if (isInMemory == true) {
+                throw new AnalysisException("Not support set 'in_memory'='true' now!");
+            }
+
+            // analyze is mutable
+            isMutable = PropertyAnalyzer.analyzeBooleanProp(properties,
+                    PropertyAnalyzer.PROPERTIES_MUTABLE, true);
+
+            tabletType = PropertyAnalyzer.analyzeTabletType(properties);
+
+            if (otherProperties == null) {
+                // check unknown properties
+                if (properties != null && !properties.isEmpty()) {
+                    if (!hasStoragePolicy) {
+                        Joiner.MapJoiner mapJoiner = Joiner.on(", ").withKeyValueSeparator(" = ");
+                        throw new AnalysisException(
+                                "Unknown properties: " + mapJoiner.join(properties));
+                    }
+                }
+            }
         } catch (Exception e) {
             throw new AnalysisException(e.getMessage(), e.getCause());
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/StepPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/StepPartition.java
index a02013d377049c..1de00369913676 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/StepPartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/StepPartition.java
@@ -23,8 +23,6 @@
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.nereids.trees.expressions.Expression;
 
-import com.google.common.collect.Maps;
-
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -38,8 +36,9 @@ public class StepPartition extends PartitionDefinition {
     private final long unit;
     private final String unitString;
 
-    public StepPartition(List<Expression> fromExpression, List<Expression> toExpression, long unit,
-            String unitString) {
+    public StepPartition(boolean ifNotExists, String partitionName, List<Expression> fromExpression,
+            List<Expression> toExpression, long unit, String unitString) {
+        super(ifNotExists, partitionName);
         this.fromExpression = fromExpression;
         this.toExpression = toExpression;
         this.unit = unit;
@@ -68,10 +67,10 @@ public MultiPartitionDesc translateToCatalogStyle() {
         try {
             if (unitString == null) {
                 return new MultiPartitionDesc(PartitionKeyDesc.createMultiFixed(fromValues, toValues, unit),
-                        replicaAllocation, Maps.newHashMap());
+                        replicaAllocation, properties);
             }
             return new MultiPartitionDesc(PartitionKeyDesc.createMultiFixed(fromValues, toValues, unit, unitString),
-                    replicaAllocation, Maps.newHashMap());
+                    replicaAllocation, properties);
         } catch (Exception e) {
             throw new AnalysisException(e.getMessage(), e.getCause());
         }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
index e03228524331b1..d818c15b4b9df7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
@@ -64,7 +64,7 @@ protected void runBeforeAll() throws Exception {
                 + ")");
         createTable("CREATE TABLE IF NOT EXISTS customer (\n"
                 + "  C_CUSTKEY     INTEGER NOT NULL,\n"
-                + "  C_NAME        VARCHAR(25) NOT NULL,\n"
+                + "  C_NAME        VARCHAR(25) NOT NULL\n"
                 + ")\n"
                 + "DUPLICATE KEY(C_CUSTKEY, C_NAME)\n"
                 + "DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 3\n"
diff --git a/regression-test/suites/correctness_p0/test_agg_materialize.groovy b/regression-test/suites/correctness_p0/test_agg_materialize.groovy
index 3f79013ea01712..eb44b85fafc522 100644
--- a/regression-test/suites/correctness_p0/test_agg_materialize.groovy
+++ b/regression-test/suites/correctness_p0/test_agg_materialize.groovy
@@ -34,7 +34,7 @@ suite("test_agg_materialize") {
     sql """drop table if exists c5749_bug_t;"""
     sql """CREATE TABLE `c5749_bug_t` (
             `org_code` varchar(255) NULL ,
-            `cash_amt` decimal(27, 9) NULL ,
+            `cash_amt` decimal(27, 9) NULL
             
             ) ENGINE=OLAP
             DUPLICATE KEY(`org_code`)
diff --git a/regression-test/suites/datatype_p0/date/test_date_key.groovy b/regression-test/suites/datatype_p0/date/test_date_key.groovy
index 22c9a018dd393a..dbbe7f6870516b 100644
--- a/regression-test/suites/datatype_p0/date/test_date_key.groovy
+++ b/regression-test/suites/datatype_p0/date/test_date_key.groovy
@@ -20,7 +20,7 @@ suite("test_date_key") {
     sql """
         create table `test_date_key` (
             `k1` datev1, `k2` int,
-            INDEX idx_k1 (`k1`) USING BITMAP,
+            INDEX idx_k1 (`k1`) USING BITMAP
         ) duplicate key(`k1`)
         distributed by hash(k2) buckets 3
         properties("replication_num" = "1");
diff --git a/regression-test/suites/datatype_p0/date/test_datetime_key.groovy b/regression-test/suites/datatype_p0/date/test_datetime_key.groovy
index 848cdc1a4c58af..e349244adc4cd2 100644
--- a/regression-test/suites/datatype_p0/date/test_datetime_key.groovy
+++ b/regression-test/suites/datatype_p0/date/test_datetime_key.groovy
@@ -20,7 +20,7 @@ suite("test_datetime_key") {
     sql """
         create table `test_datetime_key` (
             `k1` datetimev1, `k2` int,
-            INDEX idx_k1 (`k1`) USING BITMAP,
+            INDEX idx_k1 (`k1`) USING BITMAP
         ) duplicate key(`k1`)
         distributed by hash(k2) buckets 3
         properties("replication_num" = "1");
diff --git a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_calc.groovy b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_calc.groovy
index 8b404db4e694ba..399d49bc94a11c 100644
--- a/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_calc.groovy
+++ b/regression-test/suites/datatype_p0/datetimev1/test_datetimev1_calc.groovy
@@ -29,7 +29,7 @@ suite("test_datetimev1_calc", "nonConcurrent") {
     CREATE TABLE IF NOT EXISTS `${table1}` (
         `id` int,
         `value1` datetimev1 NULL COMMENT "",
-        `value2` datetimev1 NULL COMMENT "",
+        `value2` datetimev1 NULL COMMENT ""
       ) ENGINE=OLAP
       DUPLICATE KEY(`id`)
       COMMENT "OLAP"
diff --git a/regression-test/suites/datatype_p0/datev1/test_datev1_calc.groovy b/regression-test/suites/datatype_p0/datev1/test_datev1_calc.groovy
index 3c875633e9423e..ec7a2738e45bdf 100644
--- a/regression-test/suites/datatype_p0/datev1/test_datev1_calc.groovy
+++ b/regression-test/suites/datatype_p0/datev1/test_datev1_calc.groovy
@@ -29,7 +29,7 @@ suite("test_datev1_calc", "nonConcurrent") {
     CREATE TABLE IF NOT EXISTS `${table1}` (
         `id` int,
         `value1` datev1 NULL COMMENT "",
-        `value2` datev1 NULL COMMENT "",
+        `value2` datev1 NULL COMMENT ""
       ) ENGINE=OLAP
       DUPLICATE KEY(`id`)
       COMMENT "OLAP"
diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc.groovy
index e2439c6229dc09..30aa98c9fb6e46 100644
--- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc.groovy
+++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc.groovy
@@ -31,7 +31,7 @@ suite("test_decimalv2_calc", "nonConcurrent") {
         `decimal_key1` decimalv2(8, 5) NULL COMMENT "",
         `decimal_key2` decimalv2(16, 5) NULL COMMENT "",
         `decimal_value1` decimalv2(8, 5) NULL COMMENT "",
-        `decimal_value2` decimalv2(16, 5) NULL COMMENT "",
+        `decimal_value2` decimalv2(16, 5) NULL COMMENT ""
       ) ENGINE=OLAP
       DUPLICATE KEY(`decimal_key1`, `decimal_key2`)
       COMMENT "OLAP"
diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc_with_conversion.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc_with_conversion.groovy
index f500c195b26d3e..c8c196311f45db 100644
--- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc_with_conversion.groovy
+++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_calc_with_conversion.groovy
@@ -31,7 +31,7 @@ suite("test_decimalv2_calc_with_conversion", "nonConcurrent") {
         `decimal_key1` decimalv2(8, 5) NULL COMMENT "",
         `decimal_key2` decimalv2(16, 5) NULL COMMENT "",
         `decimal_value1` decimalv2(8, 5) NULL COMMENT "",
-        `decimal_value2` decimalv2(16, 5) NULL COMMENT "",
+        `decimal_value2` decimalv2(16, 5) NULL COMMENT ""
       ) ENGINE=OLAP
       DUPLICATE KEY(`decimal_key1`, `decimal_key2`)
       COMMENT "OLAP"
diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy
index f371c4c812fe97..384ba1fd8c367b 100644
--- a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy
+++ b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy
@@ -32,7 +32,7 @@ suite("test_decimalv3_overflow") {
 	def tblName2 = "test_decimalv3_overflow2"
 	sql "drop table if exists ${tblName2}"
     sql """ CREATE  TABLE ${tblName2} (
-              `c2`  decimalv3(20, 2),
+              `c2`  decimalv3(20, 2)
           ) ENGINE=OLAP
         UNIQUE KEY(`c2`)
         DISTRIBUTED BY HASH(`c2`) BUCKETS 10
diff --git a/regression-test/suites/datatype_p0/decimalv3/test_uniq_tab_decimalv2.groovy b/regression-test/suites/datatype_p0/decimalv3/test_uniq_tab_decimalv2.groovy
index 90efdd3a216180..ca2ea138d6b577 100644
--- a/regression-test/suites/datatype_p0/decimalv3/test_uniq_tab_decimalv2.groovy
+++ b/regression-test/suites/datatype_p0/decimalv3/test_uniq_tab_decimalv2.groovy
@@ -26,7 +26,7 @@ suite("test_uniq_tab_decimalv2") {
       `decimal_key1` decimalv2(8, 5) NULL COMMENT "",
       `decimal_key2` decimalv2(16, 5) NULL COMMENT "",
       `decimal_value1` decimalv2(8, 5) NULL COMMENT "",
-      `decimal_value2` decimalv2(16, 5) NULL COMMENT "",
+      `decimal_value2` decimalv2(16, 5) NULL COMMENT ""
     ) ENGINE=OLAP
     UNIQUE KEY(`decimal_key1`, `decimal_key2`)
     COMMENT "OLAP"
diff --git a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_unsupport_types.groovy b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_unsupport_types.groovy
index 45b4176c0ba699..7a6b1218b7df7c 100644
--- a/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_unsupport_types.groovy
+++ b/regression-test/suites/datatype_p0/nested_types/ddl/create_nestedtypes_with_unsupport_types.groovy
@@ -25,7 +25,7 @@ suite("create_nestedtypes_with_unsupport_types") {
             sql """
             CREATE TABLE IF NOT EXISTS ${testTablex} (
               `k1` INT(11) NULL,
-              `k2` array<${columnType}> NULL,
+              `k2` array<${columnType}> NULL
             )
             DUPLICATE KEY(`k1`)
             DISTRIBUTED BY HASH(`k1`) BUCKETS 1
@@ -48,7 +48,7 @@ suite("create_nestedtypes_with_unsupport_types") {
                 sql """
             CREATE TABLE IF NOT EXISTS ${testTablex} (
               `k1` INT(11) NULL,
-              `k2` map<${columnType}, String> NULL,
+              `k2` map<${columnType}, String> NULL
             )
             DUPLICATE KEY(`k1`)
             DISTRIBUTED BY HASH(`k1`) BUCKETS 1
@@ -60,7 +60,7 @@ suite("create_nestedtypes_with_unsupport_types") {
                 sql """
             CREATE TABLE IF NOT EXISTS ${testTablex} (
               `k1` INT(11) NULL,
-              `k2` map<String, ${columnType}> NULL,
+              `k2` map<String, ${columnType}> NULL
             )
             DUPLICATE KEY(`k1`)
             DISTRIBUTED BY HASH(`k1`) BUCKETS 1
@@ -84,7 +84,7 @@ suite("create_nestedtypes_with_unsupport_types") {
             sql """
             CREATE TABLE IF NOT EXISTS ${testTablex} (
               `k1` INT(11) NULL,
-              `k2` STRUCT<f1:${columnType}> NULL,
+              `k2` STRUCT<f1:${columnType}> NULL
             )
             DUPLICATE KEY(`k1`)
             DISTRIBUTED BY HASH(`k1`) BUCKETS 1
diff --git a/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_multi_table_create.sql b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_multi_table_create.sql
index ea0a1fe63d6e53..63ab9c268a51eb 100644
--- a/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_multi_table_create.sql
+++ b/regression-test/suites/load_p0/routine_load/ddl/dup_tbl_basic_multi_table_create.sql
@@ -29,7 +29,7 @@ CREATE TABLE routine_load_dup_tbl_basic_multi_table
     INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
     INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"),
 
-    INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+    INDEX idx_bitmap_k104 (`k02`) USING BITMAP
     
 )
 DUPLICATE KEY(k00)
diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
index 83431ede6ab55d..1a5d3ca7a0acfb 100644
--- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
+++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_or_datetime_computation_negative.groovy
@@ -23,7 +23,7 @@ suite("test_date_or_datetime_computation_negative") {
                 `dateV2` DATEV2 NOT NULL,
                 `dateV2_null` DATEV2 NULL,
                 `datetime` DATETIME NOT NULL,
-                `datetime_null` DATETIME NULL, )
+                `datetime_null` DATETIME NULL )
             DUPLICATE KEY(`row_id`)
             DISTRIBUTED BY HASH(`row_id`) BUCKETS 1
             PROPERTIES (
diff --git a/regression-test/suites/query_p0/session_variable/test_default_limit.groovy b/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
index ad28f98aa39ccd..4133ccc33976be 100644
--- a/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
+++ b/regression-test/suites/query_p0/session_variable/test_default_limit.groovy
@@ -23,7 +23,7 @@ suite('test_default_limit', "arrow_flight_sql") {
         create table baseall (
             k0 int,
             k1 int,
-            k2 int,
+            k2 int
         )
         distributed by hash(k0) buckets 16
         properties(
@@ -35,7 +35,7 @@ suite('test_default_limit', "arrow_flight_sql") {
         create table bigtable (
             k0 int,
             k1 int,
-            k2 int,
+            k2 int
         )
         distributed by hash(k0) buckets 16
         properties(
diff --git a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy
index 96cdc229de8f9e..b062920aa44991 100644
--- a/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy
+++ b/regression-test/suites/query_p0/sql_functions/aggregate_functions/test_count_distinct_with_collection_type.groovy
@@ -24,7 +24,7 @@ suite("test_count_distinct_with_collection_type", "p0") {
       `id` int(11) NULL,
       `a` array<text> NULL,
       `m` map<text, int> NULL,
-      `s` STRUCT<f1:INT, f2:VARCHAR(30), f3: DECIMAL> NULL,
+      `s` STRUCT<f1:INT, f2:VARCHAR(30), f3: DECIMAL> NULL
     ) ENGINE=OLAP
     DUPLICATE KEY(`id`)
     COMMENT 'OLAP'
diff --git a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy
index c043ada07af502..858a4bdbd37ef3 100644
--- a/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy
+++ b/regression-test/suites/query_p0/sql_functions/array_functions/test_array_functions.groovy
@@ -314,7 +314,7 @@ suite("test_array_functions") {
                 `kdcmlv3s1` decimalv3(9, 3) null,
                 `kdcmlv3s2` decimalv3(15, 5) null,
                 `kdcmlv3s3` decimalv3(27, 9) null,
-                `kadcml` array<decimal(27, 9)> null,
+                `kadcml` array<decimal(27, 9)> null
             ) engine=olap
             DISTRIBUTED BY HASH(`id`) BUCKETS 1
             properties("replication_num" = "1")
diff --git a/regression-test/suites/query_p0/sql_functions/size_funciton/test_size_function.groovy b/regression-test/suites/query_p0/sql_functions/size_funciton/test_size_function.groovy
index fe2607ecb12065..93f1b58158746c 100644
--- a/regression-test/suites/query_p0/sql_functions/size_funciton/test_size_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/size_funciton/test_size_function.groovy
@@ -37,7 +37,7 @@ suite("test_size_function") {
                     `id` int(11) NULL,
                     `c_array1` array<int(11)> NULL,
                     `c_array2` array<varchar(20)> NULL,
-                    `c_map` map<int(11), string> NULL,
+                    `c_map` map<int(11), string> NULL
                 ) ENGINE=OLAP
             DUPLICATE KEY(`id`)
             DISTRIBUTED BY HASH(`id`) BUCKETS 1

From b8fd55b0cf0f128b67ab44fb895f3a331fdc66ac Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Tue, 26 Dec 2023 11:47:04 +0800
Subject: [PATCH 028/299] [feature](Nereids) support table valued function
 http_stream (#29004)

---
 .../catalog/BuiltinTableValuedFunctions.java  |  2 +
 .../functions/table/HttpStream.java           | 56 +++++++++++++++++++
 .../visitor/TableValuedFunctionVisitor.java   |  5 ++
 3 files changed, 63 insertions(+)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/HttpStream.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
index 66b0fb49de6691..d1b63fe6237019 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/BuiltinTableValuedFunctions.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.table.FrontendsDisks;
 import org.apache.doris.nereids.trees.expressions.functions.table.GroupCommit;
 import org.apache.doris.nereids.trees.expressions.functions.table.Hdfs;
+import org.apache.doris.nereids.trees.expressions.functions.table.HttpStream;
 import org.apache.doris.nereids.trees.expressions.functions.table.IcebergMeta;
 import org.apache.doris.nereids.trees.expressions.functions.table.Jobs;
 import org.apache.doris.nereids.trees.expressions.functions.table.Local;
@@ -51,6 +52,7 @@ public class BuiltinTableValuedFunctions implements FunctionHelper {
             tableValued(Local.class, "local"),
             tableValued(IcebergMeta.class, "iceberg_meta"),
             tableValued(Hdfs.class, "hdfs"),
+            tableValued(HttpStream.class, "http_stream"),
             tableValued(Numbers.class, "numbers"),
             tableValued(Queries.class, "queries"),
             tableValued(S3.class, "s3"),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/HttpStream.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/HttpStream.java
new file mode 100644
index 00000000000000..de052b078db43e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/table/HttpStream.java
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions.table;
+
+import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.expressions.Properties;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.coercion.AnyDataType;
+import org.apache.doris.tablefunction.HttpStreamTableValuedFunction;
+import org.apache.doris.tablefunction.TableValuedFunctionIf;
+
+import java.util.Map;
+
+/** http_stream */
+public class HttpStream extends TableValuedFunction {
+    public HttpStream(Properties properties) {
+        super("http_stream", properties);
+    }
+
+    @Override
+    public FunctionSignature customSignature() {
+        return FunctionSignature.of(AnyDataType.INSTANCE_WITHOUT_INDEX, getArgumentsTypes());
+    }
+
+    @Override
+    protected TableValuedFunctionIf toCatalogFunction() {
+        try {
+            Map<String, String> arguments = getTVFProperties().getMap();
+            return new HttpStreamTableValuedFunction(arguments);
+        } catch (Throwable t) {
+            throw new AnalysisException("Can not build HttpStreamTableValuedFunction by "
+                + this + ": " + t.getMessage(), t);
+        }
+    }
+
+    @Override
+    public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+        return visitor.visitHttpStream(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
index 9967e472694db7..fe07097bc81fca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/TableValuedFunctionVisitor.java
@@ -23,6 +23,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.table.FrontendsDisks;
 import org.apache.doris.nereids.trees.expressions.functions.table.GroupCommit;
 import org.apache.doris.nereids.trees.expressions.functions.table.Hdfs;
+import org.apache.doris.nereids.trees.expressions.functions.table.HttpStream;
 import org.apache.doris.nereids.trees.expressions.functions.table.IcebergMeta;
 import org.apache.doris.nereids.trees.expressions.functions.table.Jobs;
 import org.apache.doris.nereids.trees.expressions.functions.table.Local;
@@ -74,6 +75,10 @@ default R visitHdfs(Hdfs hdfs, C context) {
         return visitTableValuedFunction(hdfs, context);
     }
 
+    default R visitHttpStream(HttpStream httpStream, C context) {
+        return visitTableValuedFunction(httpStream, context);
+    }
+
     default R visitIcebergMeta(IcebergMeta icebergMeta, C context) {
         return visitTableValuedFunction(icebergMeta, context);
     }

From 509cfea99ad55fa9d45fe7237fe2cbe5aa9d5a1a Mon Sep 17 00:00:00 2001
From: slothever <18522955+wsjz@users.noreply.github.com>
Date: Tue, 26 Dec 2023 12:29:05 +0800
Subject: [PATCH 029/299] [feature](Load)(step2)support nereids load job
 schedule (#26356)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We will  Integrate new load job manager into  new job scheduling framework so that the insert into task can be scheduled after the broker load sql  is converted to insert into TVF(table value function) sql.

issue: https://github.com/apache/doris/issues/24221

Now support:
1. load data by tvf insert into sql, but just for simple load（columns need to be defined in the table）
2. show load stmt
- job id, label name, job state, time info
- simple progress
3. cancel load from db
4. support that enable new load through Config.enable_nereids_load
5. can replay job after restarting doris

TODO:
- support partition insert job
- support show statistics from BE
- support multiple task and collect task statistic
- support transactional task
- need add ut case
---
 .../java/org/apache/doris/common/Config.java  |  12 +
 .../org/apache/doris/nereids/DorisParser.g4   |  15 +-
 .../apache/doris/analysis/CreateJobStmt.java  |  22 +-
 .../apache/doris/analysis/ShowLoadStmt.java   |   7 +
 .../java/org/apache/doris/catalog/Env.java    |   9 +
 .../apache/doris/job/base/AbstractJob.java    |  98 +++-
 .../java/org/apache/doris/job/base/Job.java   |  25 +-
 .../apache/doris/job/base/JobExecuteType.java |   2 +-
 .../apache/doris/job/common/JobStatus.java    |   1 -
 .../org/apache/doris/job/common/JobType.java  |   2 +-
 .../job/extensions/insert/InsertJob.java      | 496 +++++++++++++++---
 .../job/extensions/insert/InsertTask.java     | 128 +++--
 .../doris/job/extensions/mtmv/MTMVJob.java    |   2 +-
 .../apache/doris/job/manager/JobManager.java  | 236 ++++++++-
 .../apache/doris/job/task/AbstractTask.java   |  10 +
 .../java/org/apache/doris/load/ExportJob.java |   9 +-
 .../java/org/apache/doris/load/ExportMgr.java |  26 +-
 .../apache/doris/load/ExportTaskExecutor.java |   6 +
 .../org/apache/doris/load/loadv2/LoadJob.java | 105 ----
 .../apache/doris/load/loadv2/LoadManager.java |   8 +-
 .../doris/load/loadv2/LoadStatistic.java      | 142 +++++
 .../nereids/jobs/load/LabelProcessor.java     | 181 +++++++
 .../nereids/parser/LogicalPlanBuilder.java    |  14 +-
 .../trees/plans/commands/InsertExecutor.java  |  13 +-
 .../commands/InsertIntoTableCommand.java      |  15 +
 .../trees/plans/commands/LoadCommand.java     |  51 +-
 .../org/apache/doris/persist/EditLog.java     |  30 +-
 .../java/org/apache/doris/qe/DdlExecutor.java |   5 +-
 .../org/apache/doris/qe/ShowExecutor.java     |  17 +-
 .../executor/TransientTaskExecutor.java       |   2 +
 .../manager/TransientTaskManager.java         |   5 +-
 .../registry/ExportTaskRegister.java          |   2 +-
 .../suites/job_p0/test_base_insert_job.groovy |   2 +-
 33 files changed, 1324 insertions(+), 374 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 04d33fd4cc0bd0..c038a9f1780ceb 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1931,6 +1931,18 @@ public class Config extends ConfigBase {
     @ConfField(masterOnly = true)
     public static boolean enable_hms_events_incremental_sync = false;
 
+    /**
+     * If set to true, doris will try to parse the ddl of a hive view and try to execute the query
+     * otherwise it will throw an AnalysisException.
+     */
+    @ConfField(mutable = true, varType = VariableAnnotation.EXPERIMENTAL, description = {
+            "当前默认设置为 false，开启后支持使用新优化器的load语句导入数据，失败后会降级旧的load语句。",
+            "Now default set to true, After this function is enabled, the load statement of "
+                    + "the new optimizer can be used to import data. If this function fails, "
+                    + "the old load statement will be degraded."})
+    public static boolean enable_nereids_load = false;
+
+
     /**
      * Maximum number of events to poll in each RPC.
      */
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index a2f90ea21667c8..dc202fef7431c0 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -71,11 +71,6 @@ statement
         (withRemoteStorageSystem)?
         (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)?
         (commentSpec)?                                                 #load
-    | LOAD LABEL lableName=identifier
-        LEFT_PAREN dataDescs+=dataDesc (COMMA dataDescs+=dataDesc)* RIGHT_PAREN
-        resourceDesc
-        (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)?
-        (commentSpec)?                                                 #resourceLoad
     | LOAD mysqlDataDesc
         (PROPERTIES LEFT_PAREN properties=propertyItemList RIGHT_PAREN)?
         (commentSpec)?                                                 #mysqlLoad
@@ -131,7 +126,7 @@ dataDesc
         (PARTITION partition=identifierList)?
         (COLUMNS TERMINATED BY comma=STRING_LITERAL)?
         (LINES TERMINATED BY separator=STRING_LITERAL)?
-        (FORMAT AS format=identifier)?
+        (FORMAT AS format=identifierOrStringLiteral)?
         (columns=identifierList)?
         (columnsFromPath=colFromPath)?
         (columnMapping=colMappingList)?
@@ -167,6 +162,11 @@ refreshMethod
     : COMPLETE | AUTO
     ;
 
+identifierOrStringLiteral
+    : identifier
+    | STRING_LITERAL
+    ;
+
 identifierOrText
     : errorCapturingIdentifier
     | STRING_LITERAL
@@ -224,7 +224,8 @@ mappingExpr
     ;
 
 withRemoteStorageSystem
-    : WITH S3 LEFT_PAREN
+    : resourceDesc
+    | WITH S3 LEFT_PAREN
         brokerProperties=propertyItemList
         RIGHT_PAREN
     | WITH HDFS LEFT_PAREN
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
index ececccc3169715..ef76aedba2b400 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
@@ -117,10 +117,8 @@ public void analyze(Analyzer analyzer) throws UserException {
         analyzerSqlStmt();
         // check its insert stmt,currently only support insert stmt
         //todo when support other stmt,need to check stmt type and generate jobInstance
-        InsertJob job = new InsertJob();
         JobExecutionConfiguration jobExecutionConfiguration = new JobExecutionConfiguration();
         jobExecutionConfiguration.setExecuteType(executeType);
-        job.setCreateTimeMs(System.currentTimeMillis());
         TimerDefinition timerDefinition = new TimerDefinition();
 
         if (null != onceJobStartTimestamp) {
@@ -148,17 +146,19 @@ public void analyze(Analyzer analyzer) throws UserException {
         }
         checkJobName(labelName.getLabelName());
         jobExecutionConfiguration.setTimerDefinition(timerDefinition);
-        job.setJobConfig(jobExecutionConfiguration);
-
-        job.setComment(comment);
-        job.setCurrentDbName(labelName.getDbName());
-        job.setJobName(labelName.getLabelName());
-        job.setCreateUser(ConnectContext.get().getCurrentUserIdentity());
-        job.setJobStatus(JobStatus.RUNNING);
-        job.setJobId(Env.getCurrentEnv().getNextId());
         String originStmt = getOrigStmt().originStmt;
         String executeSql = parseExecuteSql(originStmt);
-        job.setExecuteSql(executeSql);
+        // create job use label name as its job name
+        String jobName = labelName.getLabelName();
+        InsertJob job = new InsertJob(jobName,
+                JobStatus.RUNNING,
+                labelName.getDbName(),
+                comment,
+                ConnectContext.get().getCurrentUserIdentity(),
+                jobExecutionConfiguration,
+                System.currentTimeMillis(),
+                executeSql);
+        //job.checkJobParams();
         jobInstance = job;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java
index ae8283f6ddfa72..1e00b5d8897ca6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowLoadStmt.java
@@ -108,6 +108,13 @@ public Set<JobState> getStates() {
         return states;
     }
 
+    public org.apache.doris.load.loadv2.JobState getStateV2() {
+        if (Strings.isNullOrEmpty(stateValue)) {
+            return null;
+        }
+        return org.apache.doris.load.loadv2.JobState.valueOf(stateValue);
+    }
+
     public boolean isAccurateMatch() {
         return isAccurateMatch;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 9ae7d4f1617660..cd77f70f59c4fa 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -182,6 +182,7 @@
 import org.apache.doris.mysql.privilege.AccessControllerManager;
 import org.apache.doris.mysql.privilege.Auth;
 import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.jobs.load.LabelProcessor;
 import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVPropertyInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVRefreshInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
@@ -362,6 +363,7 @@ public class Env {
 
     private ExportTaskRegister exportTaskRegister;
     private JobManager<? extends AbstractJob<?, ?>, ?> jobManager;
+    private LabelProcessor labelProcessor;
     private TransientTaskManager transientTaskManager;
 
     private MasterDaemon labelCleaner; // To clean old LabelInfo, ExportJobInfos
@@ -641,8 +643,11 @@ private Env(boolean isCheckpointCatalog) {
         }
         this.metastoreEventsProcessor = new MetastoreEventsProcessor();
         this.jobManager = new JobManager<>();
+        this.labelProcessor = new LabelProcessor();
         this.transientTaskManager = new TransientTaskManager();
         this.exportTaskRegister = new ExportTaskRegister(transientTaskManager);
+        this.transientTaskManager = new TransientTaskManager();
+
         this.replayedJournalId = new AtomicLong(0L);
         this.stmtIdCounter = new AtomicLong(0L);
         this.isElectable = false;
@@ -3907,6 +3912,10 @@ public JobManager getJobManager() {
         return jobManager;
     }
 
+    public LabelProcessor getLabelProcessor() {
+        return labelProcessor;
+    }
+
     public TransientTaskManager getTransientTaskManager() {
         return transientTaskManager;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
index 83f02326d82f26..6e9cb48da1ca44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
@@ -23,6 +23,8 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
+import org.apache.doris.common.util.LogBuilder;
+import org.apache.doris.common.util.LogKey;
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.job.common.JobStatus;
 import org.apache.doris.job.common.TaskStatus;
@@ -76,14 +78,55 @@ public abstract class AbstractJob<T extends AbstractTask, C> implements Job<T, C
     private JobExecutionConfiguration jobConfig;
 
     @SerializedName(value = "ctms")
-    private Long createTimeMs;
+    private long createTimeMs;
 
-    @SerializedName(value = "sql")
-    String executeSql;
+    @SerializedName(value = "stm")
+    private long startTimeMs = -1L;
 
     @SerializedName(value = "ftm")
     private long finishTimeMs;
 
+    @SerializedName(value = "sql")
+    String executeSql;
+
+    public AbstractJob() {}
+
+    public AbstractJob(Long id) {
+        jobId = id;
+    }
+
+    /**
+     * executeSql and runningTasks is not required for load.
+     */
+    public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
+                            String currentDbName,
+                            String comment,
+                            UserIdentity createUser,
+                            JobExecutionConfiguration jobConfig) {
+        this(jobId, jobName, jobStatus, currentDbName, comment,
+                createUser, jobConfig, System.currentTimeMillis(), null, null);
+    }
+
+    public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
+                       String currentDbName,
+                       String comment,
+                       UserIdentity createUser,
+                       JobExecutionConfiguration jobConfig,
+                       Long createTimeMs,
+                       String executeSql,
+                       List<T> runningTasks) {
+        this.jobId = jobId;
+        this.jobName = jobName;
+        this.jobStatus = jobStatus;
+        this.currentDbName = currentDbName;
+        this.comment = comment;
+        this.createUser = createUser;
+        this.jobConfig = jobConfig;
+        this.createTimeMs = createTimeMs;
+        this.executeSql = executeSql;
+        this.runningTasks = runningTasks;
+    }
+
     private List<T> runningTasks = new ArrayList<>();
 
     @Override
@@ -109,6 +152,10 @@ public void cancelAllTasks() throws JobException {
                     .add("Comment")
                     .build();
 
+    protected static long getNextJobId() {
+        return System.nanoTime() + RandomUtils.nextInt();
+    }
+
     @Override
     public void cancelTaskById(long taskId) throws JobException {
         if (CollectionUtils.isEmpty(runningTasks)) {
@@ -154,17 +201,18 @@ public List<T> commonCreateTasks(TaskType taskType, C taskContext) {
         return createTasks(taskType, taskContext);
     }
 
-    public void initTasks(List<? extends AbstractTask> tasks) {
+    public void initTasks(Collection<? extends T> tasks, TaskType taskType) {
+        if (CollectionUtils.isEmpty(getRunningTasks())) {
+            runningTasks = new ArrayList<>();
+        }
         tasks.forEach(task -> {
-            task.setJobId(jobId);
-            task.setTaskId(getNextId());
+            task.setTaskType(taskType);
+            task.setJobId(getJobId());
             task.setCreateTimeMs(System.currentTimeMillis());
             task.setStatus(TaskStatus.PENDING);
         });
-        if (CollectionUtils.isEmpty(getRunningTasks())) {
-            setRunningTasks(new ArrayList<>());
-        }
-        getRunningTasks().addAll((Collection<? extends T>) tasks);
+        getRunningTasks().addAll(tasks);
+        this.startTimeMs = System.currentTimeMillis();
     }
 
     public void checkJobParams() {
@@ -208,10 +256,22 @@ public void updateJobStatus(JobStatus newJobStatus) throws JobException {
     public static AbstractJob readFields(DataInput in) throws IOException {
         String jsonJob = Text.readString(in);
         AbstractJob job = GsonUtils.GSON.fromJson(jsonJob, AbstractJob.class);
-        job.setRunningTasks(new ArrayList<>());
+        job.runningTasks = new ArrayList<>();
         return job;
     }
 
+    public void logCreateOperation() {
+        Env.getCurrentEnv().getEditLog().logCreateJob(this);
+    }
+
+    public void logFinalOperation() {
+        Env.getCurrentEnv().getEditLog().logEndJob(this);
+    }
+
+    public void logUpdateOperation() {
+        Env.getCurrentEnv().getEditLog().logUpdateJob(this);
+    }
+
     @Override
     public void onTaskFail(T task) throws JobException {
         updateJobStatusIfEnd();
@@ -303,7 +363,19 @@ public ShowResultSetMetaData getJobMetaData() {
         return builder.build();
     }
 
-    private static long getNextId() {
-        return System.nanoTime() + RandomUtils.nextInt();
+    @Override
+    public void onRegister() throws JobException {}
+
+    @Override
+    public void onUnRegister() throws JobException {}
+
+    @Override
+    public void onReplayCreate() throws JobException {
+        log.info(new LogBuilder(LogKey.SCHEDULER_JOB, getJobId()).add("msg", "replay create scheduler job").build());
+    }
+
+    @Override
+    public void onReplayEnd(AbstractJob<?, C> replayJob) throws JobException {
+        log.info(new LogBuilder(LogKey.SCHEDULER_JOB, getJobId()).add("msg", "replay delete scheduler job").build());
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java
index ee352a0f417667..1124e7f2d28028 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/Job.java
@@ -99,11 +99,34 @@ public interface Job<T extends AbstractTask, C> {
 
     /**
      * Cancels all running tasks of this job.
-     *
      * @throws JobException If cancelling a running task fails.
      */
     void cancelAllTasks() throws JobException;
 
+    /**
+     * register job
+     * @throws JobException If register job failed.
+     */
+    void onRegister() throws JobException;
+
+    /**
+     * register job failed
+     * @throws JobException If failed.
+     */
+    void onUnRegister() throws JobException;
+
+    /**
+     * replay create job
+     * @throws JobException  If replay create failed.
+     */
+    void onReplayCreate() throws JobException;
+
+    /**
+     * replay finished or cancelled job
+     * @throws JobException If replay end failed.
+     */
+    void onReplayEnd(AbstractJob<?, C> replayJob) throws JobException;
+
     /**
      * Notifies the job when a task execution fails.
      *
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java
index ea9ddb3b0203d4..3529a2efeff7da 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/JobExecuteType.java
@@ -37,7 +37,7 @@ public enum JobExecuteType {
      */
     MANUAL,
     /**
-     * The job will be executed immediately.
+     * The job will be executed only once and immediately.
      */
     INSTANT,
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java
index 2df65e4654d414..22b799225d1a06 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobStatus.java
@@ -36,7 +36,6 @@ public enum JobStatus {
      * The stop state cannot be resumed
      */
     STOPPED,
-
     /**
      * When the task is finished, the finished state will be triggered.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java
index 1beb4e0a3840f3..084a39ddf096bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/common/JobType.java
@@ -19,5 +19,5 @@
 
 public enum JobType {
     INSERT,
-    MV
+    MV,
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
index 74581b8f1b0b49..9256864efcacc1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
@@ -17,43 +17,74 @@
 
 package org.apache.doris.job.extensions.insert;
 
+import org.apache.doris.analysis.LoadStmt;
+import org.apache.doris.analysis.UserIdentity;
+import org.apache.doris.catalog.AuthorizationInfo;
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.common.Config;
-import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.LabelAlreadyUsedException;
 import org.apache.doris.common.io.Text;
+import org.apache.doris.common.util.LogBuilder;
+import org.apache.doris.common.util.LogKey;
+import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.job.base.AbstractJob;
 import org.apache.doris.job.base.JobExecuteType;
+import org.apache.doris.job.base.JobExecutionConfiguration;
+import org.apache.doris.job.common.JobStatus;
 import org.apache.doris.job.common.JobType;
 import org.apache.doris.job.common.TaskType;
 import org.apache.doris.job.exception.JobException;
-import org.apache.doris.load.loadv2.LoadJob;
+import org.apache.doris.load.FailMsg;
+import org.apache.doris.load.loadv2.LoadStatistic;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.mysql.privilege.Privilege;
 import org.apache.doris.nereids.trees.plans.commands.InsertIntoTableCommand;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.ShowResultSetMetaData;
 import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.thrift.TUniqueId;
+import org.apache.doris.transaction.ErrorTabletInfo;
+import org.apache.doris.transaction.TabletCommitInfo;
 
+import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
 import com.google.gson.annotations.SerializedName;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
+import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.stream.Collectors;
 
+@EqualsAndHashCode(callSuper = true)
 @Data
 @Slf4j
-public class InsertJob extends AbstractJob<InsertTask, Map> {
+public class InsertJob extends AbstractJob<InsertTask, Map<Object, Object>> {
 
     public static final ImmutableList<Column> SCHEMA = ImmutableList.of(
             new Column("Id", ScalarType.createStringType()),
@@ -66,40 +97,145 @@ public class InsertJob extends AbstractJob<InsertTask, Map> {
             new Column("CreateTime", ScalarType.createStringType()),
             new Column("Comment", ScalarType.createStringType()));
 
+    private static final ShowResultSetMetaData TASK_META_DATA =
+            ShowResultSetMetaData.builder()
+                    .addColumn(new Column("TaskId", ScalarType.createVarchar(80)))
+                    .addColumn(new Column("Label", ScalarType.createVarchar(80)))
+                    .addColumn(new Column("Status", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("EtlInfo", ScalarType.createVarchar(100)))
+                    .addColumn(new Column("TaskInfo", ScalarType.createVarchar(100)))
+                    .addColumn(new Column("ErrorMsg", ScalarType.createVarchar(100)))
+
+                    .addColumn(new Column("CreateTimeMs", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("FinishTimeMs", ScalarType.createVarchar(20)))
+                    .addColumn(new Column("TrackingUrl", ScalarType.createVarchar(200)))
+                    .addColumn(new Column("LoadStatistic", ScalarType.createVarchar(200)))
+                    .addColumn(new Column("User", ScalarType.createVarchar(50)))
+                    .build();
+
     public static final ImmutableMap<String, Integer> COLUMN_TO_INDEX;
 
     static {
-        ImmutableMap.Builder<String, Integer> builder = new ImmutableMap.Builder();
+        ImmutableMap.Builder<String, Integer> builder = new ImmutableMap.Builder<>();
         for (int i = 0; i < SCHEMA.size(); i++) {
             builder.put(SCHEMA.get(i).getName().toLowerCase(), i);
         }
         COLUMN_TO_INDEX = builder.build();
     }
 
-    @SerializedName(value = "lp")
-    String labelPrefix;
+    @SerializedName("tis")
+    ConcurrentLinkedQueue<Long> historyTaskIdList;
+    @SerializedName("did")
+    private final long dbId;
+    @SerializedName("ln")
+    private String labelName;
+    @SerializedName("lt")
+    private InsertJob.LoadType loadType;
+    // 0: the job status is pending
+    // n/100: n is the number of task which has been finished
+    // 99: all tasks have been finished
+    // 100: txn status is visible and load has been finished
+    @SerializedName("pg")
+    private int progress;
+    @SerializedName("fm")
+    private FailMsg failMsg;
+    @SerializedName("plans")
+    private List<InsertIntoTableCommand> plans = new ArrayList<>();
+    private LoadStatistic loadStatistic = new LoadStatistic();
+    private Set<Long> finishedTaskIds = new HashSet<>();
+    private ConcurrentHashMap<Long, InsertTask> idToTasks = new ConcurrentHashMap<>();
+    private Map<String, String> properties = new HashMap<>();
+    private Set<String> tableNames;
+    private AuthorizationInfo authorizationInfo;
 
-    InsertIntoTableCommand command;
+    private ConnectContext ctx;
+    private StmtExecutor stmtExecutor;
+    private List<ErrorTabletInfo> errorTabletInfos = new ArrayList<>();
+    private List<TabletCommitInfo> commitInfos = new ArrayList<>();
 
-    StmtExecutor stmtExecutor;
+    // max save task num, do we need to config it?
+    private static final int MAX_SAVE_TASK_NUM = 100;
 
-    ConnectContext ctx;
+    /**
+     * load job type
+     */
+    public enum LoadType {
+        BULK,
+        SPARK,
+        LOCAL_FILE,
+        UNKNOWN
 
-    @SerializedName("tis")
-    ConcurrentLinkedQueue<Long> historyTaskIdList;
+    }
+
+    public enum Priority {
+        HIGH(0),
+        NORMAL(1),
+        LOW(2);
+
+        Priority(int value) {
+            this.value = value;
+        }
+
+        private final int value;
+
+        public int getValue() {
+            return value;
+        }
+    }
+
+    public InsertJob(String jobName,
+                     JobStatus jobStatus,
+                     String dbName,
+                     String comment,
+                     UserIdentity createUser,
+                     JobExecutionConfiguration jobConfig,
+                     Long createTimeMs,
+                     String executeSql) {
+        super(getNextJobId(), jobName, jobStatus, dbName, comment, createUser,
+                jobConfig, createTimeMs, executeSql, null);
+        this.dbId = ConnectContext.get().getCurrentDbId();
+    }
+
+    public InsertJob(ConnectContext ctx,
+                      StmtExecutor executor,
+                      String labelName,
+                      List<InsertIntoTableCommand> plans,
+                      Set<String> sinkTableNames,
+                      Map<String, String> properties,
+                      String comment,
+                      JobExecutionConfiguration jobConfig) {
+        super(getNextJobId(), labelName, JobStatus.RUNNING, null,
+                comment, ctx.getCurrentUserIdentity(), jobConfig);
+        this.ctx = ctx;
+        this.plans = plans;
+        this.stmtExecutor = executor;
+        this.dbId = ctx.getCurrentDbId();
+        this.labelName = labelName;
+        this.tableNames = sinkTableNames;
+        this.properties = properties;
+        // TODO: not support other type yet
+        this.loadType = InsertJob.LoadType.BULK;
+    }
 
     @Override
-    public List<InsertTask> createTasks(TaskType taskType, Map taskContext) {
-        //nothing need to do in insert job
-        InsertTask task = new InsertTask(null, getCurrentDbName(), getExecuteSql(), getCreateUser());
-        task.setJobId(getJobId());
-        task.setTaskType(taskType);
-        task.setTaskId(Env.getCurrentEnv().getNextId());
-        ArrayList<InsertTask> tasks = new ArrayList<>();
-        tasks.add(task);
-        super.initTasks(tasks);
-        recordTask(task.getTaskId());
-        return tasks;
+    public List<InsertTask> createTasks(TaskType taskType, Map<Object, Object> taskContext) {
+        if (plans.isEmpty()) {
+            InsertTask task = new InsertTask(labelName, getCurrentDbName(), getExecuteSql(), getCreateUser());
+            idToTasks.put(task.getTaskId(), task);
+            recordTask(task.getTaskId());
+        } else {
+            // use for load stmt
+            for (InsertIntoTableCommand logicalPlan : plans) {
+                if (!logicalPlan.getLabelName().isPresent()) {
+                    throw new IllegalArgumentException("Load plan need label name.");
+                }
+                InsertTask task = new InsertTask(logicalPlan, ctx, stmtExecutor, loadStatistic);
+                idToTasks.put(task.getTaskId(), task);
+                recordTask(task.getTaskId());
+            }
+        }
+        initTasks(idToTasks.values(), taskType);
+        return new ArrayList<>(idToTasks.values());
     }
 
     public void recordTask(long id) {
@@ -116,7 +252,6 @@ public void recordTask(long id) {
         if (historyTaskIdList.size() >= Config.max_persistence_task_count) {
             historyTaskIdList.poll();
         }
-        Env.getCurrentEnv().getEditLog().logUpdateJob(this);
     }
 
     @Override
@@ -125,23 +260,27 @@ public void cancelTaskById(long taskId) throws JobException {
     }
 
     @Override
-    public boolean isReadyForScheduling(Map taskContext) {
-        return CollectionUtils.isEmpty(getRunningTasks());
+    public void cancelAllTasks() throws JobException {
+        try {
+            checkAuth("CANCEL LOAD");
+            super.cancelAllTasks();
+            this.failMsg = new FailMsg(FailMsg.CancelType.USER_CANCEL, "user cancel");
+        } catch (DdlException e) {
+            throw new JobException(e);
+        }
     }
 
-
     @Override
-    public void cancelAllTasks() throws JobException {
-        super.cancelAllTasks();
+    public boolean isReadyForScheduling(Map<Object, Object> taskContext) {
+        return CollectionUtils.isEmpty(getRunningTasks());
     }
 
-
     @Override
     protected void checkJobParamsInternal() {
-        if (command == null && StringUtils.isBlank(getExecuteSql())) {
+        if (plans.isEmpty() && StringUtils.isBlank(getExecuteSql())) {
             throw new IllegalArgumentException("command or sql is null,must be set");
         }
-        if (null != command && !getJobConfig().getExecuteType().equals(JobExecuteType.INSTANT)) {
+        if (!plans.isEmpty() && !getJobConfig().getExecuteType().equals(JobExecuteType.INSTANT)) {
             throw new IllegalArgumentException("command must be null when executeType is not instant");
         }
     }
@@ -153,27 +292,22 @@ public List<InsertTask> queryTasks() {
         }
         //TODO it's will be refactor, we will storage task info in job inner and query from it
         List<Long> taskIdList = new ArrayList<>(this.historyTaskIdList);
+
         Collections.reverse(taskIdList);
-        List<LoadJob> loadJobs = Env.getCurrentEnv().getLoadManager().queryLoadJobsByJobIds(taskIdList);
-        if (CollectionUtils.isEmpty(loadJobs)) {
+        return queryLoadTasksByTaskIds(taskIdList);
+    }
+
+    public List<InsertTask> queryLoadTasksByTaskIds(List<Long> taskIdList) {
+        if (taskIdList.isEmpty()) {
             return new ArrayList<>();
         }
-        List<InsertTask> tasks = new ArrayList<>();
-        loadJobs.forEach(loadJob -> {
-            InsertTask task;
-            try {
-                task = new InsertTask(loadJob.getLabel(), loadJob.getDb().getFullName(), null, getCreateUser());
-                task.setCreateTimeMs(loadJob.getCreateTimestamp());
-            } catch (MetaNotFoundException e) {
-                log.warn("load job not found, job id is {}", loadJob.getId());
-                return;
+        List<InsertTask> jobs = new ArrayList<>();
+        taskIdList.forEach(id -> {
+            if (null != idToTasks.get(id)) {
+                jobs.add(idToTasks.get(id));
             }
-            task.setJobId(getJobId());
-            task.setTaskId(loadJob.getId());
-            task.setLoadJob(loadJob);
-            tasks.add(task);
         });
-        return tasks;
+        return jobs;
     }
 
     @Override
@@ -193,6 +327,12 @@ public ShowResultSetMetaData getTaskMetaData() {
 
     @Override
     public void onTaskFail(InsertTask task) {
+        try {
+            updateJobStatus(JobStatus.STOPPED);
+            this.failMsg = new FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL, task.getErrMsg());
+        } catch (JobException e) {
+            throw new RuntimeException(e);
+        }
         getRunningTasks().remove(task);
     }
 
@@ -203,7 +343,129 @@ public void onTaskSuccess(InsertTask task) throws JobException {
 
     @Override
     public List<String> getShowInfo() {
-        return super.getCommonShowInfo();
+        try {
+            // check auth
+            checkAuth("SHOW LOAD");
+            List<String> jobInfo = Lists.newArrayList();
+            // jobId
+            jobInfo.add(getJobId().toString());
+            // label
+            if (StringUtils.isEmpty(getLabelName())) {
+                jobInfo.add(FeConstants.null_string);
+            } else {
+                jobInfo.add(getLabelName());
+            }
+            // state
+            if (getJobStatus() == JobStatus.STOPPED) {
+                jobInfo.add("CANCELLED");
+            } else {
+                jobInfo.add(getJobStatus().name());
+            }
+
+            // progress
+            String progress = Env.getCurrentProgressManager().getProgressInfo(String.valueOf(getJobId()));
+            switch (getJobStatus()) {
+                case RUNNING:
+                    if (isPending()) {
+                        jobInfo.add("ETL:0%; LOAD:0%");
+                    } else {
+                        jobInfo.add("ETL:100%; LOAD:" + progress + "%");
+                    }
+                    break;
+                case FINISHED:
+                    jobInfo.add("ETL:100%; LOAD:100%");
+                    break;
+                case STOPPED:
+                default:
+                    jobInfo.add("ETL:N/A; LOAD:N/A");
+                    break;
+            }
+            // type
+            jobInfo.add(loadType.name());
+
+            // etl info
+            if (loadStatistic.getCounters().size() == 0) {
+                jobInfo.add(FeConstants.null_string);
+            } else {
+                jobInfo.add(Joiner.on("; ").withKeyValueSeparator("=").join(loadStatistic.getCounters()));
+            }
+
+            // task info
+            jobInfo.add("cluster:" + getResourceName() + "; timeout(s):" + getTimeout()
+                    + "; max_filter_ratio:" + getMaxFilterRatio() + "; priority:" + getPriority());
+            // error msg
+            if (failMsg == null) {
+                jobInfo.add(FeConstants.null_string);
+            } else {
+                jobInfo.add("type:" + failMsg.getCancelType() + "; msg:" + failMsg.getMsg());
+            }
+
+            // create time
+            jobInfo.add(TimeUtils.longToTimeString(getCreateTimeMs()));
+            // etl start time
+            jobInfo.add(TimeUtils.longToTimeString(getStartTimeMs()));
+            // etl end time
+            jobInfo.add(TimeUtils.longToTimeString(getStartTimeMs()));
+            // load start time
+            jobInfo.add(TimeUtils.longToTimeString(getStartTimeMs()));
+            // load end time
+            jobInfo.add(TimeUtils.longToTimeString(getFinishTimeMs()));
+            // tracking urls
+            List<String> trackingUrl = idToTasks.values().stream()
+                    .map(task -> {
+                        if (StringUtils.isNotEmpty(task.getTrackingUrl())) {
+                            return task.getTrackingUrl();
+                        } else {
+                            return FeConstants.null_string;
+                        }
+                    })
+                    .collect(Collectors.toList());
+            if (trackingUrl.isEmpty()) {
+                jobInfo.add(FeConstants.null_string);
+            } else {
+                jobInfo.add(trackingUrl.toString());
+            }
+            // job details
+            jobInfo.add(loadStatistic.toJson());
+            // transaction id
+            jobInfo.add(String.valueOf(0));
+            // error tablets
+            jobInfo.add(errorTabletsToJson());
+            // user, some load job may not have user info
+            if (getCreateUser() == null || getCreateUser().getQualifiedUser() == null) {
+                jobInfo.add(FeConstants.null_string);
+            } else {
+                jobInfo.add(getCreateUser().getQualifiedUser());
+            }
+            // comment
+            jobInfo.add(getComment());
+            return jobInfo;
+        } catch (DdlException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private String getPriority() {
+        return properties.getOrDefault(LoadStmt.PRIORITY, Priority.NORMAL.name());
+    }
+
+    public double getMaxFilterRatio() {
+        return Double.parseDouble(properties.getOrDefault(LoadStmt.MAX_FILTER_RATIO_PROPERTY, "0.0"));
+    }
+
+    public long getTimeout() {
+        if (properties.containsKey(LoadStmt.TIMEOUT_PROPERTY)) {
+            return Long.parseLong(properties.get(LoadStmt.TIMEOUT_PROPERTY));
+        }
+        return Config.broker_load_default_timeout_second;
+    }
+
+
+    public static InsertJob readFields(DataInput in) throws IOException {
+        String jsonJob = Text.readString(in);
+        InsertJob job = GsonUtils.GSON.fromJson(jsonJob, InsertJob.class);
+        job.setRunningTasks(new ArrayList<>());
+        return job;
     }
 
     @Override
@@ -211,19 +473,129 @@ public void write(DataOutput out) throws IOException {
         Text.writeString(out, GsonUtils.GSON.toJson(this));
     }
 
-    private static final ShowResultSetMetaData TASK_META_DATA =
-            ShowResultSetMetaData.builder()
-                    .addColumn(new Column("TaskId", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("Label", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("Status", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("EtlInfo", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("TaskInfo", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("ErrorMsg", ScalarType.createVarchar(20)))
+    public String errorTabletsToJson() {
+        Map<Long, String> map = new HashMap<>();
+        errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load)
+                .forEach(p -> map.put(p.getTabletId(), p.getMsg()));
+        Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+        return gson.toJson(map);
+    }
 
-                    .addColumn(new Column("CreateTimeMs", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("FinishTimeMs", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("TrackingUrl", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("LoadStatistic", ScalarType.createVarchar(20)))
-                    .addColumn(new Column("User", ScalarType.createVarchar(20)))
-                    .build();
+    public void updateLoadingStatus(Long beId, TUniqueId loadId, TUniqueId fragmentId, long scannedRows,
+                                    long scannedBytes, boolean isDone) {
+        loadStatistic.updateLoadProgress(beId, loadId, fragmentId, scannedRows, scannedBytes, isDone);
+        progress = (int) ((double) finishedTaskIds.size() / idToTasks.size() * 100);
+        if (progress == 100) {
+            progress = 99;
+        }
+    }
+
+    private void checkAuth(String command) throws DdlException {
+        if (authorizationInfo == null) {
+            // use the old method to check priv
+            checkAuthWithoutAuthInfo(command);
+            return;
+        }
+        if (!Env.getCurrentEnv().getAccessManager().checkPrivByAuthInfo(ConnectContext.get(), authorizationInfo,
+                PrivPredicate.LOAD)) {
+            ErrorReport.reportDdlException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
+                    Privilege.LOAD_PRIV);
+        }
+    }
+
+    /**
+     * This method is compatible with old load job without authorization info
+     * If db or table name could not be found by id, it will throw the NOT_EXISTS_ERROR
+     *
+     * @throws DdlException
+     */
+    private void checkAuthWithoutAuthInfo(String command) throws DdlException {
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId);
+        // check auth
+        if (tableNames == null || tableNames.isEmpty()) {
+            // forward compatibility
+            if (!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(), db.getFullName(),
+                    PrivPredicate.LOAD)) {
+                ErrorReport.reportDdlException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
+                        Privilege.LOAD_PRIV);
+            }
+        } else {
+            for (String tblName : tableNames) {
+                if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), db.getFullName(),
+                        tblName, PrivPredicate.LOAD)) {
+                    ErrorReport.reportDdlException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR,
+                            command,
+                            ConnectContext.get().getQualifiedUser(),
+                            ConnectContext.get().getRemoteIP(), db.getFullName() + ": " + tblName);
+                }
+            }
+        }
+    }
+
+    public void unprotectReadEndOperation(InsertJob replayLog) {
+        setJobStatus(replayLog.getJobStatus());
+        progress = replayLog.getProgress();
+        setStartTimeMs(replayLog.getStartTimeMs());
+        setFinishTimeMs(replayLog.getFinishTimeMs());
+        failMsg = replayLog.getFailMsg();
+    }
+
+    public String getResourceName() {
+        // TODO: get tvf param from tvf relation
+        return "N/A";
+    }
+
+    public boolean isRunning() {
+        return getJobStatus() != JobStatus.FINISHED;
+    }
+
+    public boolean isPending() {
+        return getJobStatus() != JobStatus.FINISHED;
+    }
+
+    public boolean isCancelled() {
+        return getJobStatus() == JobStatus.STOPPED;
+    }
+
+    @Override
+    public void onRegister() throws JobException {
+        try {
+            if (StringUtils.isNotEmpty(labelName)) {
+                Env.getCurrentEnv().getLabelProcessor().addJob(this);
+            }
+        } catch (LabelAlreadyUsedException e) {
+            throw new JobException(e);
+        }
+    }
+
+    @Override
+    public void onUnRegister() throws JobException {
+        // TODO: need record cancelled jobs in order to show cancelled job
+        // Env.getCurrentEnv().getLabelProcessor().removeJob(getDbId(), getLabelName());
+    }
+
+    @Override
+    public void onReplayCreate() throws JobException {
+        JobExecutionConfiguration jobConfig = new JobExecutionConfiguration();
+        jobConfig.setExecuteType(JobExecuteType.INSTANT);
+        setJobConfig(jobConfig);
+        onRegister();
+        checkJobParams();
+        log.info(new LogBuilder(LogKey.LOAD_JOB, getJobId()).add("msg", "replay create load job").build());
+    }
+
+    @Override
+    public void onReplayEnd(AbstractJob<?, Map<Object, Object>> replayJob) throws JobException {
+        if (!(replayJob instanceof InsertJob)) {
+            return;
+        }
+        InsertJob insertJob = (InsertJob) replayJob;
+        unprotectReadEndOperation(insertJob);
+        log.info(new LogBuilder(LogKey.LOAD_JOB,
+                insertJob.getJobId()).add("operation", insertJob).add("msg", "replay end load job").build());
+    }
+
+    public int getProgress() {
+        return progress;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
index 7e9889502678e0..e85e7a1b02708d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertTask.java
@@ -25,7 +25,8 @@
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.job.exception.JobException;
 import org.apache.doris.job.task.AbstractTask;
-import org.apache.doris.load.loadv2.LoadJob;
+import org.apache.doris.load.FailMsg;
+import org.apache.doris.load.loadv2.LoadStatistic;
 import org.apache.doris.nereids.parser.NereidsParser;
 import org.apache.doris.nereids.trees.plans.commands.InsertIntoTableCommand;
 import org.apache.doris.qe.ConnectContext;
@@ -34,12 +35,12 @@
 import org.apache.doris.thrift.TRow;
 import org.apache.doris.thrift.TUniqueId;
 
-import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.Optional;
 import java.util.UUID;
@@ -53,8 +54,6 @@ public class InsertTask extends AbstractTask {
             new Column("JobId", ScalarType.createStringType()),
             new Column("Label", ScalarType.createStringType()),
             new Column("Status", ScalarType.createStringType()),
-            new Column("EtlInfo", ScalarType.createStringType()),
-            new Column("TaskInfo", ScalarType.createStringType()),
             new Column("ErrorMsg", ScalarType.createStringType()),
             new Column("CreateTimeMs", ScalarType.createStringType()),
             new Column("FinishTimeMs", ScalarType.createStringType()),
@@ -73,30 +72,69 @@ public class InsertTask extends AbstractTask {
     }
 
     private String labelName;
-
     private InsertIntoTableCommand command;
-
     private StmtExecutor stmtExecutor;
-
     private ConnectContext ctx;
-
     private String sql;
-
     private String currentDb;
-
     private UserIdentity userIdentity;
-
+    private LoadStatistic loadStatistic;
     private AtomicBoolean isCanceled = new AtomicBoolean(false);
-
     private AtomicBoolean isFinished = new AtomicBoolean(false);
-
     private static final String LABEL_SPLITTER = "_";
 
+    private FailMsg failMsg;
+    @Getter
+    private String trackingUrl;
 
     @Getter
     @Setter
-    private LoadJob loadJob;
+    private InsertJob jobInfo;
+    private TaskType taskType = TaskType.PENDING;
+    private MergeType mergeType = MergeType.APPEND;
+
+    /**
+     * task merge type
+     */
+    enum MergeType {
+        MERGE,
+        APPEND,
+        DELETE
+    }
+
+    /**
+     * task type
+     */
+    enum TaskType {
+        UNKNOWN, // this is only for ISSUE #2354
+        PENDING,
+        LOADING,
+        FINISHED,
+        FAILED,
+        CANCELLED
+    }
+
+    public InsertTask(InsertIntoTableCommand insertInto,
+                      ConnectContext ctx, StmtExecutor executor, LoadStatistic statistic) {
+        this(insertInto.getLabelName().get(), insertInto, ctx, executor, statistic);
+    }
+
+    public InsertTask(String labelName, String currentDb, String sql, UserIdentity userIdentity) {
+        this.labelName = labelName;
+        this.sql = sql;
+        this.currentDb = currentDb;
+        this.userIdentity = userIdentity;
+    }
 
+    public InsertTask(String labelName, InsertIntoTableCommand insertInto,
+                       ConnectContext ctx, StmtExecutor executor, LoadStatistic statistic) {
+        this.labelName = labelName;
+        this.command = insertInto;
+        this.userIdentity = ctx.getCurrentUserIdentity();
+        this.ctx = ctx;
+        this.stmtExecutor = executor;
+        this.loadStatistic = statistic;
+    }
 
     @Override
     public void before() throws JobException {
@@ -109,15 +147,19 @@ public void before() throws JobException {
         ctx.setCurrentUserIdentity(userIdentity);
         ctx.getState().reset();
         ctx.setThreadLocalInfo();
-        ctx.setDatabase(currentDb);
+        if (StringUtils.isNotEmpty(currentDb)) {
+            ctx.setDatabase(currentDb);
+        }
         TUniqueId queryId = generateQueryId(UUID.randomUUID().toString());
         ctx.getSessionVariable().enableFallbackToOriginalPlanner = false;
         stmtExecutor = new StmtExecutor(ctx, (String) null);
         ctx.setQueryId(queryId);
-        NereidsParser parser = new NereidsParser();
-        this.command = (InsertIntoTableCommand) parser.parseSingle(sql);
-        this.command.setLabelName(Optional.of(getJobId() + LABEL_SPLITTER + getTaskId()));
-        this.command.setJobId(getTaskId());
+        if (StringUtils.isNotEmpty(sql)) {
+            NereidsParser parser = new NereidsParser();
+            this.command = (InsertIntoTableCommand) parser.parseSingle(sql);
+            this.command.setLabelName(Optional.of(getJobId() + LABEL_SPLITTER + getTaskId()));
+            this.command.setJobId(getTaskId());
+        }
 
         super.before();
 
@@ -128,14 +170,6 @@ protected TUniqueId generateQueryId(String taskIdString) {
         return new TUniqueId(taskId.getMostSignificantBits(), taskId.getLeastSignificantBits());
     }
 
-    public InsertTask(String labelName, String currentDb, String sql, UserIdentity userIdentity) {
-        this.labelName = labelName;
-        this.sql = sql;
-        this.currentDb = currentDb;
-        this.userIdentity = userIdentity;
-
-    }
-
     @Override
     public void run() throws JobException {
         try {
@@ -143,7 +177,7 @@ public void run() throws JobException {
                 log.info("task has been canceled, task id is {}", getTaskId());
                 return;
             }
-            command.run(ctx, stmtExecutor);
+            command.runWithUpdateInfo(ctx, stmtExecutor, loadStatistic);
         } catch (Exception e) {
             log.warn("execute insert task error, job id is {}, task id is {},sql is {}", getJobId(),
                     getTaskId(), sql, e);
@@ -178,42 +212,28 @@ public void cancel() throws JobException {
     @Override
     public TRow getTvfInfo() {
         TRow trow = new TRow();
-        if (loadJob == null) {
+        if (jobInfo == null) {
             // if task not start, load job is null,return pending task show info
             return getPendingTaskTVFInfo();
         }
-        trow.addToColumnValue(new TCell().setStringVal(String.valueOf(loadJob.getId())));
+        trow.addToColumnValue(new TCell().setStringVal(String.valueOf(jobInfo.getJobId())));
         trow.addToColumnValue(new TCell().setStringVal(String.valueOf(getJobId())));
-        trow.addToColumnValue(new TCell().setStringVal(loadJob.getLabel()));
-        trow.addToColumnValue(new TCell().setStringVal(loadJob.getState().name()));
-        // etl info
-        String etlInfo = FeConstants.null_string;
-        if (!loadJob.getLoadingStatus().getCounters().isEmpty()) {
-            etlInfo = Joiner.on("; ").withKeyValueSeparator("=").join(loadJob.getLoadingStatus().getCounters());
-        }
-        trow.addToColumnValue(new TCell().setStringVal(etlInfo));
-
-        // task info
-        String taskInfo = "cluster:" + loadJob.getResourceName() + "; timeout(s):" + loadJob.getTimeout()
-                + "; max_filter_ratio:" + loadJob.getMaxFilterRatio() + "; priority:" + loadJob.getPriority();
-        trow.addToColumnValue(new TCell().setStringVal(taskInfo));
-
+        trow.addToColumnValue(new TCell().setStringVal(labelName));
+        trow.addToColumnValue(new TCell().setStringVal(jobInfo.getJobStatus().name()));
         // err msg
         String errMsg = FeConstants.null_string;
-        if (loadJob.getFailMsg() != null) {
-            errMsg = "type:" + loadJob.getFailMsg().getCancelType() + "; msg:" + loadJob.getFailMsg().getMsg();
+        if (failMsg != null) {
+            errMsg = "type:" + failMsg.getCancelType() + "; msg:" + failMsg.getMsg();
         }
         trow.addToColumnValue(new TCell().setStringVal(errMsg));
-
         // create time
-        trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(loadJob.getCreateTimestamp())));
-
+        trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(jobInfo.getCreateTimeMs())));
         // load end time
-        trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(loadJob.getFinishTimestamp())));
+        trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(jobInfo.getFinishTimeMs())));
         // tracking url
-        trow.addToColumnValue(new TCell().setStringVal(loadJob.getLoadingStatus().getTrackingUrl()));
-        trow.addToColumnValue(new TCell().setStringVal(loadJob.getLoadStatistic().toJson()));
-        trow.addToColumnValue(new TCell().setStringVal(loadJob.getUserInfo().getQualifiedUser()));
+        trow.addToColumnValue(new TCell().setStringVal(trackingUrl));
+        trow.addToColumnValue(new TCell().setStringVal(loadStatistic.toJson()));
+        trow.addToColumnValue(new TCell().setStringVal(userIdentity.getQualifiedUser()));
         return trow;
     }
 
@@ -225,8 +245,6 @@ private TRow getPendingTaskTVFInfo() {
         trow.addToColumnValue(new TCell().setStringVal(getJobId() + LABEL_SPLITTER + getTaskId()));
         trow.addToColumnValue(new TCell().setStringVal(getStatus().name()));
         trow.addToColumnValue(new TCell().setStringVal(FeConstants.null_string));
-        trow.addToColumnValue(new TCell().setStringVal(FeConstants.null_string));
-        trow.addToColumnValue(new TCell().setStringVal(FeConstants.null_string));
         trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(getCreateTimeMs())));
         trow.addToColumnValue(new TCell().setStringVal(FeConstants.null_string));
         trow.addToColumnValue(new TCell().setStringVal(FeConstants.null_string));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java
index c500e6932959d9..669a2806a543d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java
@@ -125,7 +125,7 @@ public List<MTMVTask> createTasks(TaskType taskType, MTMVTaskContext taskContext
         task.setTaskType(taskType);
         ArrayList<MTMVTask> tasks = new ArrayList<>();
         tasks.add(task);
-        super.initTasks(tasks);
+        super.initTasks(tasks, taskType);
         return tasks;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
index 776af152c69d88..814d6b773ad1ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
@@ -17,7 +17,15 @@
 
 package org.apache.doris.job.manager;
 
+import org.apache.doris.analysis.CancelLoadStmt;
+import org.apache.doris.analysis.CompoundPredicate;
+import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.CaseSensibility;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.PatternMatcher;
+import org.apache.doris.common.PatternMatcherWrapper;
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.common.util.LogBuilder;
 import org.apache.doris.common.util.LogKey;
@@ -26,45 +34,86 @@
 import org.apache.doris.job.common.JobType;
 import org.apache.doris.job.common.TaskType;
 import org.apache.doris.job.exception.JobException;
+import org.apache.doris.job.extensions.insert.InsertJob;
 import org.apache.doris.job.scheduler.JobScheduler;
 import org.apache.doris.job.task.AbstractTask;
+import org.apache.doris.load.loadv2.JobState;
 
+import com.google.common.collect.Lists;
 import lombok.extern.log4j.Log4j2;
 import org.apache.commons.lang3.StringUtils;
 
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 
 @Log4j2
 public class JobManager<T extends AbstractJob<?, C>, C> implements Writable {
 
-
     private final ConcurrentHashMap<Long, T> jobMap = new ConcurrentHashMap<>(32);
 
-    private JobScheduler jobScheduler;
+    private JobScheduler<T, C> jobScheduler;
+
+    // lock for job
+    // lock is private and must use after db lock
+    private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
+
+    private void readLock() {
+        lock.readLock().lock();
+    }
+
+    private void readUnlock() {
+        lock.readLock().unlock();
+    }
+
+    private void writeLock() {
+        lock.writeLock().lock();
+    }
+
+    private void writeUnlock() {
+        lock.writeLock().unlock();
+    }
 
     public void start() {
-        jobScheduler = new JobScheduler(jobMap);
+        jobScheduler = new JobScheduler<T, C>(jobMap);
         jobScheduler.start();
     }
 
+
+    /**
+     * get running job
+     *
+     * @param jobId id
+     * @return running job
+     */
+    public T getJob(long jobId) {
+        return jobMap.get(jobId);
+    }
+
     public void registerJob(T job) throws JobException {
-        job.checkJobParams();
-        checkJobNameExist(job.getJobName());
-        if (jobMap.get(job.getJobId()) != null) {
-            throw new JobException("job id exist, jobId:" + job.getJobId());
+        writeLock();
+        try {
+            job.onRegister();
+            job.checkJobParams();
+            checkJobNameExist(job.getJobName());
+            if (jobMap.get(job.getJobId()) != null) {
+                throw new JobException("job id exist, jobId:" + job.getJobId());
+            }
+            jobMap.put(job.getJobId(), job);
+            //check its need to scheduler
+            jobScheduler.scheduleOneJob(job);
+            job.logCreateOperation();
+        } finally {
+            writeUnlock();
         }
-        Env.getCurrentEnv().getEditLog().logCreateJob(job);
-        jobMap.put(job.getJobId(), job);
-        //check its need to scheduler
-        jobScheduler.scheduleOneJob(job);
     }
 
-
     private void checkJobNameExist(String jobName) throws JobException {
         if (jobMap.values().stream().anyMatch(a -> a.getJobName().equals(jobName))) {
             throw new JobException("job name exist, jobName:" + jobName);
@@ -72,11 +121,17 @@ private void checkJobNameExist(String jobName) throws JobException {
     }
 
     public void unregisterJob(Long jobId) throws JobException {
-        checkJobExist(jobId);
-        jobMap.get(jobId).setJobStatus(JobStatus.STOPPED);
-        jobMap.get(jobId).cancelAllTasks();
-        Env.getCurrentEnv().getEditLog().logDeleteJob(jobMap.get(jobId));
-        jobMap.remove(jobId);
+        writeLock();
+        try {
+            checkJobExist(jobId);
+            jobMap.get(jobId).setJobStatus(JobStatus.STOPPED);
+            jobMap.get(jobId).cancelAllTasks();
+            jobMap.get(jobId).logFinalOperation();
+            jobMap.get(jobId).onUnRegister();
+            jobMap.remove(jobId);
+        } finally {
+            writeUnlock();
+        }
     }
 
     public void unregisterJob(String jobName) throws JobException {
@@ -95,7 +150,7 @@ public void unregisterJob(String jobName) throws JobException {
     public void alterJobStatus(Long jobId, JobStatus status) throws JobException {
         checkJobExist(jobId);
         jobMap.get(jobId).updateJobStatus(status);
-        Env.getCurrentEnv().getEditLog().logUpdateJob(jobMap.get(jobId));
+        jobMap.get(jobId).logUpdateOperation();
     }
 
     public void alterJobStatus(String jobName, JobStatus jobStatus) throws JobException {
@@ -169,13 +224,12 @@ public void triggerJob(long jobId, C context) throws JobException {
         jobScheduler.schedulerInstantJob(jobMap.get(jobId), TaskType.MANUAL, context);
     }
 
-    public void replayCreateJob(T job) {
+    public void replayCreateJob(T job) throws JobException {
         if (jobMap.containsKey(job.getJobId())) {
             return;
         }
         jobMap.putIfAbsent(job.getJobId(), job);
-        log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId())
-                .add("msg", "replay create scheduler job").build());
+        job.onReplayCreate();
     }
 
     /**
@@ -187,13 +241,12 @@ public void replayUpdateJob(T job) {
                 .add("msg", "replay update scheduler job").build());
     }
 
-    public void replayDeleteJob(T job) {
-        if (null == jobMap.get(job.getJobId())) {
+    public void replayEndJob(T replayJob) throws JobException {
+        T job = jobMap.get(replayJob.getJobId());
+        if (null == job) {
             return;
         }
-        jobMap.remove(job.getJobId());
-        log.info(new LogBuilder(LogKey.SCHEDULER_JOB, job.getJobId())
-                .add("msg", "replay delete scheduler job").build());
+        job.onReplayEnd(replayJob);
     }
 
     public void cancelTaskById(String jobName, Long taskId) throws JobException {
@@ -236,4 +289,135 @@ public T getJob(Long jobId) {
         return jobMap.get(jobId);
     }
 
+
+    /**
+     * get load info by db
+     *
+     * @param dbId          db id
+     * @param dbName        db name
+     * @param labelValue    label name
+     * @param accurateMatch accurate match
+     * @param jobState      state
+     * @return load infos
+     * @throws AnalysisException ex
+     */
+    public List<List<Comparable>> getLoadJobInfosByDb(long dbId, String dbName,
+                                                      String labelValue,
+                                                      boolean accurateMatch,
+                                                      JobState jobState) throws AnalysisException {
+        LinkedList<List<Comparable>> loadJobInfos = new LinkedList<>();
+        if (!Env.getCurrentEnv().getLabelProcessor().existJobs(dbId)) {
+            return loadJobInfos;
+        }
+        readLock();
+        try {
+            List<InsertJob> loadJobList = Env.getCurrentEnv().getLabelProcessor()
+                    .filterJobs(dbId, labelValue, accurateMatch);
+            // check state
+            for (InsertJob loadJob : loadJobList) {
+                try {
+                    if (jobState != null && !validState(jobState, loadJob)) {
+                        continue;
+                    }
+                    // add load job info, convert String list to Comparable list
+                    loadJobInfos.add(new ArrayList<>(loadJob.getShowInfo()));
+                } catch (RuntimeException e) {
+                    // ignore this load job
+                    log.warn("get load job info failed. job id: {}", loadJob.getJobId(), e);
+                }
+            }
+            return loadJobInfos;
+        } finally {
+            readUnlock();
+        }
+    }
+
+    private static boolean validState(JobState jobState, InsertJob loadJob) {
+        JobStatus status = loadJob.getJobStatus();
+        switch (status) {
+            case RUNNING:
+                return jobState == JobState.PENDING || jobState == JobState.ETL
+                        || jobState == JobState.LOADING || jobState == JobState.COMMITTED;
+            case STOPPED:
+                return jobState == JobState.CANCELLED;
+            case FINISHED:
+                return jobState == JobState.FINISHED;
+            default:
+                return false;
+        }
+    }
+
+    public void cancelLoadJob(CancelLoadStmt cs)
+            throws JobException, AnalysisException, DdlException {
+        String dbName = cs.getDbName();
+        String label = cs.getLabel();
+        String state = cs.getState();
+        CompoundPredicate.Operator operator = cs.getOperator();
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbName);
+        // List of load jobs waiting to be cancelled
+        List<InsertJob> unfinishedLoadJob;
+        readLock();
+        try {
+            List<InsertJob> loadJobs = Env.getCurrentEnv().getLabelProcessor().getJobs(db);
+            List<InsertJob> matchLoadJobs = Lists.newArrayList();
+            addNeedCancelLoadJob(label, state, operator, loadJobs, matchLoadJobs);
+            if (matchLoadJobs.isEmpty()) {
+                throw new JobException("Load job does not exist");
+            }
+            // check state here
+            unfinishedLoadJob =
+                    matchLoadJobs.stream().filter(InsertJob::isRunning)
+                            .collect(Collectors.toList());
+            if (unfinishedLoadJob.isEmpty()) {
+                throw new JobException("There is no uncompleted job");
+            }
+        } finally {
+            readUnlock();
+        }
+        for (InsertJob loadJob : unfinishedLoadJob) {
+            try {
+                unregisterJob(loadJob.getJobId());
+            } catch (JobException e) {
+                log.warn("Fail to cancel job, its label: {}", loadJob.getLabelName());
+            }
+        }
+    }
+
+    private static void addNeedCancelLoadJob(String label, String state,
+                                             CompoundPredicate.Operator operator, List<InsertJob> loadJobs,
+                                             List<InsertJob> matchLoadJobs)
+            throws AnalysisException {
+        PatternMatcher matcher = PatternMatcherWrapper.createMysqlPattern(label,
+                CaseSensibility.LABEL.getCaseSensibility());
+        matchLoadJobs.addAll(
+                loadJobs.stream()
+                        .filter(job -> !job.isCancelled())
+                        .filter(job -> {
+                            if (operator != null) {
+                                // compound
+                                boolean labelFilter =
+                                        label.contains("%") ? matcher.match(job.getLabelName())
+                                                : job.getLabelName().equalsIgnoreCase(label);
+                                boolean stateFilter = job.getJobStatus().name().equalsIgnoreCase(state);
+                                return CompoundPredicate.Operator.AND.equals(operator) ? labelFilter && stateFilter :
+                                        labelFilter || stateFilter;
+                            }
+                            if (StringUtils.isNotEmpty(label)) {
+                                return label.contains("%") ? matcher.match(job.getLabelName())
+                                        : job.getLabelName().equalsIgnoreCase(label);
+                            }
+                            if (StringUtils.isNotEmpty(state)) {
+                                return job.getJobStatus().name().equalsIgnoreCase(state);
+                            }
+                            return false;
+                        }).collect(Collectors.toList())
+        );
+    }
+    //    public void updateJobProgress(Long jobId, Long beId, TUniqueId loadId, TUniqueId fragmentId, long scannedRows,
+    //                                  long scannedBytes, boolean isDone) {
+    //        AbstractJob job = jobMap.get(jobId);
+    //        if (job != null) {
+    //            job.updateLoadingStatus(beId, loadId, fragmentId, scannedRows, scannedBytes, isDone);
+    //        }
+    //    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/task/AbstractTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/task/AbstractTask.java
index 7327183e95eda2..71f6ff1c4f7eeb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/task/AbstractTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/task/AbstractTask.java
@@ -27,6 +27,7 @@
 import com.google.gson.annotations.SerializedName;
 import lombok.Data;
 import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.RandomUtils;
 
 @Data
 @Log4j2
@@ -52,6 +53,15 @@ public abstract class AbstractTask implements Task {
     @SerializedName(value = "emg")
     private String errMsg;
 
+    public AbstractTask() {
+        taskId = getNextTaskId();
+    }
+
+    private static long getNextTaskId() {
+        // do not use Env.getNextId(), just generate id without logging
+        return System.nanoTime() + RandomUtils.nextInt();
+    }
+
     @Override
     public void onFail(String msg) throws JobException {
         status = TaskStatus.FAILED;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
index 996c3ccbb32f6f..fc2f6fca9c5cf6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportJob.java
@@ -73,6 +73,7 @@
 import org.apache.doris.qe.OriginStatement;
 import org.apache.doris.qe.SessionVariable;
 import org.apache.doris.scheduler.exception.JobException;
+import org.apache.doris.scheduler.executor.TransientTaskExecutor;
 import org.apache.doris.thrift.TNetworkAddress;
 
 import com.google.common.base.Preconditions;
@@ -198,7 +199,7 @@ public class ExportJob implements Writable {
 
     private List<ExportTaskExecutor> jobExecutorList;
 
-    private ConcurrentHashMap<Long, ExportTaskExecutor> taskIdToExecutor = new ConcurrentHashMap<>();
+    private ConcurrentHashMap<Long, TransientTaskExecutor> taskIdToExecutor = new ConcurrentHashMap<>();
 
     private Integer finishedTaskCount = 0;
     private List<List<OutfileInfo>> allOutfileInfo = Lists.newArrayList();
@@ -380,6 +381,10 @@ private StatementBase generateLogicalPlanAdapter(LogicalPlan outfileLogicalPlan)
         return statementBase;
     }
 
+    public List<? extends TransientTaskExecutor> getTaskExecutors() {
+        return jobExecutorList;
+    }
+
     private void generateExportJobExecutor() {
         jobExecutorList = Lists.newArrayList();
         for (List<StatementBase> selectStmts : selectStmtListPerParallel) {
@@ -607,7 +612,7 @@ private void cancelExportTask(ExportFailMsg.CancelType type, String msg) throws
         // we need cancel all task
         taskIdToExecutor.keySet().forEach(id -> {
             try {
-                Env.getCurrentEnv().getExportTaskRegister().cancelTask(id);
+                Env.getCurrentEnv().getTransientTaskManager().cancelMemoryTask(id);
             } catch (JobException e) {
                 LOG.warn("cancel export task {} exception: {}", id, e);
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
index ae7a175b896a63..90be7f9f10e871 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportMgr.java
@@ -59,23 +59,26 @@
 
 public class ExportMgr {
     private static final Logger LOG = LogManager.getLogger(ExportJob.class);
-
-    // lock for export job
-    // lock is private and must use after db lock
-    private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
-
     private Map<Long, ExportJob> exportIdToJob = Maps.newHashMap(); // exportJobId to exportJob
     // dbid -> <label -> job>
     private Map<Long, Map<String, Long>> dbTolabelToExportJobId = Maps.newHashMap();
 
+    // lock for export job
+    // lock is private and must use after db lock
+    private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
+
     public ExportMgr() {
     }
 
-    public void readLock() {
+    public List<ExportJob> getJobs() {
+        return Lists.newArrayList(exportIdToJob.values());
+    }
+
+    private void readLock() {
         lock.readLock().lock();
     }
 
-    public void readUnlock() {
+    private void readUnlock() {
         lock.readLock().unlock();
     }
 
@@ -87,10 +90,6 @@ private void writeUnlock() {
         lock.writeLock().unlock();
     }
 
-    public List<ExportJob> getJobs() {
-        return Lists.newArrayList(exportIdToJob.values());
-    }
-
     public void addExportJobAndRegisterTask(ExportJob job) throws Exception {
         long jobId = Env.getCurrentEnv().getNextId();
         job.setId(jobId);
@@ -101,9 +100,8 @@ public void addExportJobAndRegisterTask(ExportJob job) throws Exception {
                 throw new LabelAlreadyUsedException(job.getLabel());
             }
             unprotectAddJob(job);
-            job.getJobExecutorList().forEach(executor -> {
-                Long taskId = Env.getCurrentEnv().getExportTaskRegister().registerTask(executor);
-                executor.setTaskId(taskId);
+            job.getTaskExecutors().forEach(executor -> {
+                Long taskId = Env.getCurrentEnv().getTransientTaskManager().addMemoryTask(executor);
                 job.getTaskIdToExecutor().put(taskId, executor);
             });
             Env.getCurrentEnv().getEditLog().logExportCreate(job);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/ExportTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/load/ExportTaskExecutor.java
index 6960efc7e677bf..26e873afd80b43 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/ExportTaskExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/ExportTaskExecutor.java
@@ -66,12 +66,18 @@ public class ExportTaskExecutor implements TransientTaskExecutor {
     private AtomicBoolean isFinished;
 
     ExportTaskExecutor(List<StatementBase> selectStmtLists, ExportJob exportJob) {
+        this.taskId = UUID.randomUUID().getMostSignificantBits();
         this.selectStmtLists = selectStmtLists;
         this.exportJob = exportJob;
         this.isCanceled = new AtomicBoolean(false);
         this.isFinished = new AtomicBoolean(false);
     }
 
+    @Override
+    public Long getId() {
+        return taskId;
+    }
+
     @Override
     public void execute() throws JobException {
         if (isCanceled.get()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
index 6e5d56aa55ef40..073f47a66cbfd3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
@@ -37,7 +37,6 @@
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
-import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.common.util.LogBuilder;
 import org.apache.doris.common.util.LogKey;
 import org.apache.doris.common.util.TimeUtils;
@@ -61,11 +60,9 @@
 import org.apache.doris.transaction.TransactionState;
 
 import com.google.common.base.Joiner;
-import com.google.common.collect.HashBasedTable;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import com.google.common.collect.Table;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.annotations.SerializedName;
@@ -138,108 +135,6 @@ public abstract class LoadJob extends AbstractTxnStateChangeCallback implements
     protected String comment = "";
 
 
-    public static class LoadStatistic {
-        // number of rows processed on BE, this number will be updated periodically by query report.
-        // A load job may has several load tasks(queries), and each task has several fragments.
-        // each fragment will report independently.
-        // load task id -> fragment id -> rows count
-        private Table<TUniqueId, TUniqueId, Long> counterTbl = HashBasedTable.create();
-
-        // load task id -> fragment id -> load bytes
-        private Table<TUniqueId, TUniqueId, Long> loadBytes = HashBasedTable.create();
-
-        // load task id -> unfinished backend id list
-        private Map<TUniqueId, List<Long>> unfinishedBackendIds = Maps.newHashMap();
-        // load task id -> all backend id list
-        private Map<TUniqueId, List<Long>> allBackendIds = Maps.newHashMap();
-
-        // number of file to be loaded
-        public int fileNum = 0;
-        public long totalFileSizeB = 0;
-
-        // init the statistic of specified load task
-        public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds, List<Long> relatedBackendIds) {
-            counterTbl.rowMap().remove(loadId);
-            for (TUniqueId fragId : fragmentIds) {
-                counterTbl.put(loadId, fragId, 0L);
-            }
-            loadBytes.rowMap().remove(loadId);
-            for (TUniqueId fragId : fragmentIds) {
-                loadBytes.put(loadId, fragId, 0L);
-            }
-            allBackendIds.put(loadId, relatedBackendIds);
-            // need to get a copy of relatedBackendIds, so that when we modify the "relatedBackendIds" in
-            // allBackendIds, the list in unfinishedBackendIds will not be changed.
-            unfinishedBackendIds.put(loadId, Lists.newArrayList(relatedBackendIds));
-        }
-
-        public synchronized void removeLoad(TUniqueId loadId) {
-            counterTbl.rowMap().remove(loadId);
-            loadBytes.rowMap().remove(loadId);
-            unfinishedBackendIds.remove(loadId);
-            allBackendIds.remove(loadId);
-        }
-
-        public synchronized void updateLoadProgress(long backendId, TUniqueId loadId, TUniqueId fragmentId,
-                                                    long rows, long bytes, boolean isDone) {
-            if (counterTbl.contains(loadId, fragmentId)) {
-                counterTbl.put(loadId, fragmentId, rows);
-            }
-
-            if (loadBytes.contains(loadId, fragmentId)) {
-                loadBytes.put(loadId, fragmentId, bytes);
-            }
-            if (isDone && unfinishedBackendIds.containsKey(loadId)) {
-                unfinishedBackendIds.get(loadId).remove(backendId);
-            }
-        }
-
-        public synchronized long getScannedRows() {
-            long total = 0;
-            for (long rows : counterTbl.values()) {
-                total += rows;
-            }
-            return total;
-        }
-
-        public synchronized long getLoadBytes() {
-            long total = 0;
-            for (long bytes : loadBytes.values()) {
-                total += bytes;
-            }
-            return total;
-        }
-
-        public synchronized String toJson() {
-            long total = 0;
-            for (long rows : counterTbl.values()) {
-                total += rows;
-            }
-            long totalBytes = 0;
-            for (long bytes : loadBytes.values()) {
-                totalBytes += bytes;
-            }
-
-            Map<String, Object> details = Maps.newHashMap();
-            details.put("ScannedRows", total);
-            details.put("LoadBytes", totalBytes);
-            details.put("FileNumber", fileNum);
-            details.put("FileSize", totalFileSizeB);
-            details.put("TaskNumber", counterTbl.rowMap().size());
-            details.put("Unfinished backends", getPrintableMap(unfinishedBackendIds));
-            details.put("All backends", getPrintableMap(allBackendIds));
-            Gson gson = new Gson();
-            return gson.toJson(details);
-        }
-
-        private Map<String, List<Long>> getPrintableMap(Map<TUniqueId, List<Long>> map) {
-            Map<String, List<Long>> newMap = Maps.newHashMap();
-            for (Map.Entry<TUniqueId, List<Long>> entry : map.entrySet()) {
-                newMap.put(DebugUtil.printId(entry.getKey()), entry.getValue());
-            }
-            return newMap;
-        }
-    }
 
     public LoadJob(EtlJobType jobType) {
         this.jobType = jobType;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index 1fb83ae762dd1b..72bc61239cf8d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -278,7 +278,7 @@ public static void addNeedCancelLoadJob(CancelLoadStmt stmt, List<LoadJob> loadJ
     public void cancelLoadJob(CancelLoadStmt stmt) throws DdlException, AnalysisException {
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(stmt.getDbName());
         // List of load jobs waiting to be cancelled
-        List<LoadJob> uncompletedLoadJob = Lists.newArrayList();
+        List<LoadJob> unfinishedLoadJob;
         readLock();
         try {
             Map<String, List<LoadJob>> labelToLoadJobs = dbIdToLabelToLoadJobs.get(db.getId());
@@ -293,15 +293,15 @@ public void cancelLoadJob(CancelLoadStmt stmt) throws DdlException, AnalysisExce
                 throw new DdlException("Load job does not exist");
             }
             // check state here
-            uncompletedLoadJob =
+            unfinishedLoadJob =
                     matchLoadJobs.stream().filter(entity -> !entity.isTxnDone()).collect(Collectors.toList());
-            if (uncompletedLoadJob.isEmpty()) {
+            if (unfinishedLoadJob.isEmpty()) {
                 throw new DdlException("There is no uncompleted job");
             }
         } finally {
             readUnlock();
         }
-        for (LoadJob loadJob : uncompletedLoadJob) {
+        for (LoadJob loadJob : unfinishedLoadJob) {
             try {
                 loadJob.cancelJob(new FailMsg(FailMsg.CancelType.USER_CANCEL, "user cancel"));
             } catch (DdlException e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java
new file mode 100644
index 00000000000000..0c65aa27851e4e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadStatistic.java
@@ -0,0 +1,142 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.load.loadv2;
+
+import org.apache.doris.common.util.DebugUtil;
+import org.apache.doris.thrift.TUniqueId;
+
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Table;
+import com.google.gson.Gson;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class LoadStatistic {
+    // number of rows processed on BE, this number will be updated periodically by query report.
+    // A load job may has several load tasks(queries), and each task has several fragments.
+    // each fragment will report independently.
+    // load task id -> fragment id -> rows count
+    private Table<TUniqueId, TUniqueId, Long> counterTbl = HashBasedTable.create();
+
+    // load task id -> fragment id -> load bytes
+    private Table<TUniqueId, TUniqueId, Long> loadBytes = HashBasedTable.create();
+
+    // load task id -> unfinished backend id list
+    private Map<TUniqueId, List<Long>> unfinishedBackendIds = Maps.newHashMap();
+    // load task id -> all backend id list
+    private Map<TUniqueId, List<Long>> allBackendIds = Maps.newHashMap();
+
+    private Map<String, String> counters = new HashMap<>();
+
+    // number of file to be loaded
+    public int fileNum = 0;
+    public long totalFileSizeB = 0;
+
+    // init the statistic of specified load task
+    public synchronized void initLoad(TUniqueId loadId, Set<TUniqueId> fragmentIds, List<Long> relatedBackendIds) {
+        counterTbl.rowMap().remove(loadId);
+        for (TUniqueId fragId : fragmentIds) {
+            counterTbl.put(loadId, fragId, 0L);
+        }
+        loadBytes.rowMap().remove(loadId);
+        for (TUniqueId fragId : fragmentIds) {
+            loadBytes.put(loadId, fragId, 0L);
+        }
+        allBackendIds.put(loadId, relatedBackendIds);
+        // need to get a copy of relatedBackendIds, so that when we modify the "relatedBackendIds" in
+        // allBackendIds, the list in unfinishedBackendIds will not be changed.
+        unfinishedBackendIds.put(loadId, Lists.newArrayList(relatedBackendIds));
+    }
+
+    public synchronized void removeLoad(TUniqueId loadId) {
+        counterTbl.rowMap().remove(loadId);
+        loadBytes.rowMap().remove(loadId);
+        unfinishedBackendIds.remove(loadId);
+        allBackendIds.remove(loadId);
+    }
+
+    public synchronized void updateLoadProgress(long backendId, TUniqueId loadId, TUniqueId fragmentId,
+                                                long rows, long bytes, boolean isDone) {
+        if (counterTbl.contains(loadId, fragmentId)) {
+            counterTbl.put(loadId, fragmentId, rows);
+        }
+
+        if (loadBytes.contains(loadId, fragmentId)) {
+            loadBytes.put(loadId, fragmentId, bytes);
+        }
+        if (isDone && unfinishedBackendIds.containsKey(loadId)) {
+            unfinishedBackendIds.get(loadId).remove(backendId);
+        }
+    }
+
+    public synchronized long getScannedRows() {
+        long total = 0;
+        for (long rows : counterTbl.values()) {
+            total += rows;
+        }
+        return total;
+    }
+
+    public synchronized long getLoadBytes() {
+        long total = 0;
+        for (long bytes : loadBytes.values()) {
+            total += bytes;
+        }
+        return total;
+    }
+
+    public Map<String, String> getCounters() {
+        // TODO: add extra statistics to counters
+        return counters;
+    }
+
+    public synchronized String toJson() {
+        long total = 0;
+        for (long rows : counterTbl.values()) {
+            total += rows;
+        }
+        long totalBytes = 0;
+        for (long bytes : loadBytes.values()) {
+            totalBytes += bytes;
+        }
+
+        Map<String, Object> details = Maps.newHashMap();
+        details.put("ScannedRows", total);
+        details.put("LoadBytes", totalBytes);
+        details.put("FileNumber", fileNum);
+        details.put("FileSize", totalFileSizeB);
+        details.put("TaskNumber", counterTbl.rowMap().size());
+        details.put("Unfinished backends", getPrintableMap(unfinishedBackendIds));
+        details.put("All backends", getPrintableMap(allBackendIds));
+        Gson gson = new Gson();
+        return gson.toJson(details);
+    }
+
+    private Map<String, List<Long>> getPrintableMap(Map<TUniqueId, List<Long>> map) {
+        Map<String, List<Long>> newMap = Maps.newHashMap();
+        for (Map.Entry<TUniqueId, List<Long>> entry : map.entrySet()) {
+            newMap.put(DebugUtil.printId(entry.getKey()), entry.getValue());
+        }
+        return newMap;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java
new file mode 100644
index 00000000000000..88454eaecdc4e0
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/load/LabelProcessor.java
@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.jobs.load;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.CaseSensibility;
+import org.apache.doris.common.LabelAlreadyUsedException;
+import org.apache.doris.common.PatternMatcher;
+import org.apache.doris.common.PatternMatcherWrapper;
+import org.apache.doris.job.exception.JobException;
+import org.apache.doris.job.extensions.insert.InsertJob;
+
+import com.google.common.base.Strings;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+
+/**
+ * label manager
+ */
+public class LabelProcessor {
+    private final Map<Long, Map<String, List<InsertJob>>> dbIdToLabelToLoadJobs = new ConcurrentHashMap<>();
+    private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);
+
+    private void readLock() {
+        lock.readLock().lock();
+    }
+
+    private void readUnlock() {
+        lock.readLock().unlock();
+    }
+
+    private void writeLock() {
+        lock.writeLock().lock();
+    }
+
+    private void writeUnlock() {
+        lock.writeLock().unlock();
+    }
+
+    /**
+     * get jobs with label
+     * @param db db
+     * @return jobs
+     * @throws JobException e
+     */
+    public List<InsertJob> getJobs(Database db) throws JobException {
+        readLock();
+        try {
+            Map<String, List<InsertJob>> labelToLoadJobs = dbIdToLabelToLoadJobs.get(db.getId());
+            if (labelToLoadJobs == null) {
+                throw new JobException("Load job does not exist");
+            }
+            return labelToLoadJobs.values().stream().flatMap(Collection::stream).collect(Collectors.toList());
+        } finally {
+            readUnlock();
+        }
+    }
+
+    /**
+     * add job with label
+     *
+     * @param job job with label
+     * @throws LabelAlreadyUsedException e
+     */
+    public void addJob(InsertJob job) throws LabelAlreadyUsedException {
+        writeLock();
+        try {
+            Map<String, List<InsertJob>> labelToLoadJobs;
+            if (!dbIdToLabelToLoadJobs.containsKey(job.getDbId())) {
+                labelToLoadJobs = new ConcurrentHashMap<>();
+                dbIdToLabelToLoadJobs.put(job.getDbId(), labelToLoadJobs);
+            }
+            labelToLoadJobs = dbIdToLabelToLoadJobs.get(job.getDbId());
+            if (labelToLoadJobs.containsKey(job.getLabelName())) {
+                throw new LabelAlreadyUsedException(job.getLabelName());
+            } else {
+                labelToLoadJobs.put(job.getLabelName(), new ArrayList<>());
+            }
+            labelToLoadJobs.get(job.getLabelName()).add(job);
+        } finally {
+            writeUnlock();
+        }
+    }
+
+    /**
+     * support remove label job
+     * @param dbId db id
+     * @param labelName label name
+     */
+    public void removeJob(long dbId, String labelName) {
+        writeLock();
+        try {
+            if (dbIdToLabelToLoadJobs.containsKey(dbId)) {
+                dbIdToLabelToLoadJobs.get(dbId).remove(labelName);
+            }
+        } finally {
+            writeUnlock();
+        }
+    }
+
+    public void cleanOldLabels() throws JobException {
+        // TODO: remain this method to implement label cleaner
+    }
+
+    /**
+     * filterJobs with label and support quick match label
+     * @param dbId dbId
+     * @param labelValue label
+     * @param accurateMatch direct find label from map
+     * @return jobs with label
+     */
+    public List<InsertJob> filterJobs(long dbId, String labelValue, boolean accurateMatch)
+            throws AnalysisException {
+        List<InsertJob> loadJobList = new ArrayList<>();
+        readLock();
+        try {
+            Map<String, List<InsertJob>> labelToLoadJobs = this.dbIdToLabelToLoadJobs.get(dbId);
+            if (Strings.isNullOrEmpty(labelValue)) {
+                loadJobList.addAll(
+                        labelToLoadJobs.values().stream().flatMap(Collection::stream).collect(Collectors.toList()));
+            } else {
+                // check label value
+                if (accurateMatch) {
+                    if (!labelToLoadJobs.containsKey(labelValue)) {
+                        return new ArrayList<>();
+                    }
+                    loadJobList.addAll(labelToLoadJobs.get(labelValue));
+                } else {
+                    // non-accurate match
+                    PatternMatcher matcher =
+                            PatternMatcherWrapper.createMysqlPattern(labelValue,
+                                    CaseSensibility.LABEL.getCaseSensibility());
+                    for (Map.Entry<String, List<InsertJob>> entry : labelToLoadJobs.entrySet()) {
+                        if (matcher.match(entry.getKey())) {
+                            loadJobList.addAll(entry.getValue());
+                        }
+                    }
+                }
+            }
+        } finally {
+            readUnlock();
+        }
+        return loadJobList;
+    }
+
+    /**
+     * check jobs in database
+     * @param dbId dbId
+     * @return has jobs
+     */
+    public boolean existJobs(long dbId) {
+        readLock();
+        try {
+            return dbIdToLabelToLoadJobs.containsKey(dbId);
+        } finally {
+            readUnlock();
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 0d24a8b9377048..95ae97192c9b37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -945,7 +945,8 @@ public LogicalPlan visitLoad(DorisParser.LoadContext ctx) {
             LoadTask.MergeType mergeType = ddc.mergeType() == null ? LoadTask.MergeType.APPEND
                         : LoadTask.MergeType.valueOf(ddc.mergeType().getText());
 
-            Optional<String> fileFormat = ddc.format == null ? Optional.empty() : Optional.of(ddc.format.getText());
+            Optional<String> fileFormat = ddc.format == null ? Optional.empty()
+                    : Optional.of(visitIdentifierOrStringLiteral(ddc.format));
             Optional<String> separator = ddc.separator == null ? Optional.empty() : Optional.of(ddc.separator.getText()
                         .substring(1, ddc.separator.getText().length() - 1));
             Optional<String> comma = ddc.comma == null ? Optional.empty() : Optional.of(ddc.comma.getText()
@@ -970,7 +971,7 @@ public LogicalPlan visitLoad(DorisParser.LoadContext ctx) {
         }
         String labelName = ctx.lableName.getText();
         Map<String, String> properties = visitPropertyItemList(ctx.properties);
-        String commentSpec = ctx.commentSpec() == null ? "" : ctx.commentSpec().STRING_LITERAL().getText();
+        String commentSpec = ctx.commentSpec() == null ? "''" : ctx.commentSpec().STRING_LITERAL().getText();
         String comment =
                 LogicalPlanBuilderAssistant.escapeBackSlash(commentSpec.substring(1, commentSpec.length() - 1));
         return new LoadCommand(labelName, dataDescriptions.build(), bulkDesc, properties, comment);
@@ -1044,6 +1045,15 @@ public String visitIdentifierOrText(IdentifierOrTextContext ctx) {
         }
     }
 
+    @Override
+    public String visitIdentifierOrStringLiteral(DorisParser.IdentifierOrStringLiteralContext ctx) {
+        if (ctx.STRING_LITERAL() != null) {
+            return ctx.STRING_LITERAL().getText().substring(1, ctx.STRING_LITERAL().getText().length() - 1);
+        } else {
+            return ctx.identifier().getText();
+        }
+    }
+
     @Override
     public UserIdentity visitUserIdentify(UserIdentifyContext ctx) {
         String user = visitIdentifierOrText(ctx.user);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertExecutor.java
index 000fd477b317ac..90fb95325850b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertExecutor.java
@@ -318,11 +318,14 @@ public void executeSingleInsertTransaction(StmtExecutor executor, long jobId) {
             if (0 != jobId) {
                 etlJobType = EtlJobType.INSERT_JOB;
             }
-            ctx.getEnv().getLoadManager()
-                    .recordFinishedLoadJob(labelName, txnId, database.getFullName(),
-                            table.getId(),
-                            etlJobType, createAt, throwable == null ? "" : throwable.getMessage(),
-                            coordinator.getTrackingUrl(), userIdentity, jobId);
+            if (!Config.enable_nereids_load) {
+                // just record for loadv2 here
+                ctx.getEnv().getLoadManager()
+                        .recordFinishedLoadJob(labelName, txnId, database.getFullName(),
+                                table.getId(),
+                                etlJobType, createAt, throwable == null ? "" : throwable.getMessage(),
+                                coordinator.getTrackingUrl(), userIdentity, jobId);
+            }
         } catch (MetaNotFoundException e) {
             LOG.warn("Record info of insert load with error {}", e.getMessage(), e);
             errMsg = "Record info of insert load with error " + e.getMessage();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
index 506637d189f7ac..11a7926e5dd8b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
@@ -27,6 +27,7 @@
 import org.apache.doris.common.ErrorReport;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.ProfileManager.ProfileType;
+import org.apache.doris.load.loadv2.LoadStatistic;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.nereids.NereidsPlanner;
 import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -95,6 +96,10 @@ public InsertIntoTableCommand(LogicalPlan logicalQuery, Optional<String> labelNa
         this.allowAutoPartition = true;
     }
 
+    public Optional<String> getLabelName() {
+        return labelName;
+    }
+
     public void setLabelName(Optional<String> labelName) {
         this.labelName = labelName;
     }
@@ -109,6 +114,16 @@ public void setAllowAutoPartition(boolean allowAutoPartition) {
 
     @Override
     public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        runInternal(ctx, executor);
+    }
+
+    public void runWithUpdateInfo(ConnectContext ctx, StmtExecutor executor,
+                                  LoadStatistic loadStatistic) throws Exception {
+        // TODO: add coordinator statistic
+        runInternal(ctx, executor);
+    }
+
+    private void runInternal(ConnectContext ctx, StmtExecutor executor) throws Exception {
         if (!ctx.getSessionVariable().isEnableNereidsDML()) {
             try {
                 ctx.getSessionVariable().enableFallbackToOriginalPlannerOnce();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
index ed4133ffd08efe..ce61a3bbb3ffba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/LoadCommand.java
@@ -18,16 +18,20 @@
 package org.apache.doris.nereids.trees.plans.commands;
 
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
 import org.apache.doris.common.ErrorCode;
 import org.apache.doris.common.NereidsException;
-import org.apache.doris.common.UserException;
 import org.apache.doris.common.profile.Profile;
 import org.apache.doris.common.util.FileFormatConstants;
 import org.apache.doris.common.util.FileFormatUtils;
 import org.apache.doris.datasource.property.constants.S3Properties;
+import org.apache.doris.job.base.JobExecuteType;
+import org.apache.doris.job.base.JobExecutionConfiguration;
+import org.apache.doris.job.extensions.insert.InsertJob;
 import org.apache.doris.load.loadv2.LoadTask;
 import org.apache.doris.nereids.analyzer.UnboundAlias;
 import org.apache.doris.nereids.analyzer.UnboundSlot;
@@ -53,7 +57,6 @@
 import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.RelationUtil;
 import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.QueryStateException;
 import org.apache.doris.qe.StmtExecutor;
 import org.apache.doris.tablefunction.HdfsTableValuedFunction;
 import org.apache.doris.tablefunction.S3TableValuedFunction;
@@ -83,10 +86,11 @@ public class LoadCommand extends Command implements ForwardWithSync {
 
     private final String labelName;
     private final BulkStorageDesc bulkStorageDesc;
+    private final Set<String> sinkTableNames = new HashSet<>();
     private final List<BulkLoadDataDesc> sourceInfos;
     private final Map<String, String> properties;
     private final String comment;
-    private final List<LogicalPlan> plans = new ArrayList<>();
+    private List<InsertIntoTableCommand> plans = new ArrayList<>();
     private Profile profile;
 
     /**
@@ -119,15 +123,19 @@ public List<LogicalPlan> parseToInsertIntoPlan(ConnectContext ctx) throws Analys
 
     @Override
     public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
-        // TODO: begin txn form multi insert sql
-        /* this.profile = new Profile("Query", ctx.getSessionVariable().enableProfile);
-          profile.getSummaryProfile().setQueryBeginTime();
-          for (BulkLoadDataDesc dataDesc : sourceInfos) {
-               plans.add(new InsertIntoTableCommand(completeQueryPlan(ctx, dataDesc), Optional.of(labelName), false));
-          }
-          profile.getSummaryProfile().setQueryPlanFinishTime();
-         * executeInsertStmtPlan(ctx, executor, plans);  */
-        throw new AnalysisException("Fallback to legacy planner temporary.");
+        if (!Config.enable_nereids_load) {
+            throw new AnalysisException("Fallback to legacy planner temporary.");
+        }
+        this.profile = new Profile("Query", ctx.getSessionVariable().enableProfile);
+        profile.getSummaryProfile().setQueryBeginTime();
+        if (sourceInfos.size() == 1) {
+            plans = ImmutableList.of(new InsertIntoTableCommand(completeQueryPlan(ctx, sourceInfos.get(0)),
+                    Optional.of(labelName)));
+        } else {
+            throw new AnalysisException("Multi insert into statements are unsupported.");
+        }
+        profile.getSummaryProfile().setQueryPlanFinishTime();
+        submitInsertStmtPlan(ctx, executor, plans);
     }
 
     private LogicalPlan completeQueryPlan(ConnectContext ctx, BulkLoadDataDesc dataDesc)
@@ -151,6 +159,7 @@ private LogicalPlan completeQueryPlan(ConnectContext ctx, BulkLoadDataDesc dataD
         boolean scanAllTvfCol = (tvfProjects.get(0) instanceof UnboundStar);
 
         OlapTable olapTable = getOlapTable(ctx, dataDesc);
+        sinkTableNames.add(olapTable.getName());
         List<Column> olapSchema = olapTable.getBaseSchema();
         // map column index to mapping expr
         Map<String, Expression> mappingExpressions = dataDesc.getColumnMappings();
@@ -471,20 +480,14 @@ private static Map<String, String> getTvfProperties(BulkLoadDataDesc dataDesc, B
         return tvfProperties;
     }
 
-    private void executeInsertStmtPlan(ConnectContext ctx, StmtExecutor executor, List<InsertIntoTableCommand> plans) {
+    private void submitInsertStmtPlan(ConnectContext ctx, StmtExecutor executor, List<InsertIntoTableCommand> plans) {
         try {
-            for (LogicalPlan logicalPlan : plans) {
-                ((Command) logicalPlan).run(ctx, executor);
-            }
-        } catch (QueryStateException e) {
-            ctx.setState(e.getQueryState());
-            throw new NereidsException("Command process failed", new AnalysisException(e.getMessage(), e));
-        } catch (UserException e) {
-            // Return message to info client what happened.
-            ctx.getState().setError(e.getMysqlErrorCode(), e.getMessage());
-            throw new NereidsException("Command process failed", new AnalysisException(e.getMessage(), e));
+            JobExecutionConfiguration jobExecutionConfiguration = new JobExecutionConfiguration();
+            jobExecutionConfiguration.setExecuteType(JobExecuteType.INSTANT);
+            InsertJob jobExecutor = new InsertJob(ctx, executor, labelName, plans,
+                    sinkTableNames, properties, comment, jobExecutionConfiguration);
+            Env.getCurrentEnv().getJobManager().registerJob(jobExecutor);
         } catch (Exception e) {
-            // Maybe our bug
             ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, e.getMessage());
             throw new NereidsException("Command process failed.", new AnalysisException(e.getMessage(), e));
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index ea8292712af6ba..f78f417e66f545 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -65,7 +65,6 @@
 import org.apache.doris.load.ExportJobState;
 import org.apache.doris.load.ExportJobStateTransfer;
 import org.apache.doris.load.ExportMgr;
-import org.apache.doris.load.LoadJob;
 import org.apache.doris.load.StreamLoadRecordMgr.FetchStreamLoadRecord;
 import org.apache.doris.load.loadv2.LoadJob.LoadJobStateUpdateInfo;
 import org.apache.doris.load.loadv2.LoadJobFinalOperation;
@@ -673,7 +672,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
                 }
                 case OperationType.OP_DELETE_SCHEDULER_JOB: {
                     AbstractJob job = (AbstractJob) journal.getData();
-                    Env.getCurrentEnv().getJobManager().replayDeleteJob(job);
+                    Env.getCurrentEnv().getJobManager().replayEndJob(job);
                     break;
                 }
                 /*case OperationType.OP_CREATE_SCHEDULER_TASK: {
@@ -1347,30 +1346,6 @@ public void logRecoverTable(RecoverInfo info) {
         logEdit(OperationType.OP_RECOVER_TABLE, info);
     }
 
-    public void logLoadStart(LoadJob job) {
-        logEdit(OperationType.OP_LOAD_START, job);
-    }
-
-    public void logLoadEtl(LoadJob job) {
-        logEdit(OperationType.OP_LOAD_ETL, job);
-    }
-
-    public void logLoadLoading(LoadJob job) {
-        logEdit(OperationType.OP_LOAD_LOADING, job);
-    }
-
-    public void logLoadQuorum(LoadJob job) {
-        logEdit(OperationType.OP_LOAD_QUORUM, job);
-    }
-
-    public void logLoadCancel(LoadJob job) {
-        logEdit(OperationType.OP_LOAD_CANCEL, job);
-    }
-
-    public void logLoadDone(LoadJob job) {
-        logEdit(OperationType.OP_LOAD_DONE, job);
-    }
-
     public void logDropRollup(DropInfo info) {
         logEdit(OperationType.OP_DROP_ROLLUP, info);
     }
@@ -1644,7 +1619,7 @@ public void logUpdateJob(AbstractJob job) {
         logEdit(OperationType.OP_UPDATE_SCHEDULER_JOB, job);
     }
 
-    public void logDeleteJob(AbstractJob job) {
+    public void logEndJob(AbstractJob job) {
         logEdit(OperationType.OP_DELETE_SCHEDULER_JOB, job);
     }
 
@@ -1985,6 +1960,7 @@ public void logDeleteTableStats(TableStatsDeletionLog log) {
 
     public void logAlterMTMV(AlterMTMV log) {
         logEdit(OperationType.OP_ALTER_MTMV, log);
+
     }
 
     public String getNotReadyReason() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
index 47682cd74f04e4..51dc1eb73199d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java
@@ -174,7 +174,10 @@ public static void execute(Env env, DdlStmt ddlStmt) throws Exception {
         } else if (ddlStmt instanceof CancelExportStmt) {
             env.getExportMgr().cancelExportJob((CancelExportStmt) ddlStmt);
         } else if (ddlStmt instanceof CancelLoadStmt) {
-            env.getLoadManager().cancelLoadJob((CancelLoadStmt) ddlStmt);
+            CancelLoadStmt cs = (CancelLoadStmt) ddlStmt;
+            // cancel all
+            env.getJobManager().cancelLoadJob(cs);
+            env.getLoadManager().cancelLoadJob(cs);
         } else if (ddlStmt instanceof CreateRoutineLoadStmt) {
             env.getRoutineLoadManager().createRoutineLoadJob((CreateRoutineLoadStmt) ddlStmt);
         } else if (ddlStmt instanceof PauseRoutineLoadStmt) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index b7ee282d24880d..0a895276909b04 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -183,6 +183,7 @@
 import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.datasource.HMSExternalCatalog;
 import org.apache.doris.datasource.MaxComputeExternalCatalog;
+import org.apache.doris.job.manager.JobManager;
 import org.apache.doris.load.DeleteHandler;
 import org.apache.doris.load.ExportJobState;
 import org.apache.doris.load.ExportMgr;
@@ -1169,32 +1170,36 @@ private void handleShowLoad() throws AnalysisException {
         Env env = ctx.getEnv();
         DatabaseIf db = ctx.getCurrentCatalog().getDbOrAnalysisException(showStmt.getDbName());
         long dbId = db.getId();
-
+        List<List<Comparable>> loadInfos;
         // combine the List<LoadInfo> of load(v1) and loadManager(v2)
         Load load = env.getLoadInstance();
-        List<List<Comparable>> loadInfos = load.getLoadJobInfosByDb(dbId, db.getFullName(), showStmt.getLabelValue(),
+        loadInfos = load.getLoadJobInfosByDb(dbId, db.getFullName(), showStmt.getLabelValue(),
                 showStmt.isAccurateMatch(), showStmt.getStates());
         Set<String> statesValue = showStmt.getStates() == null ? null : showStmt.getStates().stream()
                 .map(entity -> entity.name())
                 .collect(Collectors.toSet());
         loadInfos.addAll(env.getLoadManager()
                 .getLoadJobInfosByDb(dbId, showStmt.getLabelValue(), showStmt.isAccurateMatch(), statesValue));
+        // add the nerieds load info
+        JobManager loadMgr = env.getJobManager();
+        loadInfos.addAll(loadMgr.getLoadJobInfosByDb(dbId, db.getFullName(), showStmt.getLabelValue(),
+                showStmt.isAccurateMatch(), showStmt.getStateV2()));
 
         // order the result of List<LoadInfo> by orderByPairs in show stmt
         List<OrderByPair> orderByPairs = showStmt.getOrderByPairs();
-        ListComparator<List<Comparable>> comparator = null;
+        ListComparator<List<Comparable>> comparator;
         if (orderByPairs != null) {
             OrderByPair[] orderByPairArr = new OrderByPair[orderByPairs.size()];
-            comparator = new ListComparator<List<Comparable>>(orderByPairs.toArray(orderByPairArr));
+            comparator = new ListComparator<>(orderByPairs.toArray(orderByPairArr));
         } else {
             // sort by id asc
-            comparator = new ListComparator<List<Comparable>>(0);
+            comparator = new ListComparator<>(0);
         }
         Collections.sort(loadInfos, comparator);
 
         List<List<String>> rows = Lists.newArrayList();
         for (List<Comparable> loadInfo : loadInfos) {
-            List<String> oneInfo = new ArrayList<String>(loadInfo.size());
+            List<String> oneInfo = new ArrayList<>(loadInfo.size());
 
             // replace QUORUM_FINISHED -> FINISHED
             if (loadInfo.get(LoadProcDir.STATE_INDEX).equals(JobState.QUORUM_FINISHED.name())) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/TransientTaskExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/TransientTaskExecutor.java
index f297753bbd8cde..f4205a6f4bb6d5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/TransientTaskExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/executor/TransientTaskExecutor.java
@@ -34,5 +34,7 @@ public interface TransientTaskExecutor {
      * Cancel the memory task.
      */
     void cancel() throws JobException;
+
+    Long getId();
 }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java
index 9602b19ca2ae6b..5f94fb5d998cfc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/manager/TransientTaskManager.java
@@ -23,7 +23,6 @@
 
 import lombok.Setter;
 
-import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class TransientTaskManager {
@@ -51,8 +50,8 @@ public TransientTaskExecutor getMemoryTaskExecutor(Long taskId) {
         return taskExecutorMap.get(taskId);
     }
 
-    public Long registerMemoryTask(TransientTaskExecutor executor) {
-        Long taskId = UUID.randomUUID().getMostSignificantBits();
+    public Long addMemoryTask(TransientTaskExecutor executor) {
+        Long taskId = executor.getId();
         taskExecutorMap.put(taskId, executor);
         disruptor.tryPublishTask(taskId);
         return taskId;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/ExportTaskRegister.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/ExportTaskRegister.java
index 46d89c96ebc7e7..0241f57fea0026 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/ExportTaskRegister.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/registry/ExportTaskRegister.java
@@ -30,7 +30,7 @@ public ExportTaskRegister(TransientTaskManager transientTaskManager) {
 
     @Override
     public Long registerTask(TransientTaskExecutor executor) {
-        return transientTaskManager.registerMemoryTask(executor);
+        return transientTaskManager.addMemoryTask(executor);
     }
 
     @Override
diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
index a6f151141b921f..b41bf50738b150 100644
--- a/regression-test/suites/job_p0/test_base_insert_job.groovy
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -111,7 +111,7 @@ suite("test_base_insert_job") {
      """
 
     Thread.sleep(25000)
-    def onceJob = sql """select id,ExecuteSql from jobs("type"="insert") where Name='${jobName}'"""
+    def onceJob = sql """ select id,ExecuteSql from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='ONE_TIME' """
     assert onceJob.size() == 1
     def onceJobId= onceJob.get(0).get(0);
     def onceJobSql= onceJob.get(0).get(1);

From c8ed14f11c6da93909af57ca53781d07f3986365 Mon Sep 17 00:00:00 2001
From: plat1ko <platonekosama@gmail.com>
Date: Tue, 26 Dec 2023 13:37:30 +0800
Subject: [PATCH 030/299] [enhance](tablet) Reduce log in tablet meta (#28719)

---
 be/src/olap/tablet_meta.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index c46378198decfe..a67103a8939264 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -284,7 +284,6 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
     }
 
     init_from_pb(tablet_meta_pb);
-    LOG(INFO) << "init tablet meta from pb: " << tablet_meta_pb.ShortDebugString();
 }
 
 TabletMeta::TabletMeta(const TabletMeta& b)

From f7a624de38bdea75fdac515cc1bb3f18be7dfafb Mon Sep 17 00:00:00 2001
From: zzzxl <33418555+zzzxl1993@users.noreply.github.com>
Date: Tue, 26 Dec 2023 14:08:41 +0800
Subject: [PATCH 031/299] [fix](case) Add sync to test case (#29034)

---
 .../suites/inverted_index_p0/test_count_on_index_2.groovy       | 2 ++
 .../inverted_index_p0/test_index_match_phrase_prefix.groovy     | 2 ++
 .../suites/inverted_index_p0/test_index_match_regexp.groovy     | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy b/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy
index 6866c81f964822..de94067e2f86f3 100644
--- a/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy
+++ b/regression-test/suites/inverted_index_p0/test_count_on_index_2.groovy
@@ -152,6 +152,8 @@ suite("test_count_on_index_2", "p0"){
         load_httplogs_data.call(indexTbName1, indexTbName1, 'true', 'json', 'documents-1000.json')
         load_httplogs_data.call(indexTbName2, indexTbName2, 'true', 'json', 'documents-1000.json')
 
+        sql "sync"
+
         qt_sql """ select count() from ${indexTbName1} where `@timestamp` >= 893964736 and `@timestamp` <= 893966453; """
         qt_sql """ select count() from ${indexTbName2} where `@timestamp` >= 893964736 and `@timestamp` <= 893966453; """
 
diff --git a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy
index b23bc1b5a8b82a..68f9624035fcb7 100644
--- a/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy
+++ b/regression-test/suites/inverted_index_p0/test_index_match_phrase_prefix.groovy
@@ -78,6 +78,8 @@ suite("test_index_match_phrase_prefix", "p0"){
     try {
         load_httplogs_data.call(indexTbName1, 'test_index_match_phrase_prefix', 'true', 'json', 'documents-1000.json')
 
+        sql "sync"
+
         qt_sql """ select count() from test_index_match_phrase_prefix where request match_phrase_prefix 'ima'; """
         qt_sql """ select count() from test_index_match_phrase_prefix where request like '%ima%'; """
 
diff --git a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy
index 4c1ee1a5b0b484..7567102d596ee8 100644
--- a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy
+++ b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy
@@ -78,6 +78,8 @@ suite("test_index_match_regexp", "p0"){
     try {
         load_httplogs_data.call(indexTbName1, 'test_index_match_regexp', 'true', 'json', 'documents-1000.json')
 
+        sql "sync"
+
         qt_sql """ select count() from test_index_match_regexp where request match_regexp '^h'; """
         qt_sql """ select count() from test_index_match_regexp where request match_regexp '^team'; """
         qt_sql """ select count() from test_index_match_regexp where request match_regexp 's\$'; """

From 52eeee347fa7a0de7a2b67292000231ac6bea6b7 Mon Sep 17 00:00:00 2001
From: zzzxl <33418555+zzzxl1993@users.noreply.github.com>
Date: Tue, 26 Dec 2023 14:10:53 +0800
Subject: [PATCH 032/299] [opt](compound) Optimize by deleting the compound
 expr after obtaining the final result (#28934)

---
 .../rowset/segment_v2/segment_iterator.cpp    | 15 ++++-
 .../data/inverted_index_p0/test_compound.out  |  7 +++
 .../inverted_index_p0/test_compound.groovy    | 63 +++++++++++++++++++
 3 files changed, 82 insertions(+), 3 deletions(-)
 create mode 100644 regression-test/data/inverted_index_p0/test_compound.out
 create mode 100644 regression-test/suites/inverted_index_p0/test_compound.groovy

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index f85ed5abce5982..c4c9b8d2297a23 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -490,11 +490,20 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() {
     if (config::enable_index_apply_preds_except_leafnode_of_andnode) {
         RETURN_IF_ERROR(_apply_index_except_leafnode_of_andnode());
         if (_can_filter_by_preds_except_leafnode_of_andnode()) {
-            for (auto& expr : _remaining_conjunct_roots) {
+            for (auto it = _remaining_conjunct_roots.begin();
+                 it != _remaining_conjunct_roots.end();) {
                 _pred_except_leafnode_of_andnode_evaluate_result.clear();
-                auto res = _execute_predicates_except_leafnode_of_andnode(expr);
+                auto res = _execute_predicates_except_leafnode_of_andnode(*it);
                 if (res.ok() && _pred_except_leafnode_of_andnode_evaluate_result.size() == 1) {
                     _row_bitmap &= _pred_except_leafnode_of_andnode_evaluate_result[0];
+                    // Delete expr after it obtains the final result.
+                    {
+                        std::erase_if(_common_expr_ctxs_push_down,
+                                      [&it](const auto& iter) { return iter->root() == *it; });
+                        it = _remaining_conjunct_roots.erase(it);
+                    }
+                } else {
+                    ++it;
                 }
             }
         }
@@ -505,7 +514,7 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() {
 
     std::shared_ptr<doris::ColumnPredicate> runtime_predicate = nullptr;
     if (_opts.use_topn_opt) {
-        auto query_ctx = _opts.runtime_state->get_query_ctx();
+        auto* query_ctx = _opts.runtime_state->get_query_ctx();
         runtime_predicate = query_ctx->get_runtime_predicate().get_predictate();
     }
 
diff --git a/regression-test/data/inverted_index_p0/test_compound.out b/regression-test/data/inverted_index_p0/test_compound.out
new file mode 100644
index 00000000000000..4da9bf65689d63
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/test_compound.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+2
+
+-- !sql --
+3
+
diff --git a/regression-test/suites/inverted_index_p0/test_compound.groovy b/regression-test/suites/inverted_index_p0/test_compound.groovy
new file mode 100644
index 00000000000000..9b1b8c7dde0c63
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_compound.groovy
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_compound", "p0"){
+    def timeout = 60000
+    def delta_time = 1000
+    def alter_res = "null"
+    def useTime = 0
+
+    def indexTblName = "test_compound"
+
+    sql "DROP TABLE IF EXISTS ${indexTblName}"
+
+    sql """
+	CREATE TABLE IF NOT EXISTS ${indexTblName}(
+	    `id` int(11) NOT NULL,
+        `a` text NULL DEFAULT "",
+        `b` text NULL DEFAULT "",
+        `c` text NULL DEFAULT "",
+        INDEX a_idx(`a`) USING INVERTED COMMENT '',
+        INDEX b_idx(`b`) USING INVERTED COMMENT '',
+        INDEX c_idx(`c`) USING INVERTED COMMENT ''
+	) ENGINE=OLAP
+	DUPLICATE KEY(`id`)
+	COMMENT 'OLAP'
+	DISTRIBUTED BY HASH(`id`) BUCKETS 1
+	PROPERTIES(
+ 	    "replication_allocation" = "tag.location.default: 1"
+	);
+    """
+
+    sql """ 
+        INSERT INTO $indexTblName VALUES 
+        (1, '1', '1', '1'), 
+        (2, '2', '2', '2'),
+        (3, '3', '3', '3'),
+        (4, '4', '4', '4'),
+        (5, '5', '5', '5'),
+        (6, '6', '6', '6'),
+        (7, '7', '7', '7'),
+        (8, '8', '8', '8'),
+        (9, '9', '9', '9'),
+        (10, '10', '10', '10');
+    """ 
+
+    qt_sql "SELECT count() FROM $indexTblName WHERE (id >= 2 AND id < 9) and (a match '2' or b match '5' and c match '5');"
+    qt_sql "SELECT count() FROM $indexTblName WHERE (id >= 2 AND id < 9) and (a match '2' or b match '5' or c match '6');"
+}
\ No newline at end of file

From 00bb099fdd8f5fec193aedf8f8ae8ad45453c518 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Tue, 26 Dec 2023 14:11:14 +0800
Subject: [PATCH 033/299] [hotfix](jdbc catalog) fix load table and column
 names npe (#28865)

This fix is for the npe situation that occurs when FE of non-Master nodes initializes Jdbc Catalog metadata.
---
 .../doris/datasource/jdbc/client/JdbcClient.java  | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
index 29f361df1c90e3..8d91791c336e9f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClient.java
@@ -74,8 +74,9 @@ public abstract class JdbcClient {
             lowerColumnToRealColumn = new ConcurrentHashMap<>();
 
     private final AtomicBoolean dbNamesLoaded = new AtomicBoolean(false);
-    private final AtomicBoolean tableNamesLoaded = new AtomicBoolean(false);
-    private final AtomicBoolean columnNamesLoaded = new AtomicBoolean(false);
+    private final ConcurrentHashMap<String, AtomicBoolean> tableNamesLoadedMap = new ConcurrentHashMap<>();
+    private final ConcurrentHashMap<String, ConcurrentHashMap<String, AtomicBoolean>> columnNamesLoadedMap
+            = new ConcurrentHashMap<>();
 
     public static JdbcClient createJdbcClient(JdbcClientConfig jdbcClientConfig) {
         String dbType = parseDbType(jdbcClientConfig.getJdbcUrl());
@@ -404,13 +405,18 @@ private void loadDatabaseNamesIfNeeded() {
     }
 
     private void loadTableNamesIfNeeded(String dbName) {
-        if (tableNamesLoaded.compareAndSet(false, true)) {
+        AtomicBoolean isLoaded = tableNamesLoadedMap.computeIfAbsent(dbName, k -> new AtomicBoolean(false));
+        if (isLoaded.compareAndSet(false, true)) {
             getTablesNameList(dbName);
         }
     }
 
     private void loadColumnNamesIfNeeded(String dbName, String tableName) {
-        if (columnNamesLoaded.compareAndSet(false, true)) {
+        ConcurrentHashMap<String, AtomicBoolean> tableMap = columnNamesLoadedMap.computeIfAbsent(dbName,
+                k -> new ConcurrentHashMap<>());
+        AtomicBoolean isLoaded = tableMap.computeIfAbsent(tableName, k -> new AtomicBoolean(false));
+
+        if (isLoaded.compareAndSet(false, true)) {
             getJdbcColumnsInfo(dbName, tableName);
         }
     }
@@ -469,6 +475,7 @@ protected ResultSet getColumns(DatabaseMetaData databaseMetaData, String catalog
 
     /**
      * Execute stmt direct via jdbc
+     *
      * @param origStmt, the raw stmt string
      */
     public void executeStmt(String origStmt) {

From a8688509c984b949ba55a2019c78c6a982de2feb Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Tue, 26 Dec 2023 14:11:34 +0800
Subject: [PATCH 034/299] [opt](Nereids) support cast bewteen numeric and 
 boolean in FE (#29006)

---
 .../trees/expressions/literal/Literal.java    | 29 +++++++++++++++----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
index 0d4ffc86ba91a9..29a3a3287ccc51 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java
@@ -185,15 +185,15 @@ public Expression checkedCastTo(DataType targetType) throws AnalysisException {
                 minVal = new BigDecimal(LargeIntType.MIN_VALUE);
             } else if (targetType.isFloatType()) {
                 maxVal = new BigDecimal(Float.MAX_VALUE);
-                minVal = new BigDecimal(-Float.MAX_VALUE);
+                minVal = BigDecimal.valueOf(-Float.MAX_VALUE);
             } else if (targetType.isDoubleType()) {
                 maxVal = new BigDecimal(Double.MAX_VALUE);
-                minVal = new BigDecimal(-Double.MAX_VALUE);
+                minVal = BigDecimal.valueOf(-Double.MAX_VALUE);
             }
 
             if (val.compareTo(maxVal) > 0 || val.compareTo(minVal) < 0) {
                 throw new AnalysisException(
-                        String.format("{} can't cast to {}", desc, targetType));
+                        String.format("%s can't cast to %s", desc, targetType));
             }
         }
         return uncheckedCastTo(targetType);
@@ -209,7 +209,24 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti
         }
         // TODO support string to complex
         String desc = getStringValue();
+        // convert boolean to byte string value to support cast boolean to numeric in FE.
+        if (this.equals(BooleanLiteral.TRUE)) {
+            desc = "1";
+        } else if (this.equals(BooleanLiteral.FALSE)) {
+            desc = "0";
+        }
         if (targetType.isBooleanType()) {
+            try {
+                // convert any non-zero numeric literal to true if target type is boolean
+                long value = Long.parseLong(desc);
+                if (value == 0) {
+                    return Literal.of(false);
+                } else {
+                    return Literal.of(true);
+                }
+            } catch (Exception e) {
+                // ignore
+            }
             if ("0".equals(desc) || "false".equals(desc.toLowerCase(Locale.ROOT))) {
                 return Literal.of(false);
             }
@@ -218,11 +235,11 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti
             }
         }
         if (targetType.isTinyIntType()) {
-            return Literal.of(Byte.valueOf(desc).byteValue());
+            return Literal.of(Byte.valueOf(desc));
         } else if (targetType.isSmallIntType()) {
-            return Literal.of(Short.valueOf(desc).shortValue());
+            return Literal.of(Short.valueOf(desc));
         } else if (targetType.isIntegerType()) {
-            return Literal.of(Integer.valueOf(desc).intValue());
+            return Literal.of(Integer.valueOf(desc));
         } else if (targetType.isBigIntType()) {
             return Literal.of(Long.valueOf(desc));
         } else if (targetType.isLargeIntType()) {

From 10623ad67178421a6299f3351fa2813e7ff646a8 Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Tue, 26 Dec 2023 14:12:37 +0800
Subject: [PATCH 035/299] [improvement](jdbc catalog) Optimize connection pool
 caching logic (#28859)

In the old caching logic, we only used jdbcurl, user, and password as cache keys. This may cause the old link to be still used when replacing the jar package, so we should concatenate all the parameters required for the connection pool as the key.
---
 .../java/org/apache/doris/jdbc/JdbcDataSource.java  | 13 +++++++++----
 .../java/org/apache/doris/jdbc/JdbcExecutor.java    | 10 +++++-----
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
index 95b3dac585a0f2..6a6a022d29c007 100644
--- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
+++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java
@@ -30,15 +30,20 @@ public static JdbcDataSource getDataSource() {
         return jdbcDataSource;
     }
 
-    public DruidDataSource getSource(String jdbcUrl) {
-        return sourcesMap.get(jdbcUrl);
+    public DruidDataSource getSource(String cacheKey) {
+        return sourcesMap.get(cacheKey);
     }
 
-    public void putSource(String jdbcUrl, DruidDataSource ds) {
-        sourcesMap.put(jdbcUrl, ds);
+    public void putSource(String cacheKey, DruidDataSource ds) {
+        sourcesMap.put(cacheKey, ds);
     }
 
     public Map<String, DruidDataSource> getSourcesMap() {
         return sourcesMap;
     }
+
+    public String createCacheKey(String jdbcUrl, String jdbcUser, String jdbcPassword, String jdbcDriverUrl,
+            String jdbcDriverClass) {
+        return jdbcUrl + jdbcUser + jdbcPassword + jdbcDriverUrl + jdbcDriverClass;
+    }
 }
diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
index 8864afa4938edb..9f98740aa61b3d 100644
--- a/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
+++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java
@@ -281,6 +281,8 @@ public boolean hasNext() throws UdfRuntimeException {
 
     private void init(String driverUrl, String sql, int batchSize, String driverClass, String jdbcUrl, String jdbcUser,
             String jdbcPassword, TJdbcOperation op, TOdbcTableType tableType) throws UdfRuntimeException {
+        String druidDataSourceKey = JdbcDataSource.getDataSource().createCacheKey(jdbcUrl, jdbcUser, jdbcPassword,
+                driverUrl, driverClass);
         try {
             if (isNebula()) {
                 batchSizeNum = batchSize;
@@ -290,10 +292,10 @@ private void init(String driverUrl, String sql, int batchSize, String driverClas
             } else {
                 ClassLoader parent = getClass().getClassLoader();
                 ClassLoader classLoader = UdfUtils.getClassLoader(driverUrl, parent);
-                druidDataSource = JdbcDataSource.getDataSource().getSource(jdbcUrl + jdbcUser + jdbcPassword);
+                druidDataSource = JdbcDataSource.getDataSource().getSource(druidDataSourceKey);
                 if (druidDataSource == null) {
                     synchronized (druidDataSourceLock) {
-                        druidDataSource = JdbcDataSource.getDataSource().getSource(jdbcUrl + jdbcUser + jdbcPassword);
+                        druidDataSource = JdbcDataSource.getDataSource().getSource(druidDataSourceKey);
                         if (druidDataSource == null) {
                             long start = System.currentTimeMillis();
                             DruidDataSource ds = new DruidDataSource();
@@ -312,11 +314,9 @@ private void init(String driverUrl, String sql, int batchSize, String driverClas
                             ds.setTimeBetweenEvictionRunsMillis(maxIdleTime / 5);
                             ds.setMinEvictableIdleTimeMillis(maxIdleTime);
                             druidDataSource = ds;
-                            // here is a cache of datasource, which using the string(jdbcUrl + jdbcUser +
-                            // jdbcPassword) as key.
                             // and the default datasource init = 1, min = 1, max = 100, if one of connection idle
                             // time greater than 10 minutes. then connection will be retrieved.
-                            JdbcDataSource.getDataSource().putSource(jdbcUrl + jdbcUser + jdbcPassword, ds);
+                            JdbcDataSource.getDataSource().putSource(druidDataSourceKey, ds);
                             LOG.info("init datasource [" + (jdbcUrl + jdbcUser) + "] cost: " + (
                                     System.currentTimeMillis() - start) + " ms");
                         }

From 983f851fb8b5005c8283eb5a13ed59835c4e7db2 Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Tue, 26 Dec 2023 14:13:43 +0800
Subject: [PATCH 036/299] [fix](Nereids) runtime filter push down failed
 (#28997)

project child not always NamedExpression

failed msg
```
org.apache.doris.common.AnalysisException: errCode = 2, detailMessage = class org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral cannot be cast to class org.apache.doris.nereids.trees.expressions.NamedExpression (org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral and org.apache.doris.nereids.trees.expressions.NamedExpression are in unnamed module of loader 'app')
    at org.apache.doris.qe.StmtExecutor.executeByNereids(StmtExecutor.java:623) ~[classes/:?]
    at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:478) ~[classes/:?]
    at org.apache.doris.qe.StmtExecutor.execute(StmtExecutor.java:457) ~[classes/:?]
    at org.apache.doris.qe.ConnectProcessor.handleQuery(ConnectProcessor.java:245) ~[classes/:?]
    at org.apache.doris.qe.MysqlConnectProcessor.handleQuery(MysqlConnectProcessor.java:166) ~[classes/:?]
    at org.apache.doris.qe.MysqlConnectProcessor.dispatch(MysqlConnectProcessor.java:193) ~[classes/:?]
    at org.apache.doris.qe.MysqlConnectProcessor.processOnce(MysqlConnectProcessor.java:246) ~[classes/:?]
    at org.apache.doris.mysql.ReadListener.lambda$handleEvent$0(ReadListener.java:52) ~[classes/:?]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?]
    at java.lang.Thread.run(Thread.java:829) ~[?:?]
Caused by: java.lang.ClassCastException: class org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral cannot be cast to class org.apache.doris.nereids.trees.expressions.NamedExpression (org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral and org.apache.doris.nereids.trees.expressions.NamedExpression are in unnamed module of loader 'app')
    at org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation.pushDownRuntimeFilter(PhysicalSetOperation.java:178) ~[classes/:?]
    at org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin.pushDownRuntimeFilter(PhysicalHashJoin.java:229) ~[classes/:?]
    at org.apache.doris.nereids.processor.post.RuntimeFilterGenerator.pushDownRuntimeFilterCommon(RuntimeFilterGenerator.java:386) ~[classes/:?]
```
---
 .../doris/nereids/trees/plans/physical/PhysicalHashJoin.java  | 4 ++--
 .../nereids/trees/plans/physical/PhysicalSetOperation.java    | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
index 183ccaabfa82ed..5086d0f36161e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHashJoin.java
@@ -181,8 +181,8 @@ public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<Runtim
             TRuntimeFilterType type, long buildSideNdv, int exprOrder) {
         if (RuntimeFilterGenerator.DENIED_JOIN_TYPES.contains(getJoinType()) || isMarkJoin()) {
             if (builderNode instanceof PhysicalHashJoin) {
-                PhysicalHashJoin<?, ?> builderJion = (PhysicalHashJoin<?, ?>) builderNode;
-                if (builderJion == this) {
+                PhysicalHashJoin<?, ?> builderJoin = (PhysicalHashJoin<?, ?>) builderNode;
+                if (builderJoin == this) {
                     return false;
                 }
             }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java
index ec9537e7cb981a..a85e2cf67a5f64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalSetOperation.java
@@ -173,9 +173,9 @@ public boolean pushDownRuntimeFilter(CascadesContext context, IdGenerator<Runtim
                 if (projIndex < 0 || projIndex >= project.getProjects().size()) {
                     continue;
                 }
-                NamedExpression newProbeExpr = project.getProjects().get(projIndex);
+                Expression newProbeExpr = project.getProjects().get(projIndex);
                 if (newProbeExpr instanceof Alias) {
-                    newProbeExpr = (NamedExpression) newProbeExpr.child(0);
+                    newProbeExpr = newProbeExpr.child(0);
                 }
                 Slot newProbeSlot = RuntimeFilterGenerator.checkTargetChild(newProbeExpr);
                 if (!RuntimeFilterGenerator.checkPushDownPreconditionsForJoin(builderNode, ctx, newProbeSlot)) {

From f371528adabd294d67fe25f5fe57d8b7046bb1a1 Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Tue, 26 Dec 2023 15:25:19 +0800
Subject: [PATCH 037/299] [fix](Nereids) generating function should not folding
 to NullLiteral (#29003)

should not fold table generating function to null when do constant folding.
we should remove Generate node and replaced it by project later.
---
 .../nereids/rules/expression/rules/FoldConstantRuleOnFE.java   | 3 ++-
 regression-test/suites/correctness/test_explode_numbers.groovy | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
index a73aab44f1fc95..9988a34da42fac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/FoldConstantRuleOnFE.java
@@ -47,6 +47,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
 import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.functions.generator.TableGeneratingFunction;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.Array;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ConnectionId;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.CurrentCatalog;
@@ -549,7 +550,7 @@ private boolean argsHasNullLiteral(Expression expression) {
     }
 
     private Optional<Expression> preProcess(Expression expression) {
-        if (expression instanceof AggregateFunction) {
+        if (expression instanceof AggregateFunction || expression instanceof TableGeneratingFunction) {
             return Optional.of(expression);
         }
         if (expression instanceof PropagateNullable && argsHasNullLiteral(expression)) {
diff --git a/regression-test/suites/correctness/test_explode_numbers.groovy b/regression-test/suites/correctness/test_explode_numbers.groovy
index cd85ba835e8069..3dc9e050ba42bb 100644
--- a/regression-test/suites/correctness/test_explode_numbers.groovy
+++ b/regression-test/suites/correctness/test_explode_numbers.groovy
@@ -17,6 +17,7 @@
 
 suite("test_explode_numbers") {
     sql 'set enable_nereids_planner=true'
+    sql 'set enable_fallback_to_original_planner=false'
     qt_select1 """
         select e1 from (select 1 k1) as t lateral view explode_numbers(5) tmp1 as e1 order by e1;
     """

From 31db633624a196bd4157e28ecf3c1884607badb7 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Tue, 26 Dec 2023 15:41:54 +0800
Subject: [PATCH 038/299] [improve](load) add profile for WaitFlushLimitTime
 (#29013)

---
 be/src/olap/delta_writer.cpp    |  9 +++++++--
 be/src/olap/delta_writer.h      |  1 +
 be/src/olap/delta_writer_v2.cpp | 10 ++++++++--
 be/src/olap/delta_writer_v2.h   |  1 +
 4 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index f49a68985d8606..ee6eb357e4ca62 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -73,6 +73,7 @@ DeltaWriter::DeltaWriter(StorageEngine& engine, WriteRequest* req, RuntimeProfil
 void BaseDeltaWriter::_init_profile(RuntimeProfile* profile) {
     _profile = profile->create_child(fmt::format("DeltaWriter {}", _req.tablet_id), true, true);
     _close_wait_timer = ADD_TIMER(_profile, "CloseWaitTime");
+    _wait_flush_limit_timer = ADD_TIMER(_profile, "WaitFlushLimitTime");
 }
 
 void DeltaWriter::_init_profile(RuntimeProfile* profile) {
@@ -126,8 +127,12 @@ Status BaseDeltaWriter::write(const vectorized::Block* block, const std::vector<
     if (!_is_init && !_is_cancelled) {
         RETURN_IF_ERROR(init());
     }
-    while (_memtable_writer->flush_running_count() >= config::memtable_flush_running_count_limit) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    {
+        SCOPED_TIMER(_wait_flush_limit_timer);
+        while (_memtable_writer->flush_running_count() >=
+               config::memtable_flush_running_count_limit) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        }
     }
     return _memtable_writer->write(block, row_idxs, is_append);
 }
diff --git a/be/src/olap/delta_writer.h b/be/src/olap/delta_writer.h
index c782c5ef3b6ad0..5effb6102f22db 100644
--- a/be/src/olap/delta_writer.h
+++ b/be/src/olap/delta_writer.h
@@ -115,6 +115,7 @@ class BaseDeltaWriter {
 
     RuntimeProfile* _profile = nullptr;
     RuntimeProfile::Counter* _close_wait_timer = nullptr;
+    RuntimeProfile::Counter* _wait_flush_limit_timer = nullptr;
 
     MonotonicStopWatch _lock_watch;
 };
diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index bca1213b047480..c47cdc6e563470 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -81,8 +81,10 @@ DeltaWriterV2::DeltaWriterV2(WriteRequest* req,
 void DeltaWriterV2::_update_profile(RuntimeProfile* profile) {
     auto child = profile->create_child(fmt::format("DeltaWriterV2 {}", _req.tablet_id), true, true);
     auto write_memtable_timer = ADD_TIMER(child, "WriteMemTableTime");
+    auto wait_flush_limit_timer = ADD_TIMER(child, "WaitFlushLimitTime");
     auto close_wait_timer = ADD_TIMER(child, "CloseWaitTime");
     COUNTER_SET(write_memtable_timer, _write_memtable_time);
+    COUNTER_SET(wait_flush_limit_timer, _wait_flush_limit_time);
     COUNTER_SET(close_wait_timer, _close_wait_time);
 }
 
@@ -152,8 +154,12 @@ Status DeltaWriterV2::write(const vectorized::Block* block, const std::vector<ui
     if (!_is_init && !_is_cancelled) {
         RETURN_IF_ERROR(init());
     }
-    while (_memtable_writer->flush_running_count() >= config::memtable_flush_running_count_limit) {
-        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    {
+        SCOPED_RAW_TIMER(&_wait_flush_limit_time);
+        while (_memtable_writer->flush_running_count() >=
+               config::memtable_flush_running_count_limit) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(10));
+        }
     }
     SCOPED_RAW_TIMER(&_write_memtable_time);
     return _memtable_writer->write(block, row_idxs, is_append);
diff --git a/be/src/olap/delta_writer_v2.h b/be/src/olap/delta_writer_v2.h
index f0581bf56a6219..c371a8e9818bb9 100644
--- a/be/src/olap/delta_writer_v2.h
+++ b/be/src/olap/delta_writer_v2.h
@@ -119,6 +119,7 @@ class DeltaWriterV2 {
     int64_t _total_received_rows = 0;
 
     int64_t _write_memtable_time = 0;
+    int64_t _wait_flush_limit_time = 0;
     int64_t _close_wait_time = 0;
 
     std::shared_ptr<MemTableWriter> _memtable_writer;

From 3ea9ff7d4a6e86904f18702c10e533b85e361c51 Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Tue, 26 Dec 2023 15:50:56 +0800
Subject: [PATCH 039/299] [opt](Nereids) let inverted index work with top opt
 (#29000)

---
 .../doris/nereids/rules/analysis/CheckAfterRewrite.java       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
index 26bb35192a1f5a..c43d794035a87a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
@@ -38,6 +38,7 @@
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.algebra.Generate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
@@ -175,7 +176,8 @@ private void checkMetricTypeIsUsedCorrectly(Plan plan) {
     private void checkMatchIsUsedCorrectly(Plan plan) {
         if (plan.getExpressions().stream().anyMatch(
                 expression -> expression instanceof Match)) {
-            if (plan instanceof LogicalFilter && plan.child(0) instanceof LogicalOlapScan) {
+            if (plan instanceof LogicalFilter && (plan.child(0) instanceof LogicalOlapScan
+                    || plan.child(0) instanceof LogicalDeferMaterializeOlapScan)) {
                 return;
             } else {
                 throw new AnalysisException(String.format(

From d672e321ccbf4ff96716fbb06feb9a779e786baf Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Tue, 26 Dec 2023 15:51:08 +0800
Subject: [PATCH 040/299] [feature](Nereids) support values inline table in
 query (#28972)

---
 .../org/apache/doris/nereids/DorisParser.g4   |  3 +-
 .../nereids/parser/LogicalPlanBuilder.java    |  2 +-
 .../rules/analysis/BindExpression.java        | 35 +++++++++++++------
 3 files changed, 28 insertions(+), 12 deletions(-)

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index dc202fef7431c0..98260c77dbd583 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -57,7 +57,7 @@ statement
         partitionSpec?  // partition define
         (WITH LABEL labelName=identifier)? cols=identifierList?  // label and columns define
         (LEFT_BRACKET hints=identifierSeq RIGHT_BRACKET)?  // hint define
-        (query | inlineTable)                                          #insertTable
+        query                                                          #insertTable
     | explain? cte? UPDATE tableName=multipartIdentifier tableAlias
         SET updateAssignmentSeq
         fromClause?
@@ -287,6 +287,7 @@ setQuantifier
 queryPrimary
     : querySpecification                                                   #queryPrimaryDefault
     | LEFT_PAREN query RIGHT_PAREN                                         #subquery
+    | inlineTable                                                          #valuesTable
     ;
 
 querySpecification
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 95ae97192c9b37..cf670fe1aa939e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -488,7 +488,7 @@ public LogicalPlan visitInsertTable(InsertTableContext ctx) {
         List<String> colNames = ctx.cols == null ? ImmutableList.of() : visitIdentifierList(ctx.cols);
         // TODO visit partitionSpecCtx
         Pair<Boolean, List<String>> partitionSpec = visitPartitionSpec(ctx.partitionSpec());
-        LogicalPlan plan = ctx.query() != null ? visitQuery(ctx.query()) : visitInlineTable(ctx.inlineTable());
+        LogicalPlan plan = visitQuery(ctx.query());
         UnboundTableSink<?> sink = new UnboundTableSink<>(
                 tableName.build(),
                 colNames,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
index 83b0eb1ea7924e..e1b2167e79641d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindExpression.java
@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.analyzer.UnboundSlot;
 import org.apache.doris.nereids.analyzer.UnboundTVFRelation;
 import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.parser.LogicalPlanBuilder;
 import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.rules.AppliedAwareRule.AppliedAwareRuleCondition;
 import org.apache.doris.nereids.rules.Rule;
@@ -36,6 +37,7 @@
 import org.apache.doris.nereids.trees.UnaryNode;
 import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.BoundStar;
+import org.apache.doris.nereids.trees.expressions.DefaultValueSlot;
 import org.apache.doris.nereids.trees.expressions.EqualTo;
 import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -43,6 +45,7 @@
 import org.apache.doris.nereids.trees.expressions.Properties;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
 import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder;
@@ -67,6 +70,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
 import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
@@ -87,6 +91,7 @@
 import com.google.common.collect.ImmutableList.Builder;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang3.StringUtils;
 
@@ -528,6 +533,26 @@ protected boolean condition(Rule rule, Plan plan) {
                     return new LogicalHaving<>(boundConjuncts, having.child());
                 })
             ),
+            RuleType.BINDING_INLINE_TABLE_SLOT.build(
+                logicalInlineTable().thenApply(ctx -> {
+                    LogicalInlineTable logicalInlineTable = ctx.root;
+                    // ensure all expressions are valid.
+                    List<LogicalPlan> relations
+                            = Lists.newArrayListWithCapacity(logicalInlineTable.getConstantExprsList().size());
+                    for (int i = 0; i < logicalInlineTable.getConstantExprsList().size(); i++) {
+                        if (logicalInlineTable.getConstantExprsList().get(i).stream()
+                                .anyMatch(DefaultValueSlot.class::isInstance)) {
+                            throw new AnalysisException("Default expression"
+                                    + " can't exist in SELECT statement at row " + (i + 1));
+                        }
+                        relations.add(new UnboundOneRowRelation(StatementScopeIdGenerator.newRelationId(),
+                                logicalInlineTable.getConstantExprsList().get(i)));
+                    }
+                    // construct union all tree
+                    return LogicalPlanBuilder.reduceToLogicalPlanTree(0, relations.size() - 1,
+                            relations, Qualifier.ALL);
+                })
+            ),
             RuleType.BINDING_ONE_ROW_RELATION_SLOT.build(
                 // we should bind UnboundAlias in the UnboundOneRowRelation
                 unboundOneRowRelation().thenApply(ctx -> {
@@ -540,16 +565,6 @@ protected boolean condition(Rule rule, Plan plan) {
                     return new LogicalOneRowRelation(oneRowRelation.getRelationId(), projects);
                 })
             ),
-            RuleType.BINDING_INLINE_TABLE_SLOT.build(
-                logicalInlineTable().thenApply(ctx -> {
-                    LogicalInlineTable logicalInlineTable = ctx.root;
-                    // ensure all expressions are valid.
-                    logicalInlineTable.getExpressions().forEach(expr ->
-                            bindSlot(expr, ImmutableList.of(), ctx.cascadesContext, false)
-                    );
-                    return null;
-                })
-            ),
             RuleType.BINDING_SET_OPERATION_SLOT.build(
                 // LogicalSetOperation don't bind again if LogicalSetOperation.outputs is not empty, this is special
                 // we should not remove LogicalSetOperation::canBind, because in default case, the plan can run into

From 1964a77d6c7694dc956ffe0d4a73b61205f3ccaf Mon Sep 17 00:00:00 2001
From: zhengyu <freeman.zhang1992@gmail.com>
Date: Tue, 26 Dec 2023 16:22:52 +0800
Subject: [PATCH 041/299] [enhencement](config) change default memtable size &
 loadStreamPerNode & default load parallelism (#28977)

We change memtable size from 200MB to 100MB to achieve smoother flush
performance. We change loadStreamPerNode from 20 to 60 to avoid stream
rpc to be the bottleneck when enable memtable_on_sink_node. We change
default s3&broker load parallelsim to make the most of CPUs on moderm
multi-core systems.

Signed-off-by: freemandealer <freeman.zhang1992@gmail.com>
---
 be/src/common/config.cpp                                        | 2 +-
 be/src/common/config.h                                          | 2 +-
 docs/en/docs/admin-manual/config/fe-config.md                   | 2 +-
 docs/zh-CN/docs/admin-manual/config/fe-config.md                | 2 +-
 fe/fe-common/src/main/java/org/apache/doris/common/Config.java  | 2 +-
 .../src/main/java/org/apache/doris/qe/SessionVariable.java      | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 1d8178ceec2e86..7dc1ec680a1f09 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -592,7 +592,7 @@ DEFINE_Int32(memory_max_alignment, "16");
 // memtable insert memory tracker will multiply input block size with this ratio
 DEFINE_mDouble(memtable_insert_memory_ratio, "1.4");
 // max write buffer size before flush, default 200MB
-DEFINE_mInt64(write_buffer_size, "209715200");
+DEFINE_mInt64(write_buffer_size, "104857600");
 // max buffer size used in memtable for the aggregated table, default 400MB
 DEFINE_mInt64(write_buffer_size_for_agg, "419430400");
 // max parallel flush task per memtable writer
diff --git a/be/src/common/config.h b/be/src/common/config.h
index affe47851fb97a..bb341002902ef7 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -642,7 +642,7 @@ DECLARE_Int32(memory_max_alignment);
 
 // memtable insert memory tracker will multiply input block size with this ratio
 DECLARE_mDouble(memtable_insert_memory_ratio);
-// max write buffer size before flush, default 200MB
+// max write buffer size before flush, default 100MB
 DECLARE_mInt64(write_buffer_size);
 // max buffer size used in memtable for the aggregated table, default 400MB
 DECLARE_mInt64(write_buffer_size_for_agg);
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
index 3721a364d5019f..ae3c0645be4762 100644
--- a/docs/en/docs/admin-manual/config/fe-config.md
+++ b/docs/en/docs/admin-manual/config/fe-config.md
@@ -1153,7 +1153,7 @@ Max bytes a broker scanner can process in one broker load job. Commonly, each Ba
 
 #### `default_load_parallelism`
 
-Default: 1
+Default: 8
 
 IsMutable：true
 
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
index 356677ae04e19b..580cfe7fee1e89 100644
--- a/docs/zh-CN/docs/admin-manual/config/fe-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -1154,7 +1154,7 @@ broker scanner 程序可以在一个 broker 加载作业中处理的最大字节
 
 #### `default_load_parallelism`
 
-默认值：1
+默认值：8
 
 是否可以动态配置：true
 
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index c038a9f1780ceb..61a135ed963469 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -144,7 +144,7 @@ public class Config extends ConfigBase {
 
     @ConfField(mutable = true, masterOnly = true, description = {"broker load 时，单个节点上 load 执行计划的默认并行度",
             "The default parallelism of the load execution plan on a single node when the broker load is submitted"})
-    public static int default_load_parallelism = 1;
+    public static int default_load_parallelism = 8;
 
     @ConfField(mutable = true, masterOnly = true, description = {
             "已完成或取消的导入作业信息的 label 会在这个时间后被删除。被删除的 label 可以被重用。",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 332e0f5bfaa8f7..e33a619367433a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1325,7 +1325,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
     public boolean enableMemtableOnSinkNode = true;
 
     @VariableMgr.VarAttr(name = LOAD_STREAM_PER_NODE)
-    public int loadStreamPerNode = 20;
+    public int loadStreamPerNode = 60;
 
     @VariableMgr.VarAttr(name = GROUP_COMMIT)
     public String groupCommit = "off_mode";

From 4a60d01dc769e6c8a4ad225964a367658c080631 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Tue, 26 Dec 2023 17:08:49 +0800
Subject: [PATCH 042/299] [improve](move-memtable) increase
 load_stream_flush_token_max_tasks (#29011)

---
 be/src/common/config.cpp       |  2 +-
 be/src/runtime/load_stream.cpp | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 7dc1ec680a1f09..9a886ff19c9207 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -782,7 +782,7 @@ DEFINE_Int32(load_stream_messages_in_batch, "128");
 // brpc streaming StreamWait seconds on EAGAIN
 DEFINE_Int32(load_stream_eagain_wait_seconds, "60");
 // max tasks per flush token in load stream
-DEFINE_Int32(load_stream_flush_token_max_tasks, "5");
+DEFINE_Int32(load_stream_flush_token_max_tasks, "15");
 
 // max send batch parallelism for OlapTableSink
 // The value set by the user for send_batch_parallelism is not allowed to exceed max_send_batch_parallelism_per_job,
diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index 307cd4ef30b161..836d4da147afbb 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -43,6 +43,9 @@
 
 namespace doris {
 
+bvar::LatencyRecorder g_load_stream_flush_wait_ms("load_stream_flush_wait_ms");
+bvar::Adder<int> g_load_stream_flush_running_threads("load_stream_flush_wait_threads");
+
 TabletStream::TabletStream(PUniqueId load_id, int64_t id, int64_t txn_id,
                            LoadStreamMgr* load_stream_mgr, RuntimeProfile* profile)
         : _id(id),
@@ -130,6 +133,7 @@ Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data
     butil::IOBuf buf = data->movable();
     auto flush_func = [this, new_segid, eos, buf, header]() {
         signal::set_signal_task_id(_load_id);
+        g_load_stream_flush_running_threads << -1;
         auto st = _load_stream_writer->append_data(new_segid, header.offset(), buf);
         if (eos && st.ok()) {
             st = _load_stream_writer->close_segment(new_segid);
@@ -140,9 +144,15 @@ Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data
         }
     };
     auto& flush_token = _flush_tokens[new_segid % _flush_tokens.size()];
+    MonotonicStopWatch timer;
+    timer.start();
     while (flush_token->num_tasks() >= config::load_stream_flush_token_max_tasks) {
-        bthread_usleep(10 * 1000); // 10ms
+        bthread_usleep(2 * 1000); // 2ms
     }
+    timer.stop();
+    int64_t time_ms = timer.elapsed_time() / 1000 / 1000;
+    g_load_stream_flush_wait_ms << time_ms;
+    g_load_stream_flush_running_threads << 1;
     return flush_token->submit_func(flush_func);
 }
 

From 6440fbfab6ef97c92a0b037f82b5d0466ae1b023 Mon Sep 17 00:00:00 2001
From: Jerry Hu <mrhhsg@gmail.com>
Date: Tue, 26 Dec 2023 17:18:41 +0800
Subject: [PATCH 043/299] [feature](scan) Implement parallel scanning by
 dividing the tablets based on the row range (#28967)

* [feature](scan) parallel scann on dup/mow mode

* fix bugs
---
 be/src/olap/iterators.h                       |   2 +
 be/src/olap/parallel_scanner_builder.cpp      | 201 ++++++++++++++++++
 be/src/olap/parallel_scanner_builder.h        |  98 +++++++++
 be/src/olap/rowset/beta_rowset_reader.cpp     |  22 +-
 be/src/olap/rowset/beta_rowset_reader.h       |   1 +
 be/src/olap/rowset/rowset_reader.h            |   4 +
 .../rowset/segment_v2/segment_iterator.cpp    |   4 +
 be/src/runtime/runtime_state.h                |  16 ++
 be/src/vec/columns/column_dictionary.h        |   2 +-
 be/src/vec/exec/scan/new_olap_scan_node.cpp   |  53 ++++-
 .../org/apache/doris/qe/SessionVariable.java  |  34 ++-
 .../doris/planner/ColocatePlanTest.java       |   1 +
 gensrc/thrift/PaloInternalService.thrift      |   6 +
 .../performance_p0/redundant_conjuncts.groovy |   4 +-
 14 files changed, 436 insertions(+), 12 deletions(-)
 create mode 100644 be/src/olap/parallel_scanner_builder.cpp
 create mode 100644 be/src/olap/parallel_scanner_builder.h

diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index 1f381ac4d7143e..8b61a35c2e5b2f 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -24,6 +24,7 @@
 #include "olap/block_column_predicate.h"
 #include "olap/column_predicate.h"
 #include "olap/olap_common.h"
+#include "olap/rowset/segment_v2/row_ranges.h"
 #include "olap/tablet_schema.h"
 #include "runtime/runtime_state.h"
 #include "vec/core/block.h"
@@ -115,6 +116,7 @@ class StorageReadOptions {
     int32_t tablet_id = 0;
     // slots that cast may be eliminated in storage layer
     std::map<std::string, PrimitiveType> target_cast_type_for_variants;
+    RowRanges row_ranges;
 };
 
 class RowwiseIterator;
diff --git a/be/src/olap/parallel_scanner_builder.cpp b/be/src/olap/parallel_scanner_builder.cpp
new file mode 100644
index 00000000000000..32b163b7a1a29a
--- /dev/null
+++ b/be/src/olap/parallel_scanner_builder.cpp
@@ -0,0 +1,201 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "parallel_scanner_builder.h"
+
+#include "olap/rowset/beta_rowset.h"
+#include "vec/exec/scan/new_olap_scanner.h"
+
+namespace doris {
+
+using namespace vectorized;
+
+template <typename ParentType>
+Status ParallelScannerBuilder<ParentType>::build_scanners(std::list<VScannerSPtr>& scanners) {
+    RETURN_IF_ERROR(_load());
+    if (_is_dup_mow_key) {
+        return _build_scanners_by_rowid(scanners);
+    } else {
+        // TODO: support to split by key range
+        return Status::NotSupported("split by key range not supported yet.");
+    }
+}
+
+template <typename ParentType>
+Status ParallelScannerBuilder<ParentType>::_build_scanners_by_rowid(
+        std::list<VScannerSPtr>& scanners) {
+    DCHECK_GE(_rows_per_scanner, _min_rows_per_scanner);
+    for (auto&& [tablet, version] : _tablets) {
+        DCHECK(_all_rowsets.contains(tablet->tablet_id()));
+        auto& rowsets = _all_rowsets[tablet->tablet_id()];
+
+        TabletReader::ReadSource reade_source_with_delete_info;
+        if (!_state->skip_delete_predicate()) {
+            RETURN_IF_ERROR(tablet->capture_rs_readers(
+                    {0, version}, &reade_source_with_delete_info.rs_splits, false));
+            reade_source_with_delete_info.fill_delete_predicates();
+        }
+
+        TabletReader::ReadSource read_source;
+
+        int64_t rows_collected = 0;
+        for (auto& rowset : rowsets) {
+            auto beta_rowset = std::dynamic_pointer_cast<BetaRowset>(rowset);
+            RowsetReaderSharedPtr reader;
+            RETURN_IF_ERROR(beta_rowset->create_reader(&reader));
+            const auto rowset_id = beta_rowset->rowset_id();
+
+            DCHECK(_segment_cache_handles.contains(rowset_id));
+            auto& segment_cache_handle = _segment_cache_handles[rowset_id];
+
+            if (beta_rowset->num_rows() == 0) {
+                continue;
+            }
+
+            const auto& segments = segment_cache_handle.get_segments();
+            int segment_start = 0;
+            auto split = RowSetSplits(reader->clone());
+
+            for (size_t i = 0; i != segments.size(); ++i) {
+                const auto& segment = segments[i];
+                RowRanges row_ranges;
+                const size_t rows_of_segment = segment->num_rows();
+                int64_t offset_in_segment = 0;
+
+                // try to split large segments into RowRanges
+                while (offset_in_segment < rows_of_segment) {
+                    const int64_t remaining_rows = rows_of_segment - offset_in_segment;
+                    auto rows_need = _rows_per_scanner - rows_collected;
+
+                    // 0.9: try to avoid splitting the segments into excessively small parts.
+                    if (rows_need >= remaining_rows * 0.9) {
+                        rows_need = remaining_rows;
+                    }
+                    DCHECK_LE(rows_need, remaining_rows);
+
+                    // RowRange stands for range: [From, To), From is inclusive, To is exclusive.
+                    row_ranges.add({offset_in_segment,
+                                    offset_in_segment + static_cast<int64_t>(rows_need)});
+                    rows_collected += rows_need;
+                    offset_in_segment += rows_need;
+
+                    // If collected enough rows, build a new scanner
+                    if (rows_collected >= _rows_per_scanner) {
+                        split.segment_offsets.first = segment_start,
+                        split.segment_offsets.second = i + 1;
+                        split.segment_row_ranges.emplace_back(std::move(row_ranges));
+
+                        DCHECK_EQ(split.segment_offsets.second - split.segment_offsets.first,
+                                  split.segment_row_ranges.size());
+
+                        read_source.rs_splits.emplace_back(std::move(split));
+
+                        scanners.emplace_back(
+                                _build_scanner(tablet, version, _key_ranges,
+                                               {std::move(read_source.rs_splits),
+                                                reade_source_with_delete_info.delete_predicates}));
+
+                        read_source = TabletReader::ReadSource();
+                        split = RowSetSplits(reader->clone());
+                        row_ranges = RowRanges();
+
+                        segment_start = offset_in_segment < rows_of_segment ? i : i + 1;
+                        rows_collected = 0;
+                    }
+                }
+
+                // The non-empty `row_ranges` means there are some rows left in this segment not added into `split`.
+                if (!row_ranges.is_empty()) {
+                    DCHECK_GT(rows_collected, 0);
+                    DCHECK_EQ(row_ranges.to(), segment->num_rows());
+                    split.segment_row_ranges.emplace_back(std::move(row_ranges));
+                }
+            }
+
+            DCHECK_LE(rows_collected, _rows_per_scanner);
+            if (rows_collected > 0) {
+                split.segment_offsets.first = segment_start;
+                split.segment_offsets.second = segments.size();
+                DCHECK_GT(split.segment_offsets.second, split.segment_offsets.first);
+                DCHECK_EQ(split.segment_row_ranges.size(),
+                          split.segment_offsets.second - split.segment_offsets.first);
+                read_source.rs_splits.emplace_back(std::move(split));
+            }
+        } // end `for (auto& rowset : rowsets)`
+
+        DCHECK_LE(rows_collected, _rows_per_scanner);
+        if (rows_collected > 0) {
+            DCHECK_GT(read_source.rs_splits.size(), 0);
+#ifndef NDEBUG
+            for (auto& split : read_source.rs_splits) {
+                DCHECK(split.rs_reader != nullptr);
+                DCHECK_LT(split.segment_offsets.first, split.segment_offsets.second);
+                DCHECK_EQ(split.segment_row_ranges.size(),
+                          split.segment_offsets.second - split.segment_offsets.first);
+            }
+#endif
+            scanners.emplace_back(
+                    _build_scanner(tablet, version, _key_ranges,
+                                   {std::move(read_source.rs_splits),
+                                    reade_source_with_delete_info.delete_predicates}));
+        }
+    }
+
+    return Status::OK();
+}
+
+/**
+ * Load rowsets of each tablet with specified version, segments of each rowset.
+ */
+template <typename ParentType>
+Status ParallelScannerBuilder<ParentType>::_load() {
+    _total_rows = 0;
+    for (auto&& [tablet, version] : _tablets) {
+        const auto tablet_id = tablet->tablet_id();
+        auto& rowsets = _all_rowsets[tablet_id];
+        RETURN_IF_ERROR(tablet->capture_consistent_rowsets({0, version}, &rowsets));
+
+        for (auto& rowset : rowsets) {
+            RETURN_IF_ERROR(rowset->load());
+            const auto rowset_id = rowset->rowset_id();
+            auto& segment_cache_handle = _segment_cache_handles[rowset_id];
+
+            RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
+                    std::dynamic_pointer_cast<BetaRowset>(rowset), &segment_cache_handle, true));
+            _total_rows += rowset->num_rows();
+        }
+    }
+
+    _rows_per_scanner = _total_rows / _max_scanners_count;
+    _rows_per_scanner = std::max<size_t>(_rows_per_scanner, _min_rows_per_scanner);
+
+    return Status::OK();
+}
+
+template <typename ParentType>
+std::shared_ptr<NewOlapScanner> ParallelScannerBuilder<ParentType>::_build_scanner(
+        BaseTabletSPtr tablet, int64_t version, const std::vector<OlapScanRange*>& key_ranges,
+        TabletReader::ReadSource&& read_source) {
+    NewOlapScanner::Params params {
+            _state,  _scanner_profile.get(), key_ranges,         std::move(tablet),
+            version, std::move(read_source), _limit_per_scanner, _is_preaggregation};
+    return NewOlapScanner::create_shared(_parent, std::move(params));
+}
+
+template class ParallelScannerBuilder<NewOlapScanNode>;
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/parallel_scanner_builder.h b/be/src/olap/parallel_scanner_builder.h
new file mode 100644
index 00000000000000..b9d659abc27e30
--- /dev/null
+++ b/be/src/olap/parallel_scanner_builder.h
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "olap/rowset/segment_v2/row_ranges.h"
+#include "olap/segment_loader.h"
+#include "olap/tablet.h"
+#include "vec/exec/scan/new_olap_scanner.h"
+
+namespace doris {
+
+namespace vectorized {
+class VScanner;
+}
+
+using TabletSPtr = std::shared_ptr<Tablet>;
+using VScannerSPtr = std::shared_ptr<vectorized::VScanner>;
+
+struct TabletWithVersion {
+    TabletSPtr tablet;
+    int64_t version;
+};
+
+template <typename ParentType>
+class ParallelScannerBuilder {
+public:
+    ParallelScannerBuilder(ParentType* parent, const std::vector<TabletWithVersion>& tablets,
+                           const std::shared_ptr<RuntimeProfile>& profile,
+                           const std::vector<OlapScanRange*>& key_ranges, RuntimeState* state,
+                           int64_t limit_per_scanner, bool is_dup_mow_key, bool is_preaggregation)
+            : _parent(parent),
+              _scanner_profile(profile),
+              _state(state),
+              _limit_per_scanner(limit_per_scanner),
+              _is_dup_mow_key(is_dup_mow_key),
+              _is_preaggregation(is_preaggregation),
+              _tablets(tablets.cbegin(), tablets.cend()),
+              _key_ranges(key_ranges.cbegin(), key_ranges.cend()) {}
+
+    Status build_scanners(std::list<VScannerSPtr>& scanners);
+
+    void set_max_scanners_count(size_t count) { _max_scanners_count = count; }
+
+    void set_min_rows_per_scanner(int64_t size) { _min_rows_per_scanner = size; }
+
+private:
+    Status _load();
+
+    Status _build_scanners_by_rowid(std::list<VScannerSPtr>& scanners);
+
+    std::shared_ptr<vectorized::NewOlapScanner> _build_scanner(
+            BaseTabletSPtr tablet, int64_t version, const std::vector<OlapScanRange*>& key_ranges,
+            TabletReader::ReadSource&& read_source);
+
+    ParentType* _parent;
+
+    /// Max scanners count limit to build
+    size_t _max_scanners_count {16};
+
+    /// Min rows per scanner
+    size_t _min_rows_per_scanner {2 * 1024 * 1024};
+
+    size_t _total_rows {};
+
+    size_t _rows_per_scanner {_min_rows_per_scanner};
+
+    std::map<RowsetId, SegmentCacheHandle> _segment_cache_handles;
+
+    std::shared_ptr<RuntimeProfile> _scanner_profile;
+    RuntimeState* _state;
+    int64_t _limit_per_scanner;
+    bool _is_dup_mow_key;
+    bool _is_preaggregation;
+    std::vector<TabletWithVersion> _tablets;
+    std::vector<OlapScanRange*> _key_ranges;
+    std::unordered_map<int64_t, std::vector<RowsetSharedPtr>> _all_rowsets;
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp b/be/src/olap/rowset/beta_rowset_reader.cpp
index 18725b116d1934..568fa89d848b81 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -237,10 +237,23 @@ Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
     for (int i = seg_start; i < seg_end; i++) {
         auto& seg_ptr = segments[i];
         std::unique_ptr<RowwiseIterator> iter;
-        auto s = seg_ptr->new_iterator(_input_schema, _read_options, &iter);
-        if (!s.ok()) {
-            LOG(WARNING) << "failed to create iterator[" << seg_ptr->id() << "]: " << s.to_string();
-            return Status::Error<ROWSET_READER_INIT>(s.to_string());
+        Status status;
+
+        /// If `_segment_row_ranges` is empty, the segment is not split.
+        if (_segment_row_ranges.empty()) {
+            _read_options.row_ranges.clear();
+            status = seg_ptr->new_iterator(_input_schema, _read_options, &iter);
+        } else {
+            DCHECK_EQ(seg_end - seg_start, _segment_row_ranges.size());
+            auto local_options = _read_options;
+            local_options.row_ranges = _segment_row_ranges[i - seg_start];
+            status = seg_ptr->new_iterator(_input_schema, local_options, &iter);
+        }
+
+        if (!status.ok()) {
+            LOG(WARNING) << "failed to create iterator[" << seg_ptr->id()
+                         << "]: " << status.to_string();
+            return Status::Error<ROWSET_READER_INIT>(status.to_string());
         }
         if (iter->empty()) {
             continue;
@@ -255,6 +268,7 @@ Status BetaRowsetReader::init(RowsetReaderContext* read_context, const RowSetSpl
     _read_context = read_context;
     _read_context->rowset_id = _rowset->rowset_id();
     _segment_offsets = rs_splits.segment_offsets;
+    _segment_row_ranges = rs_splits.segment_row_ranges;
     return Status::OK();
 }
 
diff --git a/be/src/olap/rowset/beta_rowset_reader.h b/be/src/olap/rowset/beta_rowset_reader.h
index 45aef035950278..3834f2f10bb946 100644
--- a/be/src/olap/rowset/beta_rowset_reader.h
+++ b/be/src/olap/rowset/beta_rowset_reader.h
@@ -96,6 +96,7 @@ class BetaRowsetReader : public RowsetReader {
     DorisCallOnce<Status> _init_iter_once;
 
     std::pair<int, int> _segment_offsets;
+    std::vector<RowRanges> _segment_row_ranges;
 
     SchemaSPtr _input_schema;
     RowsetReaderContext* _read_context = nullptr;
diff --git a/be/src/olap/rowset/rowset_reader.h b/be/src/olap/rowset/rowset_reader.h
index 3a93888be90419..45449952431bd6 100644
--- a/be/src/olap/rowset/rowset_reader.h
+++ b/be/src/olap/rowset/rowset_reader.h
@@ -25,6 +25,7 @@
 #include "olap/iterators.h"
 #include "olap/rowset/rowset_fwd.h"
 #include "olap/rowset/rowset_reader_context.h"
+#include "olap/rowset/segment_v2/row_ranges.h"
 #include "vec/core/block.h"
 
 namespace doris {
@@ -41,6 +42,9 @@ struct RowSetSplits {
     // and pipeline
     std::pair<int, int> segment_offsets;
 
+    // RowRanges of each segment.
+    std::vector<RowRanges> segment_row_ranges;
+
     RowSetSplits(RowsetReaderSharedPtr rs_reader_)
             : rs_reader(rs_reader_), segment_offsets({0, 0}) {}
     RowSetSplits() = default;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index c4c9b8d2297a23..3bbbe0bcbc6d6a 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -381,6 +381,10 @@ Status SegmentIterator::_lazy_init() {
                    << _opts.delete_bitmap.at(segment_id())->cardinality() << ", "
                    << _opts.stats->rows_del_by_bitmap << " rows deleted by bitmap";
     }
+
+    if (!_opts.row_ranges.is_empty()) {
+        _row_bitmap &= RowRanges::ranges_to_roaring(_opts.row_ranges);
+    }
     if (_opts.read_orderby_key_reverse) {
         _range_iter.reset(new BackwardBitmapRangeIterator(_row_bitmap));
     } else {
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 6ced139eb4c8b8..ee515007eb604c 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -488,6 +488,22 @@ class RuntimeState {
                _query_options.enable_hash_join_early_start_probe;
     }
 
+    bool enable_parallel_scan() const {
+        return _query_options.__isset.enable_parallel_scan && _query_options.enable_parallel_scan;
+    }
+
+    int parallel_scan_max_scanners_count() const {
+        return _query_options.__isset.parallel_scan_max_scanners_count
+                       ? _query_options.parallel_scan_max_scanners_count
+                       : 0;
+    }
+
+    int64_t parallel_scan_min_rows_per_scanner() const {
+        return _query_options.__isset.parallel_scan_min_rows_per_scanner
+                       ? _query_options.parallel_scan_min_rows_per_scanner
+                       : 0;
+    }
+
     int repeat_max_num() const {
 #ifndef BE_TEST
         if (!_query_options.__isset.repeat_max_num) {
diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h
index 06e0a352be2c1a..6de729fd2de7e0 100644
--- a/be/src/vec/columns/column_dictionary.h
+++ b/be/src/vec/columns/column_dictionary.h
@@ -207,7 +207,7 @@ class ColumnDictionary final : public COWHelper<IColumn, ColumnDictionary<T>> {
     }
 
     Status filter_by_selector(const uint16_t* sel, size_t sel_size, IColumn* col_ptr) override {
-        auto* res_col = reinterpret_cast<vectorized::ColumnString*>(col_ptr);
+        auto* res_col = assert_cast<vectorized::ColumnString*>(col_ptr);
         StringRef strings[sel_size];
         size_t length = 0;
         for (size_t i = 0; i != sel_size; ++i) {
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp b/be/src/vec/exec/scan/new_olap_scan_node.cpp
index 38fd97f2413688..b33c656a8df409 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp
@@ -39,18 +39,15 @@
 #include "common/object_pool.h"
 #include "common/status.h"
 #include "exec/exec_node.h"
-#include "olap/rowset/rowset.h"
+#include "olap/parallel_scanner_builder.h"
 #include "olap/rowset/rowset_reader.h"
 #include "olap/storage_engine.h"
-#include "olap/tablet.h"
 #include "olap/tablet_manager.h"
 #include "olap/tablet_reader.h"
-#include "runtime/decimalv2_value.h"
 #include "runtime/define_primitive_type.h"
 #include "runtime/query_statistics.h"
 #include "runtime/runtime_state.h"
 #include "runtime/types.h"
-#include "service/backend_options.h"
 #include "util/time.h"
 #include "util/to_string.h"
 #include "vec/columns/column.h"
@@ -518,14 +515,20 @@ Status NewOlapScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
     std::vector<std::pair<BaseTabletSPtr, int64_t /* version */>> tablets_to_scan;
     tablets_to_scan.reserve(_scan_ranges.size());
 
+    std::vector<TabletWithVersion> tablets;
+
     for (auto&& scan_range : _scan_ranges) {
         auto tablet = DORIS_TRY(ExecEnv::get_tablet(scan_range->tablet_id));
         int64_t version = 0;
         std::from_chars(scan_range->version.data(),
                         scan_range->version.data() + scan_range->version.size(), version);
+        tablets.emplace_back(
+                TabletWithVersion {std::dynamic_pointer_cast<Tablet>(tablet), version});
         tablets_to_scan.emplace_back(std::move(tablet), version);
     }
 
+    bool enable_parallel_scan = _state->enable_parallel_scan();
+
     // Split tablet segment by scanner, only use in pipeline in duplicate key
     // 1. if tablet count lower than scanner thread num, count segment num of all tablet ready for scan
     // TODO: some tablet may do not have segment, may need split segment all case
@@ -562,6 +565,48 @@ Status NewOlapScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
         }
     }
 
+    bool has_cpu_limit = _state->query_options().__isset.resource_limit &&
+                         _state->query_options().resource_limit.__isset.cpu_limit;
+
+    if (enable_parallel_scan && is_dup_mow_key && !has_cpu_limit && !should_run_serial() &&
+        _push_down_agg_type == TPushAggOp::NONE) {
+        std::vector<OlapScanRange*> key_ranges;
+        for (auto& range : _cond_ranges) {
+            if (range->begin_scan_range.size() == 1 &&
+                range->begin_scan_range.get_value(0) == NEGATIVE_INFINITY) {
+                continue;
+            }
+            key_ranges.emplace_back(range.get());
+        }
+
+        ParallelScannerBuilder<NewOlapScanNode> scanner_builder(
+                this, tablets, _scanner_profile, key_ranges, _state, _limit_per_scanner,
+                is_dup_mow_key, _olap_scan_node.is_preaggregation);
+
+        int max_scanners_count = _state->parallel_scan_max_scanners_count();
+
+        // If the `max_scanners_count` was not set,
+        // use `config::doris_scanner_thread_pool_thread_num` as the default value.
+        if (max_scanners_count <= 0) {
+            max_scanners_count = config::doris_scanner_thread_pool_thread_num;
+        }
+
+        // Too small value of `min_rows_per_scanner` is meaningless.
+        auto min_rows_per_scanner =
+                std::max<int64_t>(1024, _state->parallel_scan_min_rows_per_scanner());
+        scanner_builder.set_max_scanners_count(max_scanners_count);
+        scanner_builder.set_min_rows_per_scanner(min_rows_per_scanner);
+
+        RETURN_IF_ERROR(scanner_builder.build_scanners(*scanners));
+        for (auto& scanner : *scanners) {
+            auto* olap_scanner = assert_cast<NewOlapScanner*>(scanner.get());
+            RETURN_IF_ERROR(olap_scanner->prepare(_state, _conjuncts));
+            olap_scanner->set_compound_filters(_compound_filters);
+        }
+        LOG(INFO) << "segment count: " << segment_count << ", scanners count: " << scanners->size();
+        return Status::OK();
+    }
+
     auto build_new_scanner = [&](BaseTabletSPtr tablet, int64_t version,
                                  const std::vector<OlapScanRange*>& key_ranges,
                                  TabletReader::ReadSource read_source) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index e33a619367433a..23813f28dde367 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -224,6 +224,14 @@ public class SessionVariable implements Serializable, Writable {
 
     public static final String IGNORE_STORAGE_DATA_DISTRIBUTION = "ignore_storage_data_distribution";
 
+    public static final String ENABLE_PARALLEL_SCAN = "enable_parallel_scan";
+
+    // Limit the max count of scanners to prevent generate too many scanners.
+    public static final String PARALLEL_SCAN_MAX_SCANNERS_COUNT = "parallel_scan_max_scanners_count";
+
+    // Avoid splitting small segments, each scanner should scan `parallel_scan_min_rows_per_scanner` rows.
+    public static final String PARALLEL_SCAN_MIN_ROWS_PER_SCANNER = "parallel_scan_min_rows_per_scanner";
+
     public static final String ENABLE_LOCAL_SHUFFLE = "enable_local_shuffle";
 
     public static final String ENABLE_AGG_STATE = "enable_agg_state";
@@ -784,7 +792,19 @@ public class SessionVariable implements Serializable, Writable {
 
     @VariableMgr.VarAttr(name = ENABLE_SHARED_SCAN, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,
             needForward = true)
-    private boolean enableSharedScan = false;
+    private boolean enableSharedScan = true;
+
+    @VariableMgr.VarAttr(name = ENABLE_PARALLEL_SCAN, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,
+            needForward = true)
+    private boolean enableParallelScan = true;
+
+    @VariableMgr.VarAttr(name = PARALLEL_SCAN_MAX_SCANNERS_COUNT, fuzzy = false,
+            varType = VariableAnnotation.EXPERIMENTAL, needForward = true)
+    private int parallelScanMaxScannersCount = 48;
+
+    @VariableMgr.VarAttr(name = PARALLEL_SCAN_MIN_ROWS_PER_SCANNER, fuzzy = false,
+            varType = VariableAnnotation.EXPERIMENTAL, needForward = true)
+    private long parallelScanMinRowsPerScanner = 65536; // 2MB
 
     @VariableMgr.VarAttr(name = IGNORE_STORAGE_DATA_DISTRIBUTION, fuzzy = false,
             varType = VariableAnnotation.EXPERIMENTAL, needForward = true)
@@ -2746,6 +2766,10 @@ public TQueryOptions toThrift() {
 
         tResult.setInvertedIndexSkipThreshold(invertedIndexSkipThreshold);
 
+        tResult.setEnableParallelScan(enableParallelScan);
+        tResult.setParallelScanMaxScannersCount(parallelScanMaxScannersCount);
+        tResult.setParallelScanMinRowsPerScanner(parallelScanMinRowsPerScanner);
+
         return tResult;
     }
 
@@ -3059,6 +3083,14 @@ public boolean getEnableSharedScan() {
         return enableSharedScan;
     }
 
+    public void setEnableSharedScan(boolean value) {
+        enableSharedScan = value;
+    }
+
+    public boolean getEnableParallelScan() {
+        return enableParallelScan;
+    }
+
     public boolean getEnablePipelineXEngine() {
         return enablePipelineXEngine;
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java
index 6b976e1f273d8d..c5275c20f0d6f7 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/ColocatePlanTest.java
@@ -169,6 +169,7 @@ public void sqlAggWithColocateTable() throws Exception {
 
     @Test
     public void checkColocatePlanFragment() throws Exception {
+        connectContext.getSessionVariable().setEnableSharedScan(false);
         String sql
                 = "select /*+ SET_VAR(enable_nereids_planner=false) */ a.k1 from db1.test_colocate a, db1.test_colocate b where a.k1=b.k1 and a.k2=b.k2 group by a.k1;";
         StmtExecutor executor = getSqlStmtExecutor(sql);
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index b1ce326c826db0..e07797b593d06c 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -263,6 +263,12 @@ struct TQueryOptions {
   93: optional i32 inverted_index_max_expansions = 50;
 
   94: optional i32 inverted_index_skip_threshold = 50;
+
+  95: optional bool enable_parallel_scan = false;
+
+  96: optional i32 parallel_scan_max_scanners_count = 0;
+
+  97: optional i64 parallel_scan_min_rows_per_scanner = 0;
 }
 
 
diff --git a/regression-test/suites/performance_p0/redundant_conjuncts.groovy b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
index 9d81b5f91ba5b9..4aec6275dc356c 100644
--- a/regression-test/suites/performance_p0/redundant_conjuncts.groovy
+++ b/regression-test/suites/performance_p0/redundant_conjuncts.groovy
@@ -32,10 +32,10 @@ suite("redundant_conjuncts") {
     """
     
     qt_redundant_conjuncts """
-    EXPLAIN SELECT /*+SET_VAR(enable_nereids_planner=false, REWRITE_OR_TO_IN_PREDICATE_THRESHOLD=2, parallel_fragment_exec_instance_num = 1) */ v1 FROM redundant_conjuncts WHERE k1 = 1 AND k1 = 1;
+    EXPLAIN SELECT /*+SET_VAR(enable_nereids_planner=false, REWRITE_OR_TO_IN_PREDICATE_THRESHOLD=2, parallel_fragment_exec_instance_num = 1, enable_shared_scan = false) */ v1 FROM redundant_conjuncts WHERE k1 = 1 AND k1 = 1;
     """
 
     qt_redundant_conjuncts_gnerated_by_extract_common_filter """
-    EXPLAIN SELECT /*+SET_VAR(enable_nereids_planner=false, REWRITE_OR_TO_IN_PREDICATE_THRESHOLD=100, parallel_fragment_exec_instance_num = 1) */ v1 FROM redundant_conjuncts WHERE k1 = 1 OR k1 = 2;
+    EXPLAIN SELECT /*+SET_VAR(enable_nereids_planner=false, REWRITE_OR_TO_IN_PREDICATE_THRESHOLD=100, parallel_fragment_exec_instance_num = 1, enable_shared_scan = false) */ v1 FROM redundant_conjuncts WHERE k1 = 1 OR k1 = 2;
     """
 }

From ffc6596ceffd1a2bfff37529b6e49d1e2419c236 Mon Sep 17 00:00:00 2001
From: yujun <yu.jun.reach@gmail.com>
Date: Tue, 26 Dec 2023 17:36:05 +0800
Subject: [PATCH 044/299] [refactor](create tablet) default create tablet round
 robin  (#28911)

---
 .../java/org/apache/doris/common/Config.java  |   8 +-
 .../org/apache/doris/backup/RestoreJob.java   |   3 +-
 .../org/apache/doris/catalog/OlapTable.java   |   3 +-
 .../common/util/DynamicPartitionUtil.java     |  13 +-
 .../doris/common/util/PropertyAnalyzer.java   |   3 +-
 .../doris/datasource/InternalCatalog.java     |  40 +---
 .../doris/system/BeSelectionPolicy.java       |  15 ++
 .../doris/system/SystemInfoService.java       | 209 ++++++++----------
 .../apache/doris/backup/RestoreJobTest.java   |  10 +-
 .../doris/catalog/ReplicaAllocationTest.java  |   3 +-
 .../load/sync/canal/CanalSyncDataTest.java    |   2 +-
 .../doris/system/SystemInfoServiceTest.java   |   2 +-
 12 files changed, 149 insertions(+), 162 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 61a135ed963469..4c92449cb5736f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2075,7 +2075,13 @@ public class Config extends ConfigBase {
     public static boolean skip_localhost_auth_check  = true;
 
     @ConfField(mutable = true)
-    public static boolean enable_round_robin_create_tablet = false;
+    public static boolean enable_round_robin_create_tablet = true;
+
+    @ConfField(mutable = true, masterOnly = true, description = {
+            "创建分区时，总是从第一个 BE 开始创建。注意：这种方式可能造成BE不均衡",
+            "When creating tablet of a partition, always start from the first BE. "
+                    + "Note: This method may cause BE imbalance"})
+    public static boolean create_tablet_round_robin_from_start = false;
 
     /**
      * To prevent different types (V1, V2, V3) of behavioral inconsistencies,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 6d4f68e985d7b2..23ca4a335fdd90 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1113,6 +1113,7 @@ private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteT
         long visibleVersion = remotePart.getVisibleVersion();
 
         // tablets
+        Map<Tag, Integer> nextIndexs = Maps.newHashMap();
         for (MaterializedIndex remoteIdx : remotePart.getMaterializedIndices(IndexExtState.VISIBLE)) {
             int schemaHash = remoteTbl.getSchemaHashByIndexId(remoteIdx.getId());
             int remotetabletSize = remoteIdx.getTablets().size();
@@ -1127,7 +1128,7 @@ private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteT
                 // replicas
                 try {
                     Map<Tag, List<Long>> beIds = Env.getCurrentSystemInfo()
-                            .selectBackendIdsForReplicaCreation(replicaAlloc, null, false, false);
+                            .selectBackendIdsForReplicaCreation(replicaAlloc, nextIndexs, null, false, false);
                     for (Map.Entry<Tag, List<Long>> entry : beIds.entrySet()) {
                         for (Long beId : entry.getValue()) {
                             long newReplicaId = env.getNextId();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 0ce2df5e6f64f4..d12dc9512b5b40 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -589,6 +589,7 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore
         }
 
         // for each partition, reset rollup index map
+        Map<Tag, Integer> nextIndexs = Maps.newHashMap();
         for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
             Partition partition = entry.getValue();
             // entry.getKey() is the new partition id, use it to get the restore specified replica allocation
@@ -616,7 +617,7 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore
                     try {
                         Map<Tag, List<Long>> tag2beIds =
                                 Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(
-                                        replicaAlloc, null, false, false);
+                                        replicaAlloc, nextIndexs, null, false, false);
                         for (Map.Entry<Tag, List<Long>> entry3 : tag2beIds.entrySet()) {
                             for (Long beId : entry3.getValue()) {
                                 long newReplicaId = env.getNextId();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
index de6101181f48ef..ff2c9433057347 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java
@@ -40,10 +40,12 @@
 import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.common.UserException;
 import org.apache.doris.policy.StoragePolicy;
+import org.apache.doris.resource.Tag;
 import org.apache.doris.thrift.TStorageMedium;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
 import com.google.common.collect.Range;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -228,7 +230,8 @@ private static void checkReplicationNum(String val, Database db) throws DdlExcep
             ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_FORMAT, val);
         }
         ReplicaAllocation replicaAlloc = new ReplicaAllocation(Short.valueOf(val));
-        Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null, false, true);
+        Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, Maps.newHashMap(),
+                null, false, true);
     }
 
     private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int hotPartitionNum,
@@ -237,14 +240,16 @@ private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int h
             ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO);
         }
 
-        Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null, false, true);
+        Map<Tag, Integer> nextIndexs = Maps.newHashMap();
+        Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, nextIndexs, null,
+                false, true);
         if (hotPartitionNum <= 0) {
             return;
         }
 
         try {
-            Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, TStorageMedium.SSD, false,
-                    true);
+            Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, nextIndexs,
+                    TStorageMedium.SSD, false, true);
         } catch (DdlException e) {
             throw new DdlException("Failed to find enough backend for ssd storage medium. When setting "
                     + DynamicPartitionProperty.HOT_PARTITION_NUM + " > 0, the hot partitions will store "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index ca0310769ee277..c8c0978ca74015 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -1069,6 +1069,7 @@ private static ReplicaAllocation analyzeReplicaAllocationImpl(Map<String, String
         allocationVal = allocationVal.replaceAll(" ", "");
         String[] locations = allocationVal.split(",");
         int totalReplicaNum = 0;
+        Map<Tag, Integer> nextIndexs = Maps.newHashMap();
         for (String location : locations) {
             String[] parts = location.split(":");
             if (parts.length != 2) {
@@ -1092,7 +1093,7 @@ private static ReplicaAllocation analyzeReplicaAllocationImpl(Map<String, String
                 try {
                     SystemInfoService systemInfoService = Env.getCurrentSystemInfo();
                     systemInfoService.selectBackendIdsForReplicaCreation(
-                            replicaAlloc, null, false, true);
+                            replicaAlloc, nextIndexs, null, false, true);
                 } catch (DdlException ddlException) {
                     throw new AnalysisException(ddlException.getMessage());
                 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index dea018b6425f51..2e031faede2674 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -2743,6 +2743,7 @@ public void createTablets(MaterializedIndex index, ReplicaState replicaState,
             Set<Long> tabletIdSet, IdGeneratorBuffer idGeneratorBuffer, boolean isStorageMediumSpecified)
             throws DdlException {
         ColocateTableIndex colocateIndex = Env.getCurrentColocateIndex();
+        SystemInfoService systemInfoService = Env.getCurrentSystemInfo();
         Map<Tag, List<List<Long>>> backendsPerBucketSeq = null;
         GroupId groupId = null;
         if (colocateIndex.isColocateTable(tabletMeta.getTableId())) {
@@ -2762,16 +2763,18 @@ public void createTablets(MaterializedIndex index, ReplicaState replicaState,
             backendsPerBucketSeq = Maps.newHashMap();
         }
 
+        TStorageMedium storageMedium = Config.disable_storage_medium_check ? null : tabletMeta.getStorageMedium();
         Map<Tag, Integer> nextIndexs = new HashMap<>();
-
         if (Config.enable_round_robin_create_tablet) {
-            for (Map.Entry<Tag, Short> entry : replicaAlloc.getAllocMap().entrySet()) {
-                int startPos = Env.getCurrentSystemInfo().getStartPosOfRoundRobin(entry.getKey(),
-                        tabletMeta.getStorageMedium());
-                if (startPos == -1) {
-                    throw new DdlException("The number of BEs that match the policy is insufficient");
+            for (Tag tag : replicaAlloc.getAllocMap().keySet()) {
+                int startPos = -1;
+                if (Config.create_tablet_round_robin_from_start) {
+                    startPos = 0;
+                } else {
+                    startPos = systemInfoService.getStartPosOfRoundRobin(tag, storageMedium,
+                            isStorageMediumSpecified);
                 }
-                nextIndexs.put(entry.getKey(), startPos);
+                nextIndexs.put(tag, startPos);
             }
         }
 
@@ -2788,27 +2791,8 @@ public void createTablets(MaterializedIndex index, ReplicaState replicaState,
             if (chooseBackendsArbitrary) {
                 // This is the first colocate table in the group, or just a normal table,
                 // choose backends
-                if (Config.enable_round_robin_create_tablet) {
-                    if (!Config.disable_storage_medium_check) {
-                        chosenBackendIds = Env.getCurrentSystemInfo()
-                                .getBeIdRoundRobinForReplicaCreation(replicaAlloc, tabletMeta.getStorageMedium(),
-                                        nextIndexs);
-                    } else {
-                        chosenBackendIds = Env.getCurrentSystemInfo()
-                                .getBeIdRoundRobinForReplicaCreation(replicaAlloc, null,
-                                        nextIndexs);
-                    }
-                } else {
-                    if (!Config.disable_storage_medium_check) {
-                        chosenBackendIds = Env.getCurrentSystemInfo()
-                                .selectBackendIdsForReplicaCreation(replicaAlloc, tabletMeta.getStorageMedium(),
-                                        isStorageMediumSpecified, false);
-                    } else {
-                        chosenBackendIds = Env.getCurrentSystemInfo()
-                                .selectBackendIdsForReplicaCreation(replicaAlloc, null,
-                                        isStorageMediumSpecified, false);
-                    }
-                }
+                chosenBackendIds = systemInfoService.selectBackendIdsForReplicaCreation(replicaAlloc, nextIndexs,
+                        storageMedium, isStorageMediumSpecified, false);
 
                 for (Map.Entry<Tag, List<Long>> entry : chosenBackendIds.entrySet()) {
                     backendsPerBucketSeq.putIfAbsent(entry.getKey(), Lists.newArrayList());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java
index 3a711307bd5595..ace2ab3e1e4b73 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/BeSelectionPolicy.java
@@ -51,6 +51,11 @@ public class BeSelectionPolicy {
     public boolean preferComputeNode = false;
     public int expectBeNum = 0;
 
+    public boolean enableRoundRobin = false;
+    // if enable round robin, choose next be from nextRoundRobinIndex
+    // call SystemInfoService::selectBackendIdsByPolicy will update nextRoundRobinIndex
+    public int nextRoundRobinIndex = -1;
+
     public List<String> preferredLocations = new ArrayList<>();
 
     private BeSelectionPolicy() {
@@ -114,6 +119,16 @@ public Builder addPreLocations(List<String> preferredLocations) {
             return this;
         }
 
+        public Builder setEnableRoundRobin(boolean enableRoundRobin) {
+            policy.enableRoundRobin = enableRoundRobin;
+            return this;
+        }
+
+        public Builder setNextRoundRobinIndex(int nextRoundRobinIndex) {
+            policy.nextRoundRobinIndex = nextRoundRobinIndex;
+            return this;
+        }
+
         public BeSelectionPolicy build() {
             return policy;
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
index 3ccb6d6345642b..9c1e196923f40d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
@@ -54,6 +54,7 @@
 
 import java.io.DataInputStream;
 import java.io.IOException;
+import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -415,86 +416,42 @@ public List<Backend> getCnBackends() {
         return idToBackendRef.values().stream().filter(backend -> backend.isComputeNode()).collect(Collectors.toList());
     }
 
-    class BeComparator implements Comparator<Backend> {
+    class BeIdComparator implements Comparator<Backend> {
         public int compare(Backend a, Backend b) {
             return (int) (a.getId() - b.getId());
         }
     }
 
-    public List<Long> selectBackendIdsRoundRobinByPolicy(BeSelectionPolicy policy, int number,
-            int nextIndex) {
-        Preconditions.checkArgument(number >= -1);
-        List<Backend> candidates = getCandidates(policy);
-        if (number != -1 && candidates.size() < number) {
-            LOG.info("Not match policy: {}. candidates num: {}, expected: {}", policy, candidates.size(), number);
-            return Lists.newArrayList();
-        }
-
-        int realIndex = nextIndex % candidates.size();
-        List<Long> partialOrderList = new ArrayList<Long>();
-        partialOrderList.addAll(candidates.subList(realIndex, candidates.size())
-                .stream().map(b -> b.getId()).collect(Collectors.toList()));
-        partialOrderList.addAll(candidates.subList(0, realIndex)
-                .stream().map(b -> b.getId()).collect(Collectors.toList()));
-
-        if (number == -1) {
-            return partialOrderList;
-        } else {
-            return partialOrderList.subList(0, number);
-        }
-    }
-
-    public List<Backend> getCandidates(BeSelectionPolicy policy) {
-        List<Backend> candidates = policy.getCandidateBackends(idToBackendRef.values());
-        if (candidates.isEmpty()) {
-            LOG.info("Not match policy: {}. candidates num: {}", policy, candidates.size());
-            return Lists.newArrayList();
-        }
-
-        if (!policy.allowOnSameHost) {
-            Map<String, List<Backend>> backendMaps = Maps.newHashMap();
-            for (Backend backend : candidates) {
-                if (backendMaps.containsKey(backend.getHost())) {
-                    backendMaps.get(backend.getHost()).add(backend);
-                } else {
-                    List<Backend> list = Lists.newArrayList();
-                    list.add(backend);
-                    backendMaps.put(backend.getHost(), list);
-                }
-            }
-            candidates.clear();
-            for (List<Backend> list : backendMaps.values()) {
-                candidates.add(list.get(0));
-            }
-        }
-
-        if (candidates.isEmpty()) {
-            LOG.info("Not match policy: {}. candidates num: {}", policy, candidates.size());
-            return Lists.newArrayList();
+    class BeHostComparator implements Comparator<Backend> {
+        public int compare(Backend a, Backend b) {
+            return a.getHost().compareTo(b.getHost());
         }
-
-        Collections.sort(candidates, new BeComparator());
-        return candidates;
     }
 
     // Select the smallest number of tablets as the starting position of
     // round robin in the BE that match the policy
-    public int getStartPosOfRoundRobin(Tag tag, TStorageMedium storageMedium) {
+    public int getStartPosOfRoundRobin(Tag tag, TStorageMedium storageMedium, boolean isStorageMediumSpecified) {
         BeSelectionPolicy.Builder builder = new BeSelectionPolicy.Builder()
-                .needScheduleAvailable().needCheckDiskUsage().addTags(Sets.newHashSet(tag))
+                .needScheduleAvailable()
+                .needCheckDiskUsage()
+                .addTags(Sets.newHashSet(tag))
                 .setStorageMedium(storageMedium);
         if (FeConstants.runningUnitTest || Config.allow_replica_on_same_host) {
             builder.allowOnSameHost();
         }
 
         BeSelectionPolicy policy = builder.build();
-        List<Backend> candidates = getCandidates(policy);
+        List<Long> beIds = selectBackendIdsByPolicy(policy, -1);
+        if (beIds.isEmpty() && storageMedium != null && !isStorageMediumSpecified) {
+            storageMedium = (storageMedium == TStorageMedium.HDD) ? TStorageMedium.SSD : TStorageMedium.HDD;
+            policy = builder.setStorageMedium(storageMedium).build();
+            beIds = selectBackendIdsByPolicy(policy, -1);
+        }
 
         long minBeTabletsNum = Long.MAX_VALUE;
         int minIndex = -1;
-        for (int i = 0; i < candidates.size(); ++i) {
-            long tabletsNum = Env.getCurrentInvertedIndex()
-                    .getTabletIdsByBackendId(candidates.get(i).getId()).size();
+        for (int i = 0; i < beIds.size(); ++i) {
+            long tabletsNum = Env.getCurrentInvertedIndex().getTabletIdsByBackendId(beIds.get(i)).size();
             if (tabletsNum < minBeTabletsNum) {
                 minBeTabletsNum = tabletsNum;
                 minIndex = i;
@@ -503,40 +460,12 @@ public int getStartPosOfRoundRobin(Tag tag, TStorageMedium storageMedium) {
         return minIndex;
     }
 
-    public Map<Tag, List<Long>> getBeIdRoundRobinForReplicaCreation(
-            ReplicaAllocation replicaAlloc, TStorageMedium storageMedium,
-            Map<Tag, Integer> nextIndexs) throws DdlException {
-        Map<Tag, List<Long>> chosenBackendIds = Maps.newHashMap();
-        Map<Tag, Short> allocMap = replicaAlloc.getAllocMap();
-        short totalReplicaNum = 0;
-        for (Map.Entry<Tag, Short> entry : allocMap.entrySet()) {
-            BeSelectionPolicy.Builder builder = new BeSelectionPolicy.Builder()
-                    .needScheduleAvailable().needCheckDiskUsage().addTags(Sets.newHashSet(entry.getKey()))
-                    .setStorageMedium(storageMedium);
-            if (FeConstants.runningUnitTest || Config.allow_replica_on_same_host) {
-                builder.allowOnSameHost();
-            }
-
-            BeSelectionPolicy policy = builder.build();
-            int nextIndex = nextIndexs.get(entry.getKey());
-            List<Long> beIds = selectBackendIdsRoundRobinByPolicy(policy, entry.getValue(), nextIndex);
-            nextIndexs.put(entry.getKey(), nextIndex + beIds.size());
-
-            if (beIds.isEmpty()) {
-                throw new DdlException("Failed to find " + entry.getValue() + " backend(s) for policy: " + policy);
-            }
-            chosenBackendIds.put(entry.getKey(), beIds);
-            totalReplicaNum += beIds.size();
-        }
-        Preconditions.checkState(totalReplicaNum == replicaAlloc.getTotalReplicaNum());
-        return chosenBackendIds;
-    }
-
     /**
      * Select a set of backends for replica creation.
      * The following parameters need to be considered when selecting backends.
      *
      * @param replicaAlloc
+     * @param nextIndexs create tablet round robin next be index, when enable_round_robin_create_tablet
      * @param storageMedium
      * @param isStorageMediumSpecified
      * @param isOnlyForCheck set true if only used for check available backend
@@ -544,7 +473,8 @@ public Map<Tag, List<Long>> getBeIdRoundRobinForReplicaCreation(
      * @throws DdlException
      */
     public Map<Tag, List<Long>> selectBackendIdsForReplicaCreation(
-            ReplicaAllocation replicaAlloc, TStorageMedium storageMedium, boolean isStorageMediumSpecified,
+            ReplicaAllocation replicaAlloc, Map<Tag, Integer> nextIndexs,
+            TStorageMedium storageMedium, boolean isStorageMediumSpecified,
             boolean isOnlyForCheck)
             throws DdlException {
         Map<Long, Backend> copiedBackends = Maps.newHashMap(idToBackendRef);
@@ -561,12 +491,17 @@ public Map<Tag, List<Long>> selectBackendIdsForReplicaCreation(
             List<String> failedEntries = Lists.newArrayList();
 
             for (Map.Entry<Tag, Short> entry : allocMap.entrySet()) {
+                Tag tag = entry.getKey();
                 BeSelectionPolicy.Builder builder = new BeSelectionPolicy.Builder()
                         .needScheduleAvailable().needCheckDiskUsage().addTags(Sets.newHashSet(entry.getKey()))
                         .setStorageMedium(storageMedium);
                 if (FeConstants.runningUnitTest || Config.allow_replica_on_same_host) {
                     builder.allowOnSameHost();
                 }
+                if (Config.enable_round_robin_create_tablet) {
+                    builder.setEnableRoundRobin(true);
+                    builder.setNextRoundRobinIndex(nextIndexs.getOrDefault(tag, -1));
+                }
 
                 BeSelectionPolicy policy = builder.build();
                 List<Long> beIds = selectBackendIdsByPolicy(policy, entry.getValue());
@@ -574,9 +509,16 @@ public Map<Tag, List<Long>> selectBackendIdsForReplicaCreation(
                 // if only for check, no need to retry different storage medium to get backend
                 if (beIds.isEmpty() && storageMedium != null && !isStorageMediumSpecified && !isOnlyForCheck) {
                     storageMedium = (storageMedium == TStorageMedium.HDD) ? TStorageMedium.SSD : TStorageMedium.HDD;
-                    policy = builder.setStorageMedium(storageMedium).build();
+                    builder.setStorageMedium(storageMedium);
+                    if (Config.enable_round_robin_create_tablet) {
+                        builder.setNextRoundRobinIndex(nextIndexs.getOrDefault(tag, -1));
+                    }
+                    policy = builder.build();
                     beIds = selectBackendIdsByPolicy(policy, entry.getValue());
                 }
+                if (Config.enable_round_robin_create_tablet) {
+                    nextIndexs.put(tag, policy.nextRoundRobinIndex);
+                }
                 // after retry different storage medium, it's still empty
                 if (beIds.isEmpty()) {
                     LOG.error("failed backend(s) for policy:" + policy);
@@ -605,7 +547,7 @@ public Map<Tag, List<Long>> selectBackendIdsForReplicaCreation(
     /**
      * Select a set of backends by the given policy.
      *
-     * @param policy
+     * @param policy if policy is enableRoundRobin, will update its nextRoundRobinIndex
      * @param number number of backends which need to be selected. -1 means return as many as possible.
      * @return return #number of backend ids,
      * or empty set if no backends match the policy, or the number of matched backends is less than "number";
@@ -613,50 +555,77 @@ public Map<Tag, List<Long>> selectBackendIdsForReplicaCreation(
     public List<Long> selectBackendIdsByPolicy(BeSelectionPolicy policy, int number) {
         Preconditions.checkArgument(number >= -1);
         List<Backend> candidates = policy.getCandidateBackends(idToBackendRef.values());
-        if ((number != -1 && candidates.size() < number) || candidates.isEmpty()) {
+        if (candidates.size() < number || candidates.isEmpty()) {
             LOG.debug("Not match policy: {}. candidates num: {}, expected: {}", policy, candidates.size(), number);
             return Lists.newArrayList();
         }
+
         // If only need one Backend, just return a random one.
-        if (number == 1) {
+        if (number == 1 && !policy.enableRoundRobin) {
             Collections.shuffle(candidates);
             return Lists.newArrayList(candidates.get(0).getId());
         }
 
-        if (policy.allowOnSameHost) {
-            Collections.shuffle(candidates);
-            if (number == -1) {
-                return candidates.stream().map(b -> b.getId()).collect(Collectors.toList());
-            } else {
-                return candidates.subList(0, number).stream().map(b -> b.getId()).collect(Collectors.toList());
+        boolean hasSameHost = false;
+        if (!policy.allowOnSameHost) {
+            // for each host, random select one backend.
+            Map<String, List<Backend>> backendMaps = Maps.newHashMap();
+            for (Backend backend : candidates) {
+                if (backendMaps.containsKey(backend.getHost())) {
+                    backendMaps.get(backend.getHost()).add(backend);
+                } else {
+                    List<Backend> list = Lists.newArrayList();
+                    list.add(backend);
+                    backendMaps.put(backend.getHost(), list);
+                }
             }
-        }
 
-        // for each host, random select one backend.
-        Map<String, List<Backend>> backendMaps = Maps.newHashMap();
-        for (Backend backend : candidates) {
-            if (backendMaps.containsKey(backend.getHost())) {
-                backendMaps.get(backend.getHost()).add(backend);
-            } else {
-                List<Backend> list = Lists.newArrayList();
-                list.add(backend);
-                backendMaps.put(backend.getHost(), list);
+            candidates.clear();
+            for (List<Backend> list : backendMaps.values()) {
+                if (list.size() > 1) {
+                    Collections.shuffle(list);
+                    hasSameHost = true;
+                }
+                candidates.add(list.get(0));
             }
         }
-        candidates.clear();
-        for (List<Backend> list : backendMaps.values()) {
-            Collections.shuffle(list);
-            candidates.add(list.get(0));
-        }
-        if (number != -1 && candidates.size() < number) {
+
+        if (candidates.size() < number) {
             LOG.debug("Not match policy: {}. candidates num: {}, expected: {}", policy, candidates.size(), number);
             return Lists.newArrayList();
         }
-        Collections.shuffle(candidates);
-        if (number != -1) {
-            return candidates.subList(0, number).stream().map(b -> b.getId()).collect(Collectors.toList());
+
+        if (policy.enableRoundRobin) {
+            if (!policy.allowOnSameHost && hasSameHost) {
+                // not allow same host and has same host,
+                // then we compare them with their host
+                Collections.sort(candidates, new BeHostComparator());
+            } else {
+                Collections.sort(candidates, new BeIdComparator());
+            }
+
+            if (policy.nextRoundRobinIndex < 0) {
+                policy.nextRoundRobinIndex = new SecureRandom().nextInt(candidates.size());
+            }
+
+            int realIndex = policy.nextRoundRobinIndex % candidates.size();
+            List<Long> partialOrderList = new ArrayList<Long>();
+            partialOrderList.addAll(candidates.subList(realIndex, candidates.size())
+                    .stream().map(Backend::getId).collect(Collectors.toList()));
+            partialOrderList.addAll(candidates.subList(0, realIndex)
+                    .stream().map(Backend::getId).collect(Collectors.toList()));
+
+            List<Long> result = number == -1 ? partialOrderList : partialOrderList.subList(0, number);
+            policy.nextRoundRobinIndex = realIndex + result.size();
+
+            return result;
         } else {
-            return candidates.stream().map(b -> b.getId()).collect(Collectors.toList());
+            Collections.shuffle(candidates);
+            if (number != -1) {
+                return candidates.subList(0, number).stream().map(Backend::getId).collect(Collectors.toList());
+            } else {
+                return candidates.stream().map(Backend::getId).collect(Collectors.toList());
+            }
         }
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index 64e92b35c83079..d361777fdd56c3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -39,6 +39,7 @@
 import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.fs.FileSystemFactory;
 import org.apache.doris.persist.EditLog;
+import org.apache.doris.resource.Tag;
 import org.apache.doris.system.SystemInfoService;
 import org.apache.doris.thrift.TStorageMedium;
 
@@ -54,6 +55,7 @@
 import org.junit.Test;
 
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.zip.Adler32;
@@ -153,12 +155,14 @@ public void setUp() throws Exception {
 
         new Expectations() {
             {
-                systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any, (TStorageMedium) any,
-                        false, true);
+                systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any,
+                        Maps.newHashMap(), (TStorageMedium) any, false, true);
                 minTimes = 0;
                 result = new Delegate() {
                     public synchronized List<Long> selectBackendIdsForReplicaCreation(
-                            ReplicaAllocation replicaAlloc, String clusterName, TStorageMedium medium) {
+                            ReplicaAllocation replicaAlloc, Map<Tag, Integer> nextIndexs,
+                            TStorageMedium medium, boolean isStorageMediumSpecified,
+                            boolean isOnlyForCheck) {
                         List<Long> beIds = Lists.newArrayList();
                         beIds.add(CatalogMocker.BACKEND1_ID);
                         beIds.add(CatalogMocker.BACKEND2_ID);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java
index 14367ea731ec96..c53715cd817ed2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ReplicaAllocationTest.java
@@ -52,7 +52,8 @@ public class ReplicaAllocationTest {
     public void setUp() throws DdlException {
         new Expectations() {
             {
-                systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any, (TStorageMedium) any, false, true);
+                systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any, Maps.newHashMap(),
+                        (TStorageMedium) any, false, true);
                 minTimes = 0;
                 result = new Delegate() {
                     Map<Tag, List<Long>> selectBackendIdsForReplicaCreation() {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java
index bf57f21f02fe93..61228c821a8e02 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java
@@ -150,7 +150,7 @@ public void setUp() throws Exception {
                 result = execPlanFragmentParams;
 
                 systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any,
-                        (TStorageMedium) any, false, true);
+                        Maps.newHashMap(), (TStorageMedium) any, false, true);
                 minTimes = 0;
                 result = backendIds;
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java
index 7c5556e8cf2d49..22e12b37da392c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java
@@ -404,7 +404,7 @@ public void testSelectBackendIdsForReplicaCreation() throws Exception {
         Map<Long, Integer> beCounterMap = Maps.newHashMap();
         for (int i = 0; i < 10000; ++i) {
             Map<Tag, List<Long>> res = infoService.selectBackendIdsForReplicaCreation(replicaAlloc,
-                    TStorageMedium.HDD, false, false);
+                    Maps.newHashMap(), TStorageMedium.HDD, false, false);
             Assert.assertEquals(3, res.get(Tag.DEFAULT_BACKEND_TAG).size());
             for (Long beId : res.get(Tag.DEFAULT_BACKEND_TAG)) {
                 beCounterMap.put(beId, beCounterMap.getOrDefault(beId, 0) + 1);

From c82155614bfd6fb3b586c26a76ce7500aeb905ba Mon Sep 17 00:00:00 2001
From: Guangming Lu <71873108+LuGuangming@users.noreply.github.com>
Date: Tue, 26 Dec 2023 18:45:22 +0800
Subject: [PATCH 045/299] [Bug] optimize the collection import Lists dependency
 packge #28523 (#28579)

---
 .../doris/nereids/trees/plans/commands/DeleteFromCommand.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
index 6e9ee4ca0144d2..976200e9fe40ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromCommand.java
@@ -54,7 +54,7 @@
 import org.apache.doris.qe.StmtExecutor;
 
 import com.google.common.base.Preconditions;
-import org.apache.hadoop.util.Lists;
+import com.google.common.collect.Lists;
 
 import java.util.List;
 import java.util.Optional;

From ffa3c319ddfca2934eed5c683b906b9b4b93f34c Mon Sep 17 00:00:00 2001
From: bobhan1 <bh2444151092@outlook.com>
Date: Tue, 26 Dec 2023 21:43:14 +0800
Subject: [PATCH 046/299] [fix](regression) Fix flaky test
 test_partial_update_2pc_schema_change (#29037)

---
 .../test_partial_update_2pc_schema_change.groovy         | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
index 45fa34d60a7357..6f000786fa9cc1 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_2pc_schema_change.groovy
@@ -72,6 +72,7 @@ suite("test_partial_update_2pc_schema_change", "p0") {
                 file 'concurrency_update3.csv'
                 time 10000 // limit inflight 10s
             }
+            sql "sync;"
             qt_sql """ select * from ${tableName} order by k1;"""
 
 
@@ -153,7 +154,7 @@ suite("test_partial_update_2pc_schema_change", "p0") {
                     assertEquals("success", json.Status.toLowerCase())
                 }
             }
-
+            sql "sync;"
             sql """ alter table ${tableName} modify column v2 varchar(40);"""
             wait_for_schema_change()
 
@@ -165,7 +166,7 @@ suite("test_partial_update_2pc_schema_change", "p0") {
 
             sql """ alter table ${tableName} rename column v4 renamed_v4;"""
             wait_for_schema_change()
-
+            sql "sync;"
             streamLoad {
                 table "${tableName}"
                 set 'column_separator', ','
@@ -176,11 +177,11 @@ suite("test_partial_update_2pc_schema_change", "p0") {
                 file 'concurrency_update2.csv'
                 time 10000 // limit inflight 10s
             }
-
+            sql "sync;"
             qt_sql """ select * from ${tableName} order by k1;"""
 
             do_streamload_2pc(txnId, "commit", tableName)
-            
+            sql "sync;"
             qt_sql """ select * from ${tableName} order by k1;"""
 
             sql "drop table if exists ${tableName};"

From a8e6676640f7fcf6baa81023994a365b97e82d98 Mon Sep 17 00:00:00 2001
From: Guangming Lu <71873108+LuGuangming@users.noreply.github.com>
Date: Tue, 26 Dec 2023 21:59:47 +0800
Subject: [PATCH 047/299] [Bug](security) BE download_files function exists log
 print sensitive msg #28592 (#28594)

---
 be/src/olap/task/engine_clone_task.cpp | 15 +++++++++++----
 be/src/olap/task/engine_clone_task.h   |  2 ++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp
index 581cd3515c204a..068e87233547d1 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -31,6 +31,7 @@
 #include <memory>
 #include <mutex>
 #include <ostream>
+#include <regex>
 #include <set>
 #include <shared_mutex>
 #include <system_error>
@@ -390,7 +391,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
         status = _download_files(&data_dir, remote_url_prefix, local_data_path);
         if (!status.ok()) [[unlikely]] {
             LOG_WARNING("failed to download snapshot from remote BE")
-                    .tag("url", remote_url_prefix)
+                    .tag("url", _mask_token(remote_url_prefix))
                     .error(status);
             continue; // Try another BE
         }
@@ -528,11 +529,11 @@ Status EngineCloneTask::_download_files(DataDir* data_dir, const std::string& re
 
         std::string local_file_path = local_path + "/" + file_name;
 
-        LOG(INFO) << "clone begin to download file from: " << remote_file_url
+        LOG(INFO) << "clone begin to download file from: " << _mask_token(remote_file_url)
                   << " to: " << local_file_path << ". size(B): " << file_size
                   << ", timeout(s): " << estimate_timeout;
 
-        auto download_cb = [&remote_file_url, estimate_timeout, &local_file_path,
+        auto download_cb = [this, &remote_file_url, estimate_timeout, &local_file_path,
                             file_size](HttpClient* client) {
             RETURN_IF_ERROR(client->init(remote_file_url));
             client->set_timeout_ms(estimate_timeout * 1000);
@@ -548,7 +549,8 @@ Status EngineCloneTask::_download_files(DataDir* data_dir, const std::string& re
             }
             if (local_file_size != file_size) {
                 LOG(WARNING) << "download file length error"
-                             << ", remote_path=" << remote_file_url << ", file_size=" << file_size
+                             << ", remote_path=" << _mask_token(remote_file_url)
+                             << ", file_size=" << file_size
                              << ", local_file_size=" << local_file_size;
                 return Status::InternalError("downloaded file size is not equal");
             }
@@ -829,4 +831,9 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet,
     // TODO(plat1ko): write cooldown meta to remote if this replica is cooldown replica
 }
 
+std::string EngineCloneTask::_mask_token(const std::string& str) {
+    std::regex pattern("token=[\\w|-]+");
+    return regex_replace(str, pattern, "token=******");
+}
+
 } // namespace doris
diff --git a/be/src/olap/task/engine_clone_task.h b/be/src/olap/task/engine_clone_task.h
index 4f502ea50fd87f..dd33f263e18507 100644
--- a/be/src/olap/task/engine_clone_task.h
+++ b/be/src/olap/task/engine_clone_task.h
@@ -86,6 +86,8 @@ class EngineCloneTask : public EngineTask {
 
     Status _release_snapshot(const std::string& ip, int port, const std::string& snapshot_path);
 
+    std::string _mask_token(const std::string& str);
+
 private:
     const TCloneReq& _clone_req;
     vector<TTabletInfo>* _tablet_infos = nullptr;

From f6850f8cdbdf9d1669e844d93664c81ff9fcd32c Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Tue, 26 Dec 2023 22:20:49 +0800
Subject: [PATCH 048/299] Revert "[feature](load) enable memtable on sink node
 by default (#28963)" (#29090)

This reverts commit 17917a0adb57ac337dfd41e8853b1aa87c9d5acd.
---
 docs/en/docs/advanced/variables.md                              | 2 +-
 docs/zh-CN/docs/advanced/variables.md                           | 2 +-
 .../src/main/java/org/apache/doris/qe/SessionVariable.java      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/docs/advanced/variables.md b/docs/en/docs/advanced/variables.md
index e339b1893dc789..030e228f5234aa 100644
--- a/docs/en/docs/advanced/variables.md
+++ b/docs/en/docs/advanced/variables.md
@@ -689,7 +689,7 @@ Note that the comment must start with /*+ and can only follow the SELECT.
 * `enable_memtable_on_sink_node`
 
   <version since="2.1.0">
-  Whether to enable MemTable on DataSink node when loading data, default is true.
+  Whether to enable MemTable on DataSink node when loading data, default is false.
   </version>
 
   Build MemTable on DataSink node, and send segments to other backends through brpc streaming.
diff --git a/docs/zh-CN/docs/advanced/variables.md b/docs/zh-CN/docs/advanced/variables.md
index 510ad85c36e9ff..2a697b01bf7de4 100644
--- a/docs/zh-CN/docs/advanced/variables.md
+++ b/docs/zh-CN/docs/advanced/variables.md
@@ -677,7 +677,7 @@ try (Connection conn = DriverManager.getConnection("jdbc:mysql://127.0.0.1:9030/
 * `enable_memtable_on_sink_node`
 
   <version since="2.1.0">
-  是否在数据导入中启用 MemTable 前移，默认为 true
+  是否在数据导入中启用 MemTable 前移，默认为 false
   </version>
 
   在 DataSink 节点上构建 MemTable，并通过 brpc streaming 发送 segment 到其他 BE。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 23813f28dde367..35545059901d4f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1342,7 +1342,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
     public boolean truncateCharOrVarcharColumns = false;
 
     @VariableMgr.VarAttr(name = ENABLE_MEMTABLE_ON_SINK_NODE, needForward = true)
-    public boolean enableMemtableOnSinkNode = true;
+    public boolean enableMemtableOnSinkNode = false;
 
     @VariableMgr.VarAttr(name = LOAD_STREAM_PER_NODE)
     public int loadStreamPerNode = 60;

From a4e69f7272ea2c99c6db552d6b7e9150a0e3ed00 Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Wed, 27 Dec 2023 09:59:34 +0800
Subject: [PATCH 049/299] [improvement](statistics)Remove retry load when load
 stats cache fail (#28904)

Remove retry load when load stats cache fail. This case usually happens when BE is down or BE OOM, retry doesn't work in these cases and may increase BE work load.
---
 .../ColumnStatisticsCacheLoader.java          | 42 +------------------
 .../doris/statistics/StatisticsCache.java     |  4 --
 2 files changed, 2 insertions(+), 44 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
index 22f0cad9d00a8c..08a410b8599265 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/ColumnStatisticsCacheLoader.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.statistics;
 
-import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.ThreadPoolManager;
 import org.apache.doris.qe.InternalQueryExecutionException;
@@ -64,7 +63,8 @@ private Optional<ColumnStatistic> loadFromStatsTable(StatisticsCacheKey key) {
         try {
             columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
         } catch (InternalQueryExecutionException e) {
-            retryLoad(key);
+            LOG.info("Failed to load stats for table {} column {}. Reason:{}",
+                    key.tableId, key.colName, e.getMessage());
             return Optional.empty();
         }
         ColumnStatistic columnStatistics;
@@ -80,42 +80,4 @@ private Optional<ColumnStatistic> loadFromStatsTable(StatisticsCacheKey key) {
             return Optional.of(columnStatistics);
         }
     }
-
-    private void retryLoad(StatisticsCacheKey key) {
-        singleThreadPool.submit(new RetryTask(key, 1));
-    }
-
-    private static class RetryTask implements Runnable {
-        StatisticsCacheKey key;
-        int retryTimes;
-
-        public RetryTask(StatisticsCacheKey key, int retryTimes) {
-            this.key = key;
-            this.retryTimes = retryTimes;
-        }
-
-        @Override
-        public void run() {
-            List<ResultRow> columnResults = null;
-            try {
-                columnResults = StatisticsRepository.loadColStats(key.tableId, key.idxId, key.colName);
-            } catch (InternalQueryExecutionException e) {
-                if (this.retryTimes < StatisticConstants.LOAD_RETRY_TIMES) {
-                    retryTimes++;
-                    singleThreadPool.submit(this);
-                }
-                return;
-            }
-            ColumnStatistic columnStatistics;
-            try {
-                columnStatistics = StatisticsUtil.deserializeToColumnStatistics(columnResults);
-            } catch (Exception e) {
-                LOG.warn("Exception to deserialize column statistics", e);
-                return;
-            }
-            if (columnStatistics != null) {
-                Env.getCurrentEnv().getStatisticsCache().putCache(key, columnStatistics);
-            }
-        }
-    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index d4b91b07364ebd..fbec9a60fa00e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -39,9 +39,7 @@
 import org.apache.logging.log4j.Logger;
 
 import java.time.Duration;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -201,7 +199,6 @@ private void doPreHeat() {
         if (CollectionUtils.isEmpty(recentStatsUpdatedCols)) {
             return;
         }
-        Map<StatisticsCacheKey, ColumnStatistic> keyToColStats = new HashMap<>();
         for (ResultRow r : recentStatsUpdatedCols) {
             try {
                 StatsId statsId = new StatsId(r);
@@ -211,7 +208,6 @@ private void doPreHeat() {
                 final StatisticsCacheKey k =
                         new StatisticsCacheKey(tblId, idxId, colId);
                 final ColumnStatistic c = ColumnStatistic.fromResultRow(r);
-                keyToColStats.put(k, c);
                 putCache(k, c);
             } catch (Throwable t) {
                 LOG.warn("Error when preheating stats cache", t);

From 6d26aca4cad3757bbdb243c0d5f24729ba09c674 Mon Sep 17 00:00:00 2001
From: zhiqiang <seuhezhiqiang@163.com>
Date: Wed, 27 Dec 2023 10:06:01 +0800
Subject: [PATCH 050/299] [fix](pipeline) sort_merge should throw exception in
 has_next_block if got failed status (#29076)

Test in regression-test/suites/datatype_p0/decimalv3/test_decimalv3_overflow.groovy::249 sometimes failed when there are multiple BEs and FE process report status slowly for some reason.

explain select k1, k2, k1 * k2 from test_decimal128_overflow2 order by 1,2,3
--------------

+----------------------------------------------------------------------------------------------------------------------------+
| Explain String(Nereids Planner)                                                                                            |
+----------------------------------------------------------------------------------------------------------------------------+
| PLAN FRAGMENT 0                                                                                                            |
|   OUTPUT EXPRS:                                                                                                            |
|     k1[#5]                                                                                                                 |
|     k2[#6]                                                                                                                 |
|     (k1 * k2)[#7]                                                                                                          |
|   PARTITION: UNPARTITIONED                                                                                                 |
|                                                                                                                            |
|   HAS_COLO_PLAN_NODE: false                                                                                                |
|                                                                                                                            |
|   VRESULT SINK                                                                                                             |
|      MYSQL_PROTOCAL                                                                                                        |
|                                                                                                                            |
|   111:VMERGING-EXCHANGE                                                                                                    |
|      offset: 0                                                                                                             |
|                                                                                                                            |
| PLAN FRAGMENT 1                                                                                                            |
|                                                                                                                            |
|   PARTITION: HASH_PARTITIONED: k1[#0], k2[#1]                                                                              |
|                                                                                                                            |
|   HAS_COLO_PLAN_NODE: false                                                                                                |
|                                                                                                                            |
|   STREAM DATA SINK                                                                                                         |
|     EXCHANGE ID: 111                                                                                                       |
|     UNPARTITIONED                                                                                                          |
|                                                                                                                            |
|   108:VSORT                                                                                                                |
|   |  order by: k1[#5] ASC, k2[#6] ASC, (k1 * k2)[#7] ASC                                                                   |
|   |  offset: 0                                                                                                             |
|   |                                                                                                                        |
|   102:VOlapScanNode                                                                                                        |
|      TABLE: regression_test_datatype_p0_decimalv3.test_decimal128_overflow2(test_decimal128_overflow2), PREAGGREGATION: ON |
|      partitions=1/1 (test_decimal128_overflow2), tablets=8/8, tabletList=22841,22843,22845 ...                             |
|      cardinality=6, avgRowSize=0.0, numNodes=1                                                                             |
|      pushAggOp=NONE                                                                                                        |
|      projections: k1[#0], k2[#1], (k1[#0] * k2[#1])                                                                        |
|      project output tuple id: 1                                                                                            |
+----------------------------------------------------------------------------------------------------------------------------+
36 rows in set (0.03 sec)
Why failed:

Multiple BEs
Fragments 0 and 1 are MUST on different BEs
Pipeline task of VOlapScanNode which executes k1*k2 failed sets query status to cancelled
Pipeline task of VSort call try close, send Cancelled status to VMergeExchange
sort_curso did not throw exception when it meets error
---
 be/src/vec/core/sort_cursor.h             |  2 ++
 be/src/vec/runtime/vsorted_run_merger.cpp | 14 +++++++++-----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/be/src/vec/core/sort_cursor.h b/be/src/vec/core/sort_cursor.h
index 71db3428541cff..b43449b50b0472 100644
--- a/be/src/vec/core/sort_cursor.h
+++ b/be/src/vec/core/sort_cursor.h
@@ -244,6 +244,8 @@ struct BlockSupplierSortCursorImpl : public MergeSortCursorImpl {
             }
             MergeSortCursorImpl::reset(_block);
             return status.ok();
+        } else if (!status.ok()) {
+            throw std::runtime_error(status.msg());
         }
         return false;
     }
diff --git a/be/src/vec/runtime/vsorted_run_merger.cpp b/be/src/vec/runtime/vsorted_run_merger.cpp
index 6d2f68db6287df..3637bd54aedb71 100644
--- a/be/src/vec/runtime/vsorted_run_merger.cpp
+++ b/be/src/vec/runtime/vsorted_run_merger.cpp
@@ -69,12 +69,16 @@ void VSortedRunMerger::init_timers(RuntimeProfile* profile) {
 }
 
 Status VSortedRunMerger::prepare(const vector<BlockSupplier>& input_runs) {
-    for (const auto& supplier : input_runs) {
-        if (_use_sort_desc) {
-            _cursors.emplace_back(supplier, _desc);
-        } else {
-            _cursors.emplace_back(supplier, _ordering_expr, _is_asc_order, _nulls_first);
+    try {
+        for (const auto& supplier : input_runs) {
+            if (_use_sort_desc) {
+                _cursors.emplace_back(supplier, _desc);
+            } else {
+                _cursors.emplace_back(supplier, _ordering_expr, _is_asc_order, _nulls_first);
+            }
         }
+    } catch (const std::exception& e) {
+        return Status::Cancelled(e.what());
     }
 
     for (auto& _cursor : _cursors) {

From 58e7ad82b504897882b90d21cbe0aeef94186c7f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= <jianxie0@gmail.com>
Date: Wed, 27 Dec 2023 10:40:44 +0800
Subject: [PATCH 051/299] [feature](Nereids): support infer join when comapring
 mv (#28988)

---
 .../jobs/joinorder/hypergraph/HyperGraph.java | 171 +---------
 .../jobs/joinorder/hypergraph/edge/Edge.java  |  40 ++-
 .../joinorder/hypergraph/edge/JoinEdge.java   |  12 +
 .../exploration/mv/ComparisonResult.java      |  41 ++-
 .../exploration/mv/HyperGraphComparator.java  | 299 ++++++++++++++++++
 .../rules/exploration/mv/StructInfo.java      |   3 +-
 .../trees/plans/logical/LogicalJoin.java      |   5 +
 .../hypergraph/CompareOuterJoinTest.java      |  25 +-
 .../joinorder/hypergraph/InferJoinTest.java   | 173 ++++++++++
 .../hypergraph/PullupExpressionTest.java      |   9 +-
 .../exploration/mv/BuildStructInfoTest.java   |   4 +-
 11 files changed, 579 insertions(+), 203 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
index 7bd33c64b3c593..f094efe7180cd0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java
@@ -27,8 +27,6 @@
 import org.apache.doris.nereids.jobs.joinorder.hypergraph.node.StructInfoNode;
 import org.apache.doris.nereids.memo.Group;
 import org.apache.doris.nereids.memo.GroupExpression;
-import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
-import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
 import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
 import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.Expression;
@@ -44,20 +42,14 @@
 import org.apache.doris.nereids.util.PlanUtils;
 
 import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
 import java.util.BitSet;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -272,11 +264,11 @@ private void makeFilterConflictRules(JoinEdge joinEdge) {
         filterEdges.forEach(e -> {
             if (LongBitmap.isSubset(e.getReferenceNodes(), leftSubNodes)
                     && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(joinEdge.getJoinType())) {
-                e.addRejectEdge(joinEdge);
+                e.addLeftRejectEdge(joinEdge);
             }
             if (LongBitmap.isSubset(e.getReferenceNodes(), rightSubNodes)
                     && !PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(joinEdge.getJoinType())) {
-                e.addRejectEdge(joinEdge);
+                e.addRightRejectEdge(joinEdge);
             }
         });
     }
@@ -293,11 +285,11 @@ private void makeJoinConflictRules(JoinEdge edgeB) {
             JoinEdge childA = joinEdges.get(i);
             if (!JoinType.isAssoc(childA.getJoinType(), edgeB.getJoinType())) {
                 leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getLeftSubNodes(joinEdges));
-                childA.addRejectEdge(edgeB);
+                childA.addLeftRejectEdge(edgeB);
             }
             if (!JoinType.isLAssoc(childA.getJoinType(), edgeB.getJoinType())) {
                 leftRequired = LongBitmap.newBitmapUnion(leftRequired, childA.getRightSubNodes(joinEdges));
-                childA.addRejectEdge(edgeB);
+                childA.addLeftRejectEdge(edgeB);
             }
         }
 
@@ -305,11 +297,11 @@ private void makeJoinConflictRules(JoinEdge edgeB) {
             JoinEdge childA = joinEdges.get(i);
             if (!JoinType.isAssoc(edgeB.getJoinType(), childA.getJoinType())) {
                 rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getRightSubNodes(joinEdges));
-                childA.addRejectEdge(edgeB);
+                childA.addRightRejectEdge(edgeB);
             }
             if (!JoinType.isRAssoc(edgeB.getJoinType(), childA.getJoinType())) {
                 rightRequired = LongBitmap.newBitmapUnion(rightRequired, childA.getLeftSubNodes(joinEdges));
-                childA.addRejectEdge(edgeB);
+                childA.addRightRejectEdge(edgeB);
             }
         }
         edgeB.setLeftExtendedNodes(leftRequired);
@@ -597,157 +589,6 @@ public int edgeSize() {
         return joinEdges.size() + filterEdges.size();
     }
 
-    /**
-     * compare hypergraph
-     *
-     * @param viewHG the compared hyper graph
-     * @return Comparison result
-     */
-    public ComparisonResult isLogicCompatible(HyperGraph viewHG, LogicalCompatibilityContext ctx) {
-        // 1 try to construct a map which can be mapped from edge to edge
-        Map<Edge, Edge> queryToView = constructMapWithNode(viewHG, ctx.getQueryToViewNodeIDMapping());
-
-        // 2. compare them by expression and extract residual expr
-        ComparisonResult.Builder builder = new ComparisonResult.Builder();
-        ComparisonResult edgeCompareRes = compareEdgesWithExpr(queryToView, ctx.getQueryToViewEdgeExpressionMapping());
-        if (edgeCompareRes.isInvalid()) {
-            return ComparisonResult.INVALID;
-        }
-        builder.addComparisonResult(edgeCompareRes);
-
-        // 3. pull join edge of view is no sense, so reject them
-        if (!queryToView.values().containsAll(viewHG.joinEdges)) {
-            return ComparisonResult.INVALID;
-        }
-
-        // 4. process residual edges
-        List<Expression> residualQueryJoin =
-                processOrphanEdges(Sets.difference(Sets.newHashSet(joinEdges), queryToView.keySet()));
-        if (residualQueryJoin == null) {
-            return ComparisonResult.INVALID;
-        }
-        builder.addQueryExpressions(residualQueryJoin);
-
-        List<Expression> residualQueryFilter =
-                processOrphanEdges(Sets.difference(Sets.newHashSet(filterEdges), queryToView.keySet()));
-        if (residualQueryFilter == null) {
-            return ComparisonResult.INVALID;
-        }
-        builder.addQueryExpressions(residualQueryFilter);
-
-        List<Expression> residualViewFilter =
-                processOrphanEdges(
-                        Sets.difference(Sets.newHashSet(viewHG.filterEdges), Sets.newHashSet(queryToView.values())));
-        if (residualViewFilter == null) {
-            return ComparisonResult.INVALID;
-        }
-        builder.addViewExpressions(residualViewFilter);
-
-        return builder.build();
-    }
-
-    private List<Expression> processOrphanEdges(Set<Edge> edges) {
-        List<Expression> expressions = new ArrayList<>();
-        for (Edge edge : edges) {
-            if (!edge.canPullUp()) {
-                return null;
-            }
-            expressions.addAll(edge.getExpressions());
-        }
-        return expressions;
-    }
-
-    private Map<Edge, Edge> constructMapWithNode(HyperGraph viewHG, Map<Integer, Integer> nodeMap) {
-        // TODO use hash map to reduce loop
-        Map<Edge, Edge> joinEdgeMap = joinEdges.stream().map(qe -> {
-            Optional<JoinEdge> viewEdge = viewHG.joinEdges.stream()
-                    .filter(ve -> compareEdgeWithNode(qe, ve, nodeMap)).findFirst();
-            return Pair.of(qe, viewEdge);
-        }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
-        Map<Edge, Edge> filterEdgeMap = filterEdges.stream().map(qe -> {
-            Optional<FilterEdge> viewEdge = viewHG.filterEdges.stream()
-                    .filter(ve -> compareEdgeWithNode(qe, ve, nodeMap)).findFirst();
-            return Pair.of(qe, viewEdge);
-        }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
-        return ImmutableMap.<Edge, Edge>builder().putAll(joinEdgeMap).putAll(filterEdgeMap).build();
-    }
-
-    private boolean compareEdgeWithNode(Edge t, Edge o, Map<Integer, Integer> nodeMap) {
-        if (t instanceof FilterEdge && o instanceof FilterEdge) {
-            return compareEdgeWithFilter((FilterEdge) t, (FilterEdge) o, nodeMap);
-        } else if (t instanceof JoinEdge && o instanceof JoinEdge) {
-            return compareJoinEdge((JoinEdge) t, (JoinEdge) o, nodeMap);
-        }
-        return false;
-    }
-
-    private boolean compareEdgeWithFilter(FilterEdge t, FilterEdge o, Map<Integer, Integer> nodeMap) {
-        long tChild = t.getReferenceNodes();
-        long oChild = o.getReferenceNodes();
-        return compareNodeMap(tChild, oChild, nodeMap);
-    }
-
-    private boolean compareJoinEdge(JoinEdge t, JoinEdge o, Map<Integer, Integer> nodeMap) {
-        long tLeft = t.getLeftExtendedNodes();
-        long tRight = t.getRightExtendedNodes();
-        long oLeft = o.getLeftExtendedNodes();
-        long oRight = o.getRightExtendedNodes();
-        if (!t.getJoinType().equals(o.getJoinType()) && !t.getJoinType().swap().equals(o.getJoinType())) {
-            return false;
-        }
-        boolean matched = false;
-        if (t.getJoinType().swap().equals(o.getJoinType())) {
-            matched |= compareNodeMap(tRight, oLeft, nodeMap) && compareNodeMap(tLeft, oRight, nodeMap);
-        }
-        matched |= compareNodeMap(tLeft, oLeft, nodeMap) && compareNodeMap(tRight, oRight, nodeMap);
-        return matched;
-    }
-
-    private boolean compareNodeMap(long bitmap1, long bitmap2, Map<Integer, Integer> nodeIDMap) {
-        long newBitmap1 = LongBitmap.newBitmap();
-        for (int i : LongBitmap.getIterator(bitmap1)) {
-            int mappedI = nodeIDMap.getOrDefault(i, 0);
-            newBitmap1 = LongBitmap.set(newBitmap1, mappedI);
-        }
-        return bitmap2 == newBitmap1;
-    }
-
-    private ComparisonResult compareEdgesWithExpr(Map<Edge, Edge> queryToViewedgeMap,
-            Map<Expression, Expression> queryToView) {
-        ComparisonResult.Builder builder = new ComparisonResult.Builder();
-        for (Entry<Edge, Edge> e : queryToViewedgeMap.entrySet()) {
-            ComparisonResult res = compareEdgeWithExpr(e.getKey(), e.getValue(), queryToView);
-            if (res.isInvalid()) {
-                return ComparisonResult.INVALID;
-            }
-            builder.addComparisonResult(res);
-        }
-        return builder.build();
-    }
-
-    private ComparisonResult compareEdgeWithExpr(Edge query, Edge view, Map<Expression, Expression> queryToView) {
-        Set<? extends Expression> queryExprSet = query.getExpressionSet();
-        Set<? extends Expression> viewExprSet = view.getExpressionSet();
-
-        Set<Expression> equalViewExpr = new HashSet<>();
-        List<Expression> residualQueryExpr = new ArrayList<>();
-        for (Expression queryExpr : queryExprSet) {
-            if (queryToView.containsKey(queryExpr) && viewExprSet.contains(queryToView.get(queryExpr))) {
-                equalViewExpr.add(queryToView.get(queryExpr));
-            } else {
-                residualQueryExpr.add(queryExpr);
-            }
-        }
-        List<Expression> residualViewExpr = ImmutableList.copyOf(Sets.difference(viewExprSet, equalViewExpr));
-        if (!residualViewExpr.isEmpty() && !view.canPullUp()) {
-            return ComparisonResult.INVALID;
-        }
-        if (!residualQueryExpr.isEmpty() && !query.canPullUp()) {
-            return ComparisonResult.INVALID;
-        }
-        return new ComparisonResult(residualQueryExpr, residualViewExpr);
-    }
-
     /**
      * For the given hyperGraph, make a textual representation in the form
      * of a dotty graph. You can save this to a file and then use Graphviz
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
index c47300922b5191..9bdb9ac272fb44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/Edge.java
@@ -25,6 +25,7 @@
 import com.google.common.collect.ImmutableSet;
 
 import java.util.BitSet;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -53,7 +54,8 @@ public abstract class Edge {
     // record all sub nodes behind in this operator. It's T function in paper
     private final long subTreeNodes;
 
-    private long rejectNodes = 0;
+    private final Set<JoinEdge> leftRejectEdges;
+    private final Set<JoinEdge> rightRejectEdges;
 
     /**
      * Create simple edge.
@@ -69,14 +71,36 @@ public abstract class Edge {
         this.leftExtendedNodes = leftRequiredNodes;
         this.rightExtendedNodes = rightRequiredNodes;
         this.subTreeNodes = subTreeNodes;
+        this.leftRejectEdges = new HashSet<>();
+        this.rightRejectEdges = new HashSet<>();
     }
 
     public boolean isSimple() {
         return LongBitmap.getCardinality(leftExtendedNodes) == 1 && LongBitmap.getCardinality(rightExtendedNodes) == 1;
     }
 
-    public void addRejectEdge(Edge edge) {
-        rejectNodes = LongBitmap.newBitmapUnion(edge.getReferenceNodes(), rejectNodes);
+    public void addLeftRejectEdge(JoinEdge edge) {
+        leftRejectEdges.add(edge);
+    }
+
+    public void addRightRejectEdge(JoinEdge edge) {
+        rightRejectEdges.add(edge);
+    }
+
+    public void addLeftRejectEdges(Set<JoinEdge> edge) {
+        leftRejectEdges.addAll(edge);
+    }
+
+    public void addRightRejectEdges(Set<JoinEdge> edge) {
+        rightRejectEdges.addAll(edge);
+    }
+
+    public Set<JoinEdge> getLeftRejectEdge() {
+        return ImmutableSet.copyOf(leftRejectEdges);
+    }
+
+    public Set<JoinEdge> getRightRejectEdge() {
+        return ImmutableSet.copyOf(rightRejectEdges);
     }
 
     public void addLeftExtendNode(long left) {
@@ -183,16 +207,6 @@ public Set<? extends Expression> getExpressionSet() {
         return ImmutableSet.copyOf(getExpressions());
     }
 
-    public boolean canPullUp() {
-        // Only inner join and filter with none rejectNodes can be pull up
-        return rejectNodes == 0
-                && !(this instanceof JoinEdge && !((JoinEdge) this).getJoinType().isInnerJoin());
-    }
-
-    public long getRejectNodes() {
-        return rejectNodes;
-    }
-
     public Expression getExpression(int i) {
         return getExpressions().get(i);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java
index 81e80bee85fe37..635bf91f364631 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/edge/JoinEdge.java
@@ -45,6 +45,18 @@ public JoinEdge(LogicalJoin<? extends Plan, ? extends Plan> join, int index,
         this.join = join;
     }
 
+    /**
+     * swap the edge
+     */
+    public JoinEdge swap() {
+        JoinEdge swapEdge = new
+                JoinEdge(join.swap(), getIndex(), getRightChildEdges(),
+                getLeftChildEdges(), getSubTreeNodes(), getRightRequiredNodes(), getLeftRequiredNodes());
+        swapEdge.addLeftRejectEdges(getLeftRejectEdge());
+        swapEdge.addRightRejectEdges(getRightRejectEdge());
+        return swapEdge;
+    }
+
     public JoinType getJoinType() {
         return join.getJoinType();
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
index cc8284f33e6824..8836745465ec02 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
@@ -18,29 +18,36 @@
 package org.apache.doris.nereids.rules.exploration.mv;
 
 import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
 /**
  * comparison result of view and query
  */
 public class ComparisonResult {
-    public static final ComparisonResult INVALID = new ComparisonResult(ImmutableList.of(), ImmutableList.of(), false);
-    public static final ComparisonResult EMPTY = new ComparisonResult(ImmutableList.of(), ImmutableList.of(), true);
+    public static final ComparisonResult INVALID =
+            new ComparisonResult(ImmutableList.of(), ImmutableList.of(), ImmutableSet.of(), false);
     private final boolean valid;
     private final List<Expression> viewExpressions;
     private final List<Expression> queryExpressions;
+    private final Set<Set<Slot>> viewNoNullableSlot;
 
-    public ComparisonResult(List<Expression> queryExpressions, List<Expression> viewExpressions) {
-        this(queryExpressions, viewExpressions, true);
+    public ComparisonResult(List<Expression> queryExpressions, List<Expression> viewExpressions,
+            Set<Set<Slot>> viewNoNullableSlot) {
+        this(queryExpressions, viewExpressions, viewNoNullableSlot, true);
     }
 
-    ComparisonResult(List<Expression> queryExpressions, List<Expression> viewExpressions, boolean valid) {
+    ComparisonResult(List<Expression> queryExpressions, List<Expression> viewExpressions,
+            Set<Set<Slot>> viewNoNullableSlot, boolean valid) {
         this.viewExpressions = ImmutableList.copyOf(viewExpressions);
         this.queryExpressions = ImmutableList.copyOf(queryExpressions);
+        this.viewNoNullableSlot = ImmutableSet.copyOf(viewNoNullableSlot);
         this.valid = valid;
     }
 
@@ -52,6 +59,10 @@ public List<Expression> getQueryExpressions() {
         return queryExpressions;
     }
 
+    public Set<Set<Slot>> getViewNoNullableSlot() {
+        return viewNoNullableSlot;
+    }
+
     public boolean isInvalid() {
         return !valid;
     }
@@ -62,6 +73,7 @@ public boolean isInvalid() {
     public static class Builder {
         ImmutableList.Builder<Expression> queryBuilder = new ImmutableList.Builder<>();
         ImmutableList.Builder<Expression> viewBuilder = new ImmutableList.Builder<>();
+        ImmutableSet.Builder<Set<Slot>> viewNoNullableSlotBuilder = new ImmutableSet.Builder<>();
         boolean valid = true;
 
         /**
@@ -77,18 +89,31 @@ public Builder addComparisonResult(ComparisonResult comparisonResult) {
             return this;
         }
 
-        public Builder addQueryExpressions(Collection<Expression> expressions) {
+        public Builder addQueryExpressions(Collection<? extends Expression> expressions) {
             queryBuilder.addAll(expressions);
             return this;
         }
 
-        public Builder addViewExpressions(Collection<Expression> expressions) {
+        public Builder addViewExpressions(Collection<? extends Expression> expressions) {
             viewBuilder.addAll(expressions);
             return this;
         }
 
+        public Builder addViewNoNullableSlot(Set<Slot> viewNoNullableSlot) {
+            viewNoNullableSlotBuilder.add(ImmutableSet.copyOf(viewNoNullableSlot));
+            return this;
+        }
+
+        public boolean isInvalid() {
+            return !valid;
+        }
+
         public ComparisonResult build() {
-            return new ComparisonResult(queryBuilder.build(), viewBuilder.build(), valid);
+            if (isInvalid()) {
+                return ComparisonResult.INVALID;
+            }
+            return new ComparisonResult(queryBuilder.build(), viewBuilder.build(),
+                    viewNoNullableSlotBuilder.build(), valid);
         }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
new file mode 100644
index 00000000000000..a869fe729a94f1
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -0,0 +1,299 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.bitmap.LongBitmap;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.Edge;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.FilterEdge;
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.edge.JoinEdge;
+import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughJoin;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.JoinType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * HyperGraphComparator
+ */
+public class HyperGraphComparator {
+    // This second join can be inferred to the first join by map value,
+    // The map value means the child's should be no-nullable
+    static Map<Pair<JoinType, JoinType>, Pair<Boolean, Boolean>> canInferredJoinTypeMap = ImmutableMap
+            .<Pair<JoinType, JoinType>, Pair<Boolean, Boolean>>builder()
+            .put(Pair.of(JoinType.LEFT_SEMI_JOIN, JoinType.INNER_JOIN), Pair.of(false, false))
+            .put(Pair.of(JoinType.RIGHT_SEMI_JOIN, JoinType.INNER_JOIN), Pair.of(false, false))
+            .put(Pair.of(JoinType.INNER_JOIN, JoinType.LEFT_OUTER_JOIN), Pair.of(false, true))
+            .put(Pair.of(JoinType.INNER_JOIN, JoinType.RIGHT_OUTER_JOIN), Pair.of(true, false))
+            .put(Pair.of(JoinType.INNER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, true))
+            .put(Pair.of(JoinType.LEFT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(true, false))
+            .put(Pair.of(JoinType.RIGHT_OUTER_JOIN, JoinType.FULL_OUTER_JOIN), Pair.of(false, true))
+            .build();
+
+    // record inferred edges when comparing mv
+    private final HyperGraph queryHyperGraph;
+    private final HyperGraph viewHyperGraph;
+    private final Map<JoinEdge, Pair<JoinType, Set<Slot>>> inferredViewEdgeMap = new HashMap<>();
+    private final Map<Edge, List<? extends Expression>> pullUpQueryExprWithEdge = new HashMap<>();
+    private final Map<Edge, List<? extends Expression>> pullUpViewExprWithEdge = new HashMap<>();
+    private final LogicalCompatibilityContext logicalCompatibilityContext;
+
+    HyperGraphComparator(HyperGraph queryHyperGraph, HyperGraph viewHyperGraph,
+            LogicalCompatibilityContext logicalCompatibilityContext) {
+        this.queryHyperGraph = queryHyperGraph;
+        this.viewHyperGraph = viewHyperGraph;
+        this.logicalCompatibilityContext = logicalCompatibilityContext;
+    }
+
+    /**
+     * compare hypergraph
+     *
+     * @param viewHG the compared hyper graph
+     * @return Comparison result
+     */
+    public static ComparisonResult isLogicCompatible(HyperGraph queryHG, HyperGraph viewHG,
+            LogicalCompatibilityContext ctx) {
+        return new HyperGraphComparator(queryHG, viewHG, ctx).isLogicCompatible();
+    }
+
+    private ComparisonResult isLogicCompatible() {
+        // 1 try to construct a map which can be mapped from edge to edge
+        Map<Edge, Edge> queryToView = constructMapWithNode();
+
+        // 2. compare them by expression and extract residual expr
+        queryToView.forEach(this::compareEdgeWithExpr);
+
+        // 3. process residual edges
+        Sets.difference(getQueryJoinEdgeSet(), queryToView.keySet())
+                .forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions()));
+        Sets.difference(getQueryFilterEdgeSet(), queryToView.keySet())
+                .forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions()));
+        Sets.difference(getViewJoinEdgeSet(), Sets.newHashSet(queryToView.values()))
+                .forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions()));
+        Sets.difference(getViewFilterEdgeSet(), Sets.newHashSet(queryToView.values()))
+                .forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions()));
+
+        return buildComparisonRes();
+    }
+
+    private ComparisonResult buildComparisonRes() {
+        ComparisonResult.Builder builder = new ComparisonResult.Builder();
+        for (Entry<Edge, List<? extends Expression>> e : pullUpQueryExprWithEdge.entrySet()) {
+            if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) {
+                return ComparisonResult.INVALID;
+            }
+            builder.addQueryExpressions(e.getValue());
+        }
+        for (Entry<Edge, List<? extends Expression>> e : pullUpViewExprWithEdge.entrySet()) {
+            if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) {
+                return ComparisonResult.INVALID;
+            }
+            builder.addViewExpressions(e.getValue());
+        }
+        for (Pair<JoinType, Set<Slot>> inferredCond : inferredViewEdgeMap.values()) {
+            builder.addViewNoNullableSlot(inferredCond.second);
+        }
+        return builder.build();
+    }
+
+    private boolean canPullUp(Edge edge) {
+        // Only inner join and filter with none rejectNodes can be pull up
+        if (edge instanceof JoinEdge && !((JoinEdge) edge).getJoinType().isInnerJoin()) {
+            return false;
+        }
+        boolean pullFromLeft = edge.getLeftRejectEdge().stream()
+                .map(e -> inferredViewEdgeMap.getOrDefault(e, Pair.of(e.getJoinType(), null)))
+                .allMatch(e -> canPullFromLeft(edge, e.first));
+        boolean pullFromRight = edge.getRightRejectEdge().stream()
+                .map(e -> inferredViewEdgeMap.getOrDefault(e, Pair.of(e.getJoinType(), null)))
+                .allMatch(e -> canPullFromRight(edge, e.first));
+        return pullFromLeft && pullFromRight;
+    }
+
+    private boolean canPullFromLeft(Edge bottomEdge, JoinType topJoinType) {
+        if (bottomEdge instanceof FilterEdge) {
+            return PushDownFilterThroughJoin.COULD_PUSH_THROUGH_LEFT.contains(topJoinType);
+        } else if (bottomEdge instanceof JoinEdge) {
+            return JoinType.isAssoc(((JoinEdge) bottomEdge).getJoinType(), topJoinType)
+                    || JoinType.isLAssoc(((JoinEdge) bottomEdge).getJoinType(), topJoinType);
+        }
+        return false;
+    }
+
+    private boolean canPullFromRight(Edge bottomEdge, JoinType topJoinType) {
+        if (bottomEdge instanceof FilterEdge) {
+            return PushDownFilterThroughJoin.COULD_PUSH_THROUGH_RIGHT.contains(topJoinType);
+        } else if (bottomEdge instanceof JoinEdge) {
+            return JoinType.isAssoc(topJoinType, ((JoinEdge) bottomEdge).getJoinType())
+                    || JoinType.isRAssoc(topJoinType, ((JoinEdge) bottomEdge).getJoinType());
+        }
+        return false;
+    }
+
+    private List<JoinEdge> getQueryJoinEdges() {
+        return queryHyperGraph.getJoinEdges();
+    }
+
+    private Set<JoinEdge> getQueryJoinEdgeSet() {
+        return ImmutableSet.copyOf(queryHyperGraph.getJoinEdges());
+    }
+
+    private List<FilterEdge> getQueryFilterEdges() {
+        return queryHyperGraph.getFilterEdges();
+    }
+
+    private Set<FilterEdge> getQueryFilterEdgeSet() {
+        return ImmutableSet.copyOf(queryHyperGraph.getFilterEdges());
+    }
+
+    private Set<FilterEdge> getViewFilterEdgeSet() {
+        return ImmutableSet.copyOf(viewHyperGraph.getFilterEdges());
+    }
+
+    private Set<JoinEdge> getViewJoinEdgeSet() {
+        return ImmutableSet.copyOf(viewHyperGraph.getJoinEdges());
+    }
+
+    private List<JoinEdge> getViewJoinEdges() {
+        return viewHyperGraph.getJoinEdges();
+    }
+
+    private List<FilterEdge> getViewFilterEdges() {
+        return viewHyperGraph.getFilterEdges();
+    }
+
+    private Map<Expression, Expression> getQueryToViewExprMap() {
+        return logicalCompatibilityContext.getQueryToViewEdgeExpressionMapping();
+    }
+
+    private Map<Integer, Integer> getQueryToViewNodeIdMap() {
+        return logicalCompatibilityContext.getQueryToViewNodeIDMapping();
+    }
+
+    private Map<Edge, Edge> constructMapWithNode() {
+        // TODO use hash map to reduce loop
+        Map<Edge, Edge> joinEdgeMap = getQueryJoinEdges().stream().map(qe -> {
+            Optional<JoinEdge> viewEdge = getViewJoinEdges().stream()
+                    .filter(ve -> compareEdgeWithNode(qe, ve)).findFirst();
+            return Pair.of(qe, viewEdge);
+        }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
+        Map<Edge, Edge> filterEdgeMap = getQueryFilterEdges().stream().map(qe -> {
+            Optional<FilterEdge> viewEdge = getViewFilterEdges().stream()
+                    .filter(ve -> compareEdgeWithNode(qe, ve)).findFirst();
+            return Pair.of(qe, viewEdge);
+        }).filter(e -> e.second.isPresent()).collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second.get()));
+        return ImmutableMap.<Edge, Edge>builder().putAll(joinEdgeMap).putAll(filterEdgeMap).build();
+    }
+
+    private boolean compareEdgeWithNode(Edge query, Edge view) {
+        if (query instanceof FilterEdge && view instanceof FilterEdge) {
+            return compareEdgeWithFilter((FilterEdge) query, (FilterEdge) view);
+        } else if (query instanceof JoinEdge && view instanceof JoinEdge) {
+            return compareJoinEdge((JoinEdge) query, (JoinEdge) view);
+        }
+        return false;
+    }
+
+    private boolean compareEdgeWithFilter(FilterEdge query, FilterEdge view) {
+        long qChild = query.getReferenceNodes();
+        long vChild = view.getReferenceNodes();
+        return rewriteQueryNodeMap(qChild) == vChild;
+    }
+
+    private boolean compareJoinEdge(JoinEdge query, JoinEdge view) {
+        if (query.getJoinType().equals(view.getJoinType())
+                || canInferredJoinTypeMap.containsKey(Pair.of(query.getJoinType(), view.getJoinType()))) {
+            if (tryInferEdge(query, view)) {
+                return true;
+            }
+
+        }
+
+        if (query.getJoinType().swap().equals(view.getJoinType())
+                || canInferredJoinTypeMap.containsKey(Pair.of(query.getJoinType().swap(), view.getJoinType()))) {
+            if (tryInferEdge(query.swap(), view)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    private boolean tryInferEdge(JoinEdge query, JoinEdge view) {
+        if (rewriteQueryNodeMap(query.getLeftExtendedNodes()) != view.getLeftExtendedNodes()
+                || rewriteQueryNodeMap(query.getRightExtendedNodes()) != view.getRightExtendedNodes()) {
+            return false;
+        }
+        if (!query.getJoinType().equals(view.getJoinType())) {
+            Pair<Boolean, Boolean> noNullableChild = canInferredJoinTypeMap.getOrDefault(
+                    Pair.of(query.getJoinType(), view.getJoinType()), null);
+            if (noNullableChild == null) {
+                return false;
+            }
+            Set<Slot> noNullableSlot = Sets.union(
+                    noNullableChild.first ? view.getJoin().left().getOutputSet() : ImmutableSet.of(),
+                    noNullableChild.second ? view.getJoin().right().getOutputSet() : ImmutableSet.of()
+            );
+            inferredViewEdgeMap.put(view, Pair.of(query.getJoinType(), noNullableSlot));
+        }
+        return true;
+    }
+
+    private long rewriteQueryNodeMap(long bitmap) {
+        long newBitmap = LongBitmap.newBitmap();
+        for (int i : LongBitmap.getIterator(bitmap)) {
+            int newIdx = getQueryToViewNodeIdMap().getOrDefault(i, 0);
+            newBitmap = LongBitmap.set(newBitmap, newIdx);
+        }
+        return newBitmap;
+    }
+
+    private void compareEdgeWithExpr(Edge query, Edge view) {
+        Set<? extends Expression> queryExprSet = query.getExpressionSet();
+        Set<? extends Expression> viewExprSet = view.getExpressionSet();
+
+        Set<Expression> exprMappedOfView = new HashSet<>();
+        List<Expression> residualQueryExpr = new ArrayList<>();
+        for (Expression queryExpr : queryExprSet) {
+            if (getQueryToViewExprMap().containsKey(queryExpr) && viewExprSet.contains(
+                    getQueryToViewExprMap().get(queryExpr))) {
+                exprMappedOfView.add(getQueryToViewExprMap().get(queryExpr));
+            } else {
+                residualQueryExpr.add(queryExpr);
+            }
+        }
+        List<Expression> residualViewExpr = ImmutableList.copyOf(Sets.difference(viewExprSet, exprMappedOfView));
+        pullUpQueryExprWithEdge.put(query, residualQueryExpr);
+        pullUpViewExprWithEdge.put(query, residualViewExpr);
+    }
+
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
index 20da9ee12fdd69..2688882be97f3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -265,7 +265,8 @@ public List<? extends Expression> getExpressions() {
      */
     public static ComparisonResult isGraphLogicalEquals(StructInfo queryStructInfo, StructInfo viewStructInfo,
             LogicalCompatibilityContext compatibilityContext) {
-        return queryStructInfo.hyperGraph.isLogicCompatible(viewStructInfo.hyperGraph, compatibilityContext);
+        return HyperGraphComparator
+                .isLogicCompatible(queryStructInfo.hyperGraph, viewStructInfo.hyperGraph, compatibilityContext);
     }
 
     private static class RelationCollector extends DefaultPlanVisitor<Void, List<CatalogRelation>> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
index 773dbce5fc256e..c729aec752f3d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJoin.java
@@ -181,6 +181,11 @@ private LogicalJoin(
         this.markJoinSlotReference = markJoinSlotReference;
     }
 
+    public LogicalJoin<? extends Plan, ? extends Plan> swap() {
+        return withTypeChildren(getJoinType().swap(),
+                right(), left());
+    }
+
     public List<Expression> getOtherJoinConjuncts() {
         return otherJoinConjuncts;
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
index cfc88b2aa3cef2..105a5d450c3a81 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
@@ -20,6 +20,8 @@
 import org.apache.doris.nereids.CascadesContext;
 import org.apache.doris.nereids.rules.RuleSet;
 import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
+import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
 import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
@@ -62,7 +64,8 @@ void testStarGraphWithInnerJoin() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid());
+        Assertions.assertFalse(
+                HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid());
     }
 
     @Test
@@ -79,7 +82,8 @@ void testRandomQuery() {
                 .getAllPlan().get(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        Assertions.assertFalse(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid());
+        Assertions.assertFalse(
+                HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).isInvalid());
     }
 
     @Test
@@ -104,9 +108,9 @@ void testInnerJoinWithFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions();
-        Assertions.assertEquals(1, exprList.size());
-        Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertEquals(1, res.getQueryExpressions().size());
+        Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql());
     }
 
     @Disabled
@@ -132,7 +136,7 @@ void testInnerJoinWithFilter2() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions();
+        List<Expression> exprList = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2)).getQueryExpressions();
         Assertions.assertEquals(0, exprList.size());
     }
 
@@ -159,9 +163,9 @@ void testLeftOuterJoinWithLeftFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        List<Expression> exprList = h1.isLogicCompatible(h2, constructContext(p1, p2)).getQueryExpressions();
-        Assertions.assertEquals(1, exprList.size());
-        Assertions.assertEquals("(id = 0)", exprList.get(0).toSql());
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertEquals(1, res.getQueryExpressions().size());
+        Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql());
     }
 
     @Test
@@ -187,7 +191,8 @@ void testLeftOuterJoinWithRightFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        Assertions.assertTrue(h1.isLogicCompatible(h2, constructContext(p1, p2)).isInvalid());
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertTrue(res.isInvalid());
     }
 
     LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
new file mode 100644
index 00000000000000..33100c1180b609
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferJoinTest.java
@@ -0,0 +1,173 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.jobs.joinorder.hypergraph;
+
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.RuleSet;
+import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
+import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
+import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.stream.Collectors;
+
+class InferJoinTest extends SqlTestBase {
+    @Test
+    void testInnerInferLeft() {
+        connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 inner join T2 on T1.id = T2.id where T1.id = 0",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 left join T2 on T1.id = T2.id where T1.id = 0",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertFalse(res.isInvalid());
+        Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
+        Assertions.assertEquals("[id, score]",
+                res.getViewNoNullableSlot().iterator().next().stream().map(ExpressionTrait::toSql).collect(Collectors.toList()).toString());
+    }
+
+    @Test
+    void testInnerInferLeftWithFilter() {
+        connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 inner join T2 on T1.id = T2.id where T1.id = 0",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 left join (select * from T2 where id = 0) T2 on T1.id = T2.id",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertFalse(res.isInvalid());
+        Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
+        Assertions.assertEquals("[id, score]",
+                res.getViewNoNullableSlot().iterator().next().stream().map(ExpressionTrait::toSql).collect(Collectors.toList()).toString());
+        Assertions.assertEquals("(id = 0)", res.getViewExpressions().get(0).toSql());
+        Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql());
+    }
+
+    // should consider rebuilding the hypergraph
+    @Disabled
+    @Test
+    void testInnerInferLeftWithJoinCond() {
+        connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 inner join "
+                        + "(select T2.id from T2 inner join T3 on T2.id = T3.id) T2 "
+                        + "on T1.id = T2.id",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 left join "
+                        + "(select T2.id from T2 inner join T3 on T2.id = T3.id and T2.score = T3.score) T2 "
+                        + "on T1.id = T2.id",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertFalse(res.isInvalid());
+        Assertions.assertEquals(1, res.getViewNoNullableSlot().size());
+        Assertions.assertEquals("[id, score]",
+                res.getViewNoNullableSlot().iterator().next().stream().map(ExpressionTrait::toSql).collect(Collectors.toList()).toString());
+        Assertions.assertEquals("score = score", res.getQueryExpressions().get(0).toSql());
+    }
+
+    @Test
+    void testLeftOuterJoinWithRightFilter() {
+        connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 left outer join ( select * from T2 where T2.id = 0) T2 on T1.id = T2.id",
+                connectContext
+        );
+        connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 left outer join T2 on T1.id = T2.id",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertTrue(res.isInvalid());
+    }
+
+    LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
+        StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1,
+                null).get(0);
+        StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2,
+                null).get(0);
+        RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0);
+        SlotMapping sm = SlotMapping.generate(rm);
+        return LogicalCompatibilityContext.from(rm, sm, st1, st2);
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
index d4818a844e80ab..4da4905f8aa85a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
@@ -21,6 +21,7 @@
 import org.apache.doris.nereids.rules.RuleSet;
 import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
 import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
+import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
 import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
 import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
@@ -54,7 +55,7 @@ void testPullUpQueryFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
         Assertions.assertEquals(2, res.getQueryExpressions().size());
         Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql());
         Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(1).toSql());
@@ -81,7 +82,7 @@ void testPullUpQueryJoinCondition() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
         Assertions.assertEquals(1, res.getQueryExpressions().size());
         Assertions.assertEquals("(score = score)", res.getQueryExpressions().get(0).toSql());
     }
@@ -107,7 +108,7 @@ void testPullUpViewFilter() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
         Assertions.assertEquals(2, res.getViewExpressions().size());
         Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(0).toSql());
         Assertions.assertEquals("(id = 1)", res.getViewExpressions().get(1).toSql());
@@ -134,7 +135,7 @@ void testPullUpViewJoinCondition() {
                 .getAllPlan().get(0).child(0);
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
-        ComparisonResult res = h1.isLogicCompatible(h2, constructContext(p1, p2));
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
         Assertions.assertEquals(1, res.getViewExpressions().size());
         Assertions.assertEquals("(score = score)", res.getViewExpressions().get(0).toSql());
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
index f68365f6708b37..bf5edfa45ec694 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/BuildStructInfoTest.java
@@ -79,7 +79,8 @@ void testFilter() {
                         .when(j -> {
                             HyperGraph structInfo = HyperGraph.toStructInfo(j).get(0);
                             Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
-                            Assertions.assertEquals(3L, structInfo.getFilterEdge(0).getRejectNodes());
+                            Assertions.assertEquals(0, structInfo.getFilterEdge(0).getLeftRejectEdge().size());
+                            Assertions.assertEquals(1, structInfo.getFilterEdge(0).getRightRejectEdge().size());
                             return true;
                         }));
 
@@ -92,7 +93,6 @@ void testFilter() {
                         .when(j -> {
                             HyperGraph structInfo = HyperGraph.toStructInfo(j).get(0);
                             Assertions.assertTrue(structInfo.getJoinEdge(0).getJoinType().isLeftOuterJoin());
-                            Assertions.assertEquals(0, structInfo.getFilterEdge(0).getRejectNodes());
                             return true;
                         }));
     }

From 576a2b33cb38372a96277db49247c9f0969e5bf3 Mon Sep 17 00:00:00 2001
From: minghong <englefly@gmail.com>
Date: Wed, 27 Dec 2023 11:36:12 +0800
Subject: [PATCH 052/299] [fix](nereids) "not is null" stats estimation fix
 (#28860)

* fix not is null stats
---
 .../doris/nereids/stats/FilterEstimation.java |  3 +-
 .../nereids/stats/FilterEstimationTest.java   | 38 ++++++++++
 .../shape/query76.out                         | 40 ++++++-----
 .../shape/query95.out                         | 59 ++++++++--------
 .../shape/query97.out                         | 20 +++---
 .../noStatsRfPrune/query95.out                | 59 ++++++++--------
 .../no_stats_shape/query95.out                | 59 ++++++++--------
 .../rf_prune/query44.out                      | 69 +++++++++----------
 .../rf_prune/query76.out                      | 45 ++++++------
 .../rf_prune/query95.out                      | 59 ++++++++--------
 .../shape/query44.out                         | 69 +++++++++----------
 .../shape/query76.out                         | 45 ++++++------
 .../shape/query95.out                         | 59 ++++++++--------
 .../test_partial_update_seq_col.groovy        | 10 +--
 14 files changed, 331 insertions(+), 303 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index d5c9ded22e360d..b716b350f24b84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -414,7 +414,8 @@ public Statistics visitIsNull(IsNull isNull, EstimationContext context) {
                 .setMinValue(Double.NEGATIVE_INFINITY)
                 .setNdv(0);
         StatisticsBuilder builder = new StatisticsBuilder(context.statistics);
-        builder.putColumnStatistics(isNull.child(), colBuilder.build());
+        builder.setRowCount(outputRowCount);
+        builder.putColumnStatistics(isNull, colBuilder.build());
         context.addKeyIfSlot(isNull.child());
         return builder.build();
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
index 246e556824ef17..f5491c633313e0 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.expressions.GreaterThan;
 import org.apache.doris.nereids.trees.expressions.GreaterThanEqual;
 import org.apache.doris.nereids.trees.expressions.InPredicate;
+import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.LessThan;
 import org.apache.doris.nereids.trees.expressions.LessThanEqual;
 import org.apache.doris.nereids.trees.expressions.Not;
@@ -872,4 +873,41 @@ public void testDateRangeSelectivity() {
         Statistics result = filterEstimation.estimate(greaterThan, stats);
         Assertions.assertEquals(result.getRowCount(), 10, 0.1);
     }
+
+    @Test
+    public void testIsNull() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(100)
+                .setAvgSizeByte(4)
+                .setNumNulls(10)
+                .setMaxValue(100)
+                .setMinValue(0)
+                .setCount(100);
+        IsNull isNull = new IsNull(a);
+        Statistics stats = new Statistics(100, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(isNull, stats);
+        Assertions.assertEquals(result.getRowCount(), 10);
+    }
+
+    @Test
+    public void testIsNotNull() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(100)
+                .setAvgSizeByte(4)
+                .setNumNulls(10)
+                .setMaxValue(100)
+                .setMinValue(0)
+                .setCount(100);
+        IsNull isNull = new IsNull(a);
+        Not not = new Not(isNull);
+        Statistics stats = new Statistics(100, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(not, stats);
+        Assertions.assertEquals(result.getRowCount(), 90);
+    }
 }
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query76.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query76.out
index 5aa9559b23c27c..3f46749fc61f17 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query76.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query76.out
@@ -23,28 +23,30 @@ PhysicalResultSink
 --------------------------PhysicalOlapScan[item]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ws_item_sk]
-----------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk]
-------------------------PhysicalProject
---------------------------filter(ws_promo_sk IS NULL)
-----------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[date_dim]
+--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 ws_item_sk->[i_item_sk]
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[item] apply RFs: RF3
 ----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[item]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[cs_item_sk]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_sold_date_sk]
+--------------------------PhysicalDistribute
+----------------------------PhysicalProject
+------------------------------filter(ws_promo_sk IS NULL)
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2
+--------------------------PhysicalDistribute
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[date_dim]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[cs_item_sk]
+--------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[cs_sold_date_sk]
-------------------------PhysicalProject
---------------------------filter(cs_bill_customer_sk IS NULL)
-----------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------filter(cs_bill_customer_sk IS NULL)
+------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
 ----------------------------PhysicalOlapScan[date_dim]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[item]
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
index c82d2ae58d3489..6720aab10b8d4e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
@@ -13,42 +13,41 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalResultSink
 ----PhysicalTopN[MERGE_SORT]
 ------PhysicalTopN[LOCAL_SORT]
---------PhysicalProject
-----------hashAgg[DISTINCT_GLOBAL]
-------------PhysicalDistribute
---------------hashAgg[DISTINCT_LOCAL]
-----------------hashAgg[GLOBAL]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_returns]
-------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+--------hashAgg[DISTINCT_GLOBAL]
+----------PhysicalDistribute
+------------hashAgg[DISTINCT_LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
-----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
-------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ws_ship_addr_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_state = 'VA'))
-----------------------------------------PhysicalOlapScan[customer_address]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[web_returns]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------------------PhysicalDistribute
+--------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ws_ship_addr_sk]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
-------------------------------------filter((date_dim.d_date <= '2001-05-31') and (date_dim.d_date >= '2001-04-01'))
---------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------filter((customer_address.ca_state = 'VA'))
+--------------------------------------PhysicalOlapScan[customer_address]
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------filter((web_site.web_company_name = 'pri'))
-------------------------------------PhysicalOlapScan[web_site]
+----------------------------------filter((date_dim.d_date <= '2001-05-31') and (date_dim.d_date >= '2001-04-01'))
+------------------------------------PhysicalOlapScan[date_dim]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------filter((web_site.web_company_name = 'pri'))
+----------------------------------PhysicalOlapScan[web_site]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out
index bbc900d82cbeac..bb1f3c8efc6cc0 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query97.out
@@ -13,25 +13,25 @@ PhysicalResultSink
 --------------------PhysicalDistribute
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 ss_sold_date_sk->[d_date_sk]
+--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
---------------------------------PhysicalOlapScan[date_dim] apply RFs: RF1
+------------------------------filter(( not ss_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------filter(( not ss_sold_date_sk IS NULL))
-----------------------------------PhysicalOlapScan[store_sales]
+--------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+----------------------------------PhysicalOlapScan[date_dim]
 ----------------PhysicalProject
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute
 ----------------------hashAgg[LOCAL]
 ------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 cs_sold_date_sk->[d_date_sk]
+--------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[cs_sold_date_sk]
 ----------------------------PhysicalProject
-------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
---------------------------------PhysicalOlapScan[date_dim] apply RFs: RF0
+------------------------------filter(( not cs_sold_date_sk IS NULL))
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------filter(( not cs_sold_date_sk IS NULL))
-----------------------------------PhysicalOlapScan[catalog_sales]
+--------------------------------filter((date_dim.d_month_seq <= 1210) and (date_dim.d_month_seq >= 1199))
+----------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
index 226b8c9767817e..d2ecc4813b5cdc 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
@@ -13,42 +13,41 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalResultSink
 ----PhysicalTopN[MERGE_SORT]
 ------PhysicalTopN[LOCAL_SORT]
---------PhysicalProject
-----------hashAgg[DISTINCT_GLOBAL]
-------------PhysicalDistribute
---------------hashAgg[DISTINCT_LOCAL]
-----------------hashAgg[GLOBAL]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_returns]
-------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+--------hashAgg[DISTINCT_GLOBAL]
+----------PhysicalDistribute
+------------hashAgg[DISTINCT_LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
-----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
-------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ws_ship_addr_sk]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_ship_date_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
-----------------------------------------PhysicalOlapScan[date_dim]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[web_returns]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------------------PhysicalDistribute
+--------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ws_ship_addr_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_ship_date_sk]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
-------------------------------------filter((customer_address.ca_state = 'NC'))
---------------------------------------PhysicalOlapScan[customer_address]
+------------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
+--------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------filter((web_site.web_company_name = 'pri'))
-------------------------------------PhysicalOlapScan[web_site]
+----------------------------------filter((customer_address.ca_state = 'NC'))
+------------------------------------PhysicalOlapScan[customer_address]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------filter((web_site.web_company_name = 'pri'))
+----------------------------------PhysicalOlapScan[web_site]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
index 226b8c9767817e..d2ecc4813b5cdc 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
@@ -13,42 +13,41 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalResultSink
 ----PhysicalTopN[MERGE_SORT]
 ------PhysicalTopN[LOCAL_SORT]
---------PhysicalProject
-----------hashAgg[DISTINCT_GLOBAL]
-------------PhysicalDistribute
---------------hashAgg[DISTINCT_LOCAL]
-----------------hashAgg[GLOBAL]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_returns]
-------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+--------hashAgg[DISTINCT_GLOBAL]
+----------PhysicalDistribute
+------------hashAgg[DISTINCT_LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
-----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
-------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ws_ship_addr_sk]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_ship_date_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
-----------------------------------------PhysicalOlapScan[date_dim]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[web_returns]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------------------PhysicalDistribute
+--------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ws_ship_addr_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ws_ship_date_sk]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
-------------------------------------filter((customer_address.ca_state = 'NC'))
---------------------------------------PhysicalOlapScan[customer_address]
+------------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
+--------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------filter((web_site.web_company_name = 'pri'))
-------------------------------------PhysicalOlapScan[web_site]
+----------------------------------filter((customer_address.ca_state = 'NC'))
+------------------------------------PhysicalOlapScan[customer_address]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------filter((web_site.web_company_name = 'pri'))
+----------------------------------PhysicalOlapScan[web_site]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query44.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query44.out
index 2c3fc9a8c25db2..35ca3b8e453b6e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query44.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query44.out
@@ -5,12 +5,42 @@ PhysicalResultSink
 ----PhysicalDistribute
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN] hashCondition=((asceding.rnk = descending.rnk)) otherCondition=()
+----------hashJoin[INNER_JOIN] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF1 item_sk->[i_item_sk]
+------------PhysicalProject
+--------------PhysicalOlapScan[item] apply RFs: RF1
 ------------PhysicalDistribute
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((i1.i_item_sk = asceding.item_sk)) otherCondition=() build RFs:RF1 item_sk->[i_item_sk]
-------------------PhysicalProject
---------------------PhysicalOlapScan[item] apply RFs: RF1
+----------------hashJoin[INNER_JOIN] hashCondition=((asceding.rnk = descending.rnk)) otherCondition=()
+------------------hashJoin[INNER_JOIN] hashCondition=((i1.i_item_sk = asceding.item_sk)) otherCondition=() build RFs:RF0 item_sk->[i_item_sk]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[item] apply RFs: RF0
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((rnk < 11))
+--------------------------PhysicalWindow
+----------------------------PhysicalQuickSort[MERGE_SORT]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalQuickSort[LOCAL_SORT]
+----------------------------------PhysicalPartitionTopN
+------------------------------------PhysicalProject
+--------------------------------------NestedLoopJoin[INNER_JOIN](cast(rank_col as DOUBLE) > cast((0.9 * rank_col) as DOUBLE))
+----------------------------------------PhysicalProject
+------------------------------------------hashAgg[GLOBAL]
+--------------------------------------------PhysicalDistribute
+----------------------------------------------hashAgg[LOCAL]
+------------------------------------------------PhysicalProject
+--------------------------------------------------filter((ss1.ss_store_sk = 146))
+----------------------------------------------------PhysicalOlapScan[store_sales]
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalAssertNumRows
+--------------------------------------------PhysicalDistribute
+----------------------------------------------PhysicalProject
+------------------------------------------------hashAgg[GLOBAL]
+--------------------------------------------------PhysicalDistribute
+----------------------------------------------------hashAgg[LOCAL]
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL)
+----------------------------------------------------------PhysicalOlapScan[store_sales]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------filter((rnk < 11))
@@ -38,35 +68,4 @@ PhysicalResultSink
 ----------------------------------------------------PhysicalProject
 ------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL)
 --------------------------------------------------------PhysicalOlapScan[store_sales]
-------------PhysicalDistribute
---------------hashJoin[INNER_JOIN] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->[i_item_sk]
-----------------PhysicalProject
-------------------PhysicalOlapScan[item] apply RFs: RF0
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------filter((rnk < 11))
-----------------------PhysicalWindow
-------------------------PhysicalQuickSort[MERGE_SORT]
---------------------------PhysicalDistribute
-----------------------------PhysicalQuickSort[LOCAL_SORT]
-------------------------------PhysicalPartitionTopN
---------------------------------PhysicalProject
-----------------------------------NestedLoopJoin[INNER_JOIN](cast(rank_col as DOUBLE) > cast((0.9 * rank_col) as DOUBLE))
-------------------------------------PhysicalProject
---------------------------------------hashAgg[GLOBAL]
-----------------------------------------PhysicalDistribute
-------------------------------------------hashAgg[LOCAL]
---------------------------------------------PhysicalProject
-----------------------------------------------filter((ss1.ss_store_sk = 146))
-------------------------------------------------PhysicalOlapScan[store_sales]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalAssertNumRows
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------hashAgg[GLOBAL]
-----------------------------------------------PhysicalDistribute
-------------------------------------------------hashAgg[LOCAL]
---------------------------------------------------PhysicalProject
-----------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL)
-------------------------------------------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query76.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query76.out
index 4e8160680f38b5..00c4bba1df69bb 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query76.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query76.out
@@ -10,44 +10,41 @@ PhysicalResultSink
 --------------PhysicalUnion
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=()
+--------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 ss_item_sk->[i_item_sk]
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=()
+------------------------PhysicalOlapScan[item] apply RFs: RF1
+----------------------PhysicalDistribute
+------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 ss_sold_date_sk->[d_date_sk]
 --------------------------PhysicalProject
-----------------------------filter(ss_hdemo_sk IS NULL)
-------------------------------PhysicalOlapScan[store_sales]
+----------------------------PhysicalOlapScan[date_dim] apply RFs: RF0
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[item]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[date_dim]
+------------------------------filter(ss_hdemo_sk IS NULL)
+--------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ws_sold_date_sk]
+--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 ws_item_sk->[i_item_sk]
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ws_item_sk]
+------------------------PhysicalOlapScan[item] apply RFs: RF3
+----------------------PhysicalDistribute
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 ws_sold_date_sk->[d_date_sk]
 --------------------------PhysicalProject
-----------------------------filter(ws_bill_addr_sk IS NULL)
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3
+----------------------------PhysicalOlapScan[date_dim] apply RFs: RF2
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[item]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[date_dim]
+------------------------------filter(ws_bill_addr_sk IS NULL)
+--------------------------------PhysicalOlapScan[web_sales]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+--------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[i_item_sk]
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk]
+------------------------PhysicalOlapScan[item] apply RFs: RF5
+----------------------PhysicalDistribute
+------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 cs_sold_date_sk->[d_date_sk]
 --------------------------PhysicalProject
-----------------------------filter(cs_warehouse_sk IS NULL)
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5
+----------------------------PhysicalOlapScan[date_dim] apply RFs: RF4
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[item]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[date_dim]
+------------------------------filter(cs_warehouse_sk IS NULL)
+--------------------------------PhysicalOlapScan[catalog_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
index 4fcf27b7e65999..bde9dbef392330 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
@@ -13,42 +13,41 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalResultSink
 ----PhysicalTopN[MERGE_SORT]
 ------PhysicalTopN[LOCAL_SORT]
---------PhysicalProject
-----------hashAgg[DISTINCT_GLOBAL]
-------------PhysicalDistribute
---------------hashAgg[DISTINCT_LOCAL]
-----------------hashAgg[GLOBAL]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_returns]
-------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+--------hashAgg[DISTINCT_GLOBAL]
+----------PhysicalDistribute
+------------hashAgg[DISTINCT_LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
-----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
-------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ws_ship_addr_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_state = 'NC'))
-----------------------------------------PhysicalOlapScan[customer_address]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[web_returns]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------------------PhysicalDistribute
+--------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ws_ship_addr_sk]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
-------------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
---------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------filter((customer_address.ca_state = 'NC'))
+--------------------------------------PhysicalOlapScan[customer_address]
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------filter((web_site.web_company_name = 'pri'))
-------------------------------------PhysicalOlapScan[web_site]
+----------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
+------------------------------------PhysicalOlapScan[date_dim]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------filter((web_site.web_company_name = 'pri'))
+----------------------------------PhysicalOlapScan[web_site]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query44.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query44.out
index 2c3fc9a8c25db2..35ca3b8e453b6e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query44.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query44.out
@@ -5,12 +5,42 @@ PhysicalResultSink
 ----PhysicalDistribute
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN] hashCondition=((asceding.rnk = descending.rnk)) otherCondition=()
+----------hashJoin[INNER_JOIN] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF1 item_sk->[i_item_sk]
+------------PhysicalProject
+--------------PhysicalOlapScan[item] apply RFs: RF1
 ------------PhysicalDistribute
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((i1.i_item_sk = asceding.item_sk)) otherCondition=() build RFs:RF1 item_sk->[i_item_sk]
-------------------PhysicalProject
---------------------PhysicalOlapScan[item] apply RFs: RF1
+----------------hashJoin[INNER_JOIN] hashCondition=((asceding.rnk = descending.rnk)) otherCondition=()
+------------------hashJoin[INNER_JOIN] hashCondition=((i1.i_item_sk = asceding.item_sk)) otherCondition=() build RFs:RF0 item_sk->[i_item_sk]
+--------------------PhysicalProject
+----------------------PhysicalOlapScan[item] apply RFs: RF0
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((rnk < 11))
+--------------------------PhysicalWindow
+----------------------------PhysicalQuickSort[MERGE_SORT]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalQuickSort[LOCAL_SORT]
+----------------------------------PhysicalPartitionTopN
+------------------------------------PhysicalProject
+--------------------------------------NestedLoopJoin[INNER_JOIN](cast(rank_col as DOUBLE) > cast((0.9 * rank_col) as DOUBLE))
+----------------------------------------PhysicalProject
+------------------------------------------hashAgg[GLOBAL]
+--------------------------------------------PhysicalDistribute
+----------------------------------------------hashAgg[LOCAL]
+------------------------------------------------PhysicalProject
+--------------------------------------------------filter((ss1.ss_store_sk = 146))
+----------------------------------------------------PhysicalOlapScan[store_sales]
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalAssertNumRows
+--------------------------------------------PhysicalDistribute
+----------------------------------------------PhysicalProject
+------------------------------------------------hashAgg[GLOBAL]
+--------------------------------------------------PhysicalDistribute
+----------------------------------------------------hashAgg[LOCAL]
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL)
+----------------------------------------------------------PhysicalOlapScan[store_sales]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------filter((rnk < 11))
@@ -38,35 +68,4 @@ PhysicalResultSink
 ----------------------------------------------------PhysicalProject
 ------------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL)
 --------------------------------------------------------PhysicalOlapScan[store_sales]
-------------PhysicalDistribute
---------------hashJoin[INNER_JOIN] hashCondition=((i2.i_item_sk = descending.item_sk)) otherCondition=() build RFs:RF0 item_sk->[i_item_sk]
-----------------PhysicalProject
-------------------PhysicalOlapScan[item] apply RFs: RF0
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------filter((rnk < 11))
-----------------------PhysicalWindow
-------------------------PhysicalQuickSort[MERGE_SORT]
---------------------------PhysicalDistribute
-----------------------------PhysicalQuickSort[LOCAL_SORT]
-------------------------------PhysicalPartitionTopN
---------------------------------PhysicalProject
-----------------------------------NestedLoopJoin[INNER_JOIN](cast(rank_col as DOUBLE) > cast((0.9 * rank_col) as DOUBLE))
-------------------------------------PhysicalProject
---------------------------------------hashAgg[GLOBAL]
-----------------------------------------PhysicalDistribute
-------------------------------------------hashAgg[LOCAL]
---------------------------------------------PhysicalProject
-----------------------------------------------filter((ss1.ss_store_sk = 146))
-------------------------------------------------PhysicalOlapScan[store_sales]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalAssertNumRows
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------hashAgg[GLOBAL]
-----------------------------------------------PhysicalDistribute
-------------------------------------------------hashAgg[LOCAL]
---------------------------------------------------PhysicalProject
-----------------------------------------------------filter((store_sales.ss_store_sk = 146) and ss_addr_sk IS NULL)
-------------------------------------------------------PhysicalOlapScan[store_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out
index 7d8bcd65ad412d..00c4bba1df69bb 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query76.out
@@ -10,44 +10,41 @@ PhysicalResultSink
 --------------PhysicalUnion
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
+--------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 ss_item_sk->[i_item_sk]
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[ss_item_sk]
+------------------------PhysicalOlapScan[item] apply RFs: RF1
+----------------------PhysicalDistribute
+------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 ss_sold_date_sk->[d_date_sk]
 --------------------------PhysicalProject
-----------------------------filter(ss_hdemo_sk IS NULL)
-------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
+----------------------------PhysicalOlapScan[date_dim] apply RFs: RF0
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[item]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[date_dim]
+------------------------------filter(ss_hdemo_sk IS NULL)
+--------------------------------PhysicalOlapScan[store_sales]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ws_sold_date_sk]
+--------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 ws_item_sk->[i_item_sk]
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ws_item_sk]
+------------------------PhysicalOlapScan[item] apply RFs: RF3
+----------------------PhysicalDistribute
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 ws_sold_date_sk->[d_date_sk]
 --------------------------PhysicalProject
-----------------------------filter(ws_bill_addr_sk IS NULL)
-------------------------------PhysicalOlapScan[web_sales] apply RFs: RF2 RF3
+----------------------------PhysicalOlapScan[date_dim] apply RFs: RF2
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[item]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[date_dim]
+------------------------------filter(ws_bill_addr_sk IS NULL)
+--------------------------------PhysicalOlapScan[web_sales]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+--------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF5 cs_item_sk->[i_item_sk]
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk]
+------------------------PhysicalOlapScan[item] apply RFs: RF5
+----------------------PhysicalDistribute
+------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 cs_sold_date_sk->[d_date_sk]
 --------------------------PhysicalProject
-----------------------------filter(cs_warehouse_sk IS NULL)
-------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5
+----------------------------PhysicalOlapScan[date_dim] apply RFs: RF4
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------PhysicalOlapScan[item]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[date_dim]
+------------------------------filter(cs_warehouse_sk IS NULL)
+--------------------------------PhysicalOlapScan[catalog_sales]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
index 4fcf27b7e65999..bde9dbef392330 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
@@ -13,42 +13,41 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalResultSink
 ----PhysicalTopN[MERGE_SORT]
 ------PhysicalTopN[LOCAL_SORT]
---------PhysicalProject
-----------hashAgg[DISTINCT_GLOBAL]
-------------PhysicalDistribute
---------------hashAgg[DISTINCT_LOCAL]
-----------------hashAgg[GLOBAL]
-------------------hashAgg[LOCAL]
---------------------PhysicalProject
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_returns]
-------------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+--------hashAgg[DISTINCT_GLOBAL]
+----------PhysicalDistribute
+------------hashAgg[DISTINCT_LOCAL]
+--------------hashAgg[GLOBAL]
+----------------hashAgg[LOCAL]
+------------------PhysicalProject
+--------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
+----------------------PhysicalProject
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
-----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
-------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk]
---------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ws_ship_addr_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_state = 'NC'))
-----------------------------------------PhysicalOlapScan[customer_address]
+----------------------------PhysicalProject
+------------------------------PhysicalOlapScan[web_returns]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
+------------------------PhysicalDistribute
+--------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_web_site_sk = web_site.web_site_sk)) otherCondition=() build RFs:RF3 web_site_sk->[ws_web_site_sk]
+----------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ws_ship_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((ws1.ws_ship_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ws_ship_addr_sk]
+--------------------------------PhysicalProject
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF1 RF2 RF3
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
-------------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
---------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------filter((customer_address.ca_state = 'NC'))
+--------------------------------------PhysicalOlapScan[customer_address]
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------filter((web_site.web_company_name = 'pri'))
-------------------------------------PhysicalOlapScan[web_site]
+----------------------------------filter((date_dim.d_date <= '1999-04-02') and (date_dim.d_date >= '1999-02-01'))
+------------------------------------PhysicalOlapScan[date_dim]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------filter((web_site.web_company_name = 'pri'))
+----------------------------------PhysicalOlapScan[web_site]
 
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy
index 67f78e64207efb..111037ae18f6b8 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_seq_col.groovy
@@ -136,9 +136,9 @@ suite("test_primary_key_partial_update_seq_col", "p0") {
 
 
             def tableName2 = "nereids_partial_update_native_insert_seq_col2"
-            sql """ DROP TABLE IF EXISTS ${tableName2} """
+            sql """ DROP TABLE IF EXISTS nereids_partial_update_20231227 """
             sql """
-                    CREATE TABLE ${tableName2} (
+                    CREATE TABLE nereids_partial_update_20231227 (
                         `id` int(11) NOT NULL COMMENT "用户 ID",
                         `score` int(11) NOT NULL COMMENT "用户得分",
                         `update_time` DATETIMEV2 NULL DEFAULT CURRENT_TIMESTAMP)
@@ -152,7 +152,7 @@ suite("test_primary_key_partial_update_seq_col", "p0") {
             // the input data don't contains sequence mapping column but the sequence mapping
             // column's default value is CURRENT_TIMESTAMP, will load successfully
             streamLoad {
-                table "${tableName2}"
+                table "nereids_partial_update_20231227"
 
                 set 'column_separator', ','
                 set 'format', 'csv'
@@ -164,8 +164,8 @@ suite("test_primary_key_partial_update_seq_col", "p0") {
 
             sql "sync"
 
-            qt_sql """ select id,score from ${tableName2} order by id;"""
-            sql """ DROP TABLE IF EXISTS ${tableName2}; """
+            qt_sql """ select id,score from nereids_partial_update_20231227 order by id;"""
+            sql """ DROP TABLE IF EXISTS nereids_partial_update_20231227; """
         }
     }
 }

From 8e358617c7e1b8c27c5da5568ddf188e2b621a98 Mon Sep 17 00:00:00 2001
From: minghong <englefly@gmail.com>
Date: Wed, 27 Dec 2023 11:37:36 +0800
Subject: [PATCH 053/299] [opt](nereids)expr normalize after filter pushdown
 (#28743)

normalize expression after filter push down
---
 .../org/apache/doris/nereids/jobs/executor/Rewriter.java  | 2 ++
 .../data/nereids_tpcds_shape_sf1000_p0/shape/query13.out  | 4 ++--
 .../data/nereids_tpcds_shape_sf1000_p0/shape/query48.out  | 2 +-
 .../data/nereids_tpcds_shape_sf1000_p0/shape/query85.out  | 2 +-
 .../noStatsRfPrune/query13.out                            | 4 ++--
 .../noStatsRfPrune/query48.out                            | 2 +-
 .../noStatsRfPrune/query85.out                            | 8 ++++----
 .../no_stats_shape/query13.out                            | 4 ++--
 .../no_stats_shape/query48.out                            | 2 +-
 .../no_stats_shape/query85.out                            | 8 ++++----
 .../nereids_tpcds_shape_sf100_p0/rf_prune/query13.out     | 4 ++--
 .../nereids_tpcds_shape_sf100_p0/rf_prune/query48.out     | 2 +-
 .../nereids_tpcds_shape_sf100_p0/rf_prune/query85.out     | 2 +-
 .../data/nereids_tpcds_shape_sf100_p0/shape/query13.out   | 4 ++--
 .../data/nereids_tpcds_shape_sf100_p0/shape/query48.out   | 2 +-
 .../data/nereids_tpcds_shape_sf100_p0/shape/query85.out   | 2 +-
 .../nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out | 2 +-
 .../nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out  | 4 ++--
 .../data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out    | 2 +-
 .../data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out     | 4 ++--
 .../data/nereids_tpch_shape_sf1000_p0/shape/q19.out       | 2 +-
 .../data/nereids_tpch_shape_sf1000_p0/shape/q7.out        | 4 ++--
 .../nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out   | 2 +-
 .../nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out    | 4 ++--
 24 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index af8f266a637e3e..fc3ef04fba8d07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -222,6 +222,8 @@ public class Rewriter extends AbstractBatchJobExecutor {
                     // but top-down traverse can not cover this case in one iteration, so bottom-up is more
                     // efficient because it can find the new plans and apply transform wherever it is
                     bottomUp(RuleSet.PUSH_DOWN_FILTERS),
+                    //after push down, some new filters are generated, which needs to be optimized. (example: tpch q19)
+                    topDown(new ExpressionOptimization()),
                     topDown(
                             new MergeFilters(),
                             new ReorderJoin(),
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
index e7ef5574c7fb12..69173c83eae9be 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query13.out
@@ -19,7 +19,7 @@ PhysicalResultSink
 ------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('IL', 'TN', 'TX') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('ID', 'OH', 'WY') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('IA', 'MS', 'SC') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------filter(((((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 --------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF4
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
@@ -31,7 +31,7 @@ PhysicalResultSink
 ------------------------------PhysicalOlapScan[date_dim]
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_dep_count = 1)))
+--------------------------filter(hd_dep_count IN (1, 3))
 ----------------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalDistribute
 --------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
index bc3047ea873d11..1a75b584a26c1e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query48.out
@@ -13,7 +13,7 @@ PhysicalResultSink
 --------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'S') AND (customer_demographics.cd_education_status = 'Secondary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'M') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = 'Advanced Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+--------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
index 49c9575af06644..6d3a45d8a53b8c 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query85.out
@@ -35,7 +35,7 @@ PhysicalResultSink
 --------------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF1 RF2 RF3 RF9
 ------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
 --------------------------------------------------PhysicalProject
-----------------------------------------------------filter(((((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00)) OR ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00))) OR ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))) and ((((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00)) OR ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00))) OR ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))
+----------------------------------------------------filter((web_sales.ws_net_profit <= 300.00) and (web_sales.ws_net_profit >= 50.00) and (web_sales.ws_sales_price <= 200.00) and (web_sales.ws_sales_price >= 50.00))
 ------------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
 --------------------------------------------------PhysicalDistribute
 ----------------------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
index ad8eecd71597f5..cb98071908d717 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query13.out
@@ -14,7 +14,7 @@ PhysicalResultSink
 ----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
-----------------------------filter(((((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+----------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF2 RF3 RF4
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
@@ -29,7 +29,7 @@ PhysicalResultSink
 ------------------------PhysicalOlapScan[customer_address]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_dep_count = 1)))
+--------------------filter(hd_dep_count IN (1, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalDistribute
 --------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
index 6d35acaa62fba3..a1e2aad9c5f9a1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query48.out
@@ -13,7 +13,7 @@ PhysicalResultSink
 --------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+--------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF2 RF3
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
index 34cd3f8686b6de..fc66cb90113f7e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query85.out
@@ -22,12 +22,12 @@ PhysicalResultSink
 --------------------------------------hashJoin[INNER_JOIN] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF2 cd_demo_sk->[wr_refunded_cdemo_sk]
 ----------------------------------------PhysicalDistribute
 ------------------------------------------PhysicalProject
---------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF0 wr_item_sk->[ws_item_sk];RF1 wr_order_number->[ws_order_number]
+--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF0 ws_item_sk->[wr_item_sk];RF1 ws_order_number->[wr_order_number]
 ----------------------------------------------PhysicalProject
-------------------------------------------------filter(((((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00)) OR ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00))) OR ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))) and ((((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00)) OR ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00))) OR ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))
---------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF8
+------------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF0 RF1 RF2 RF7
 ----------------------------------------------PhysicalProject
-------------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF2 RF7
+------------------------------------------------filter((web_sales.ws_net_profit <= 300.00) and (web_sales.ws_net_profit >= 50.00) and (web_sales.ws_sales_price <= 200.00) and (web_sales.ws_sales_price >= 50.00))
+--------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8
 ----------------------------------------PhysicalDistribute
 ------------------------------------------PhysicalProject
 --------------------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
index 43c19e74dd58d2..422ab6674da3e3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query13.out
@@ -14,7 +14,7 @@ PhysicalResultSink
 ----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=() build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
-----------------------------filter(((((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+----------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
@@ -29,7 +29,7 @@ PhysicalResultSink
 ------------------------PhysicalOlapScan[customer_address]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
---------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_dep_count = 1)))
+--------------------filter(hd_dep_count IN (1, 3))
 ----------------------PhysicalOlapScan[household_demographics]
 ------------PhysicalDistribute
 --------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
index fe9aa0edcfaf7b..9d3aeff4bc49d3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query48.out
@@ -13,7 +13,7 @@ PhysicalResultSink
 --------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+--------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
index e222cc74b107fe..1e0b72101a2798 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query85.out
@@ -22,12 +22,12 @@ PhysicalResultSink
 --------------------------------------hashJoin[INNER_JOIN] hashCondition=((cd1.cd_demo_sk = web_returns.wr_refunded_cdemo_sk)) otherCondition=((((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) AND ((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00))) OR (((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary')) AND ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00)))) OR (((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree')) AND ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))) build RFs:RF2 cd_demo_sk->[wr_refunded_cdemo_sk]
 ----------------------------------------PhysicalDistribute
 ------------------------------------------PhysicalProject
---------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF0 wr_item_sk->[ws_item_sk];RF1 wr_order_number->[ws_order_number]
+--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = web_returns.wr_item_sk) and (web_sales.ws_order_number = web_returns.wr_order_number)) otherCondition=() build RFs:RF0 ws_item_sk->[wr_item_sk];RF1 ws_order_number->[wr_order_number]
 ----------------------------------------------PhysicalProject
-------------------------------------------------filter(((((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00)) OR ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00))) OR ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))) and ((((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00)) OR ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00))) OR ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))
---------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF1 RF3 RF8
+------------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF0 RF1 RF2 RF6 RF7 RF9
 ----------------------------------------------PhysicalProject
-------------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF2 RF6 RF7 RF9
+------------------------------------------------filter((web_sales.ws_net_profit <= 300.00) and (web_sales.ws_net_profit >= 50.00) and (web_sales.ws_sales_price <= 200.00) and (web_sales.ws_sales_price >= 50.00))
+--------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF3 RF8
 ----------------------------------------PhysicalDistribute
 ------------------------------------------PhysicalProject
 --------------------------------------------filter(((((cd1.cd_marital_status = 'M') AND (cd1.cd_education_status = '4 yr Degree')) OR ((cd1.cd_marital_status = 'S') AND (cd1.cd_education_status = 'Secondary'))) OR ((cd1.cd_marital_status = 'W') AND (cd1.cd_education_status = 'Advanced Degree'))))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
index fe8d050464ee17..9565e553c2f21b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query13.out
@@ -21,7 +21,7 @@ PhysicalResultSink
 --------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------filter(((((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+--------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
@@ -33,6 +33,6 @@ PhysicalResultSink
 --------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
-----------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_dep_count = 1)))
+----------------------------filter(hd_dep_count IN (1, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
index b1668554b02b68..6f8c32bfebc5b2 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query48.out
@@ -13,7 +13,7 @@ PhysicalResultSink
 --------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+--------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
index dcf3eb64f342c7..a79804a81de44b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query85.out
@@ -30,7 +30,7 @@ PhysicalResultSink
 --------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF1 RF2 RF3
 ------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
 --------------------------------------------PhysicalProject
-----------------------------------------------filter(((((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00)) OR ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00))) OR ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))) and ((((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00)) OR ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00))) OR ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))
+----------------------------------------------filter((web_sales.ws_net_profit <= 300.00) and (web_sales.ws_net_profit >= 50.00) and (web_sales.ws_sales_price <= 200.00) and (web_sales.ws_sales_price >= 50.00))
 ------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0
 --------------------------------------------PhysicalDistribute
 ----------------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
index fe8d050464ee17..9565e553c2f21b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query13.out
@@ -21,7 +21,7 @@ PhysicalResultSink
 --------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=((((ca_state IN ('KS', 'MI', 'SD') AND ((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00))) OR (ca_state IN ('CO', 'MO', 'ND') AND ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00)))) OR (ca_state IN ('NH', 'OH', 'TX') AND ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))))) build RFs:RF0 ca_address_sk->[ss_addr_sk]
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------filter(((((store_sales.ss_net_profit >= 100.00) AND (store_sales.ss_net_profit <= 200.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 300.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 250.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+--------------------------------filter((store_sales.ss_net_profit <= 300.00) and (store_sales.ss_net_profit >= 50.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
@@ -33,6 +33,6 @@ PhysicalResultSink
 --------------------------------PhysicalOlapScan[date_dim]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
-----------------------------filter(((household_demographics.hd_dep_count = 3) OR (household_demographics.hd_dep_count = 1)))
+----------------------------filter(hd_dep_count IN (1, 3))
 ------------------------------PhysicalOlapScan[household_demographics]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
index 6d0c8e6967b795..c16a92a82aa55b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query48.out
@@ -13,7 +13,7 @@ PhysicalResultSink
 --------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((customer_demographics.cd_demo_sk = store_sales.ss_cdemo_sk)) otherCondition=((((((customer_demographics.cd_marital_status = 'U') AND (customer_demographics.cd_education_status = 'Primary')) AND ((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00))) OR (((customer_demographics.cd_marital_status = 'W') AND (customer_demographics.cd_education_status = 'College')) AND ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00)))) OR (((customer_demographics.cd_marital_status = 'D') AND (customer_demographics.cd_education_status = '2 yr Degree')) AND ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))) build RFs:RF0 cd_demo_sk->[ss_cdemo_sk]
 ------------------------PhysicalProject
---------------------------filter(((((store_sales.ss_net_profit >= 0.00) AND (store_sales.ss_net_profit <= 2000.00)) OR ((store_sales.ss_net_profit >= 150.00) AND (store_sales.ss_net_profit <= 3000.00))) OR ((store_sales.ss_net_profit >= 50.00) AND (store_sales.ss_net_profit <= 25000.00))) and ((((store_sales.ss_sales_price >= 100.00) AND (store_sales.ss_sales_price <= 150.00)) OR ((store_sales.ss_sales_price >= 50.00) AND (store_sales.ss_sales_price <= 100.00))) OR ((store_sales.ss_sales_price >= 150.00) AND (store_sales.ss_sales_price <= 200.00))))
+--------------------------filter((store_sales.ss_net_profit <= 25000.00) and (store_sales.ss_net_profit >= 0.00) and (store_sales.ss_sales_price <= 200.00) and (store_sales.ss_sales_price >= 50.00))
 ----------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
index 83c0a393d11053..0918c3a587ada1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query85.out
@@ -30,7 +30,7 @@ PhysicalResultSink
 --------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF1 RF2 RF3 RF9
 ------------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ws_sold_date_sk]
 --------------------------------------------PhysicalProject
-----------------------------------------------filter(((((web_sales.ws_net_profit >= 100.00) AND (web_sales.ws_net_profit <= 200.00)) OR ((web_sales.ws_net_profit >= 150.00) AND (web_sales.ws_net_profit <= 300.00))) OR ((web_sales.ws_net_profit >= 50.00) AND (web_sales.ws_net_profit <= 250.00))) and ((((web_sales.ws_sales_price >= 100.00) AND (web_sales.ws_sales_price <= 150.00)) OR ((web_sales.ws_sales_price >= 50.00) AND (web_sales.ws_sales_price <= 100.00))) OR ((web_sales.ws_sales_price >= 150.00) AND (web_sales.ws_sales_price <= 200.00))))
+----------------------------------------------filter((web_sales.ws_net_profit <= 300.00) and (web_sales.ws_net_profit >= 50.00) and (web_sales.ws_sales_price <= 200.00) and (web_sales.ws_sales_price >= 50.00))
 ------------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF0 RF8
 --------------------------------------------PhysicalDistribute
 ----------------------------------------------PhysicalProject
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
index 14765ad432d4fb..05b2551171d988 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q19.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------hashJoin[INNER_JOIN] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalDistribute
 --------------PhysicalProject
-----------------filter(((((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00)) OR ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) OR ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
+----------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ------------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalDistribute
 --------------PhysicalProject
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out
index 0a7754c55f6b92..e4183539e3201d 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/nostats_rf_prune/q7.out
@@ -31,10 +31,10 @@ PhysicalResultSink
 ------------------------------PhysicalOlapScan[customer] apply RFs: RF4
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------filter(((n1.n_name = 'FRANCE') OR (n1.n_name = 'GERMANY')))
+--------------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ----------------------------PhysicalOlapScan[nation]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------filter(((n2.n_name = 'GERMANY') OR (n2.n_name = 'FRANCE')))
+----------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ------------------------PhysicalOlapScan[nation]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
index 4ad887fd529264..bfb550af233d59 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q19.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
---------------filter(((((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00)) OR ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) OR ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
+--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalDistribute
 --------------PhysicalProject
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out
index 4bfde7a8ef5394..bacb57df2d3e61 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/rf_prune/q7.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------PhysicalOlapScan[supplier] apply RFs: RF2
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------filter(((n1.n_name = 'FRANCE') OR (n1.n_name = 'GERMANY')))
+------------------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 --------------------------------PhysicalOlapScan[nation]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
@@ -34,6 +34,6 @@ PhysicalResultSink
 ------------------------------PhysicalOlapScan[customer] apply RFs: RF0
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------filter(((n2.n_name = 'GERMANY') OR (n2.n_name = 'FRANCE')))
+--------------------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ----------------------------------PhysicalOlapScan[nation]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
index 4ad887fd529264..bfb550af233d59 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q19.out
@@ -7,7 +7,7 @@ PhysicalResultSink
 --------PhysicalProject
 ----------hashJoin[INNER_JOIN] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalProject
---------------filter(((((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00)) OR ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) OR ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
+--------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ----------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalDistribute
 --------------PhysicalProject
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out
index 4bfde7a8ef5394..bacb57df2d3e61 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape/q7.out
@@ -20,7 +20,7 @@ PhysicalResultSink
 ----------------------------PhysicalOlapScan[supplier] apply RFs: RF2
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
-------------------------------filter(((n1.n_name = 'FRANCE') OR (n1.n_name = 'GERMANY')))
+------------------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 --------------------------------PhysicalOlapScan[nation]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
@@ -34,6 +34,6 @@ PhysicalResultSink
 ------------------------------PhysicalOlapScan[customer] apply RFs: RF0
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------filter(((n2.n_name = 'GERMANY') OR (n2.n_name = 'FRANCE')))
+--------------------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ----------------------------------PhysicalOlapScan[nation]
 
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
index 14765ad432d4fb..05b2551171d988 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q19.out
@@ -8,7 +8,7 @@ PhysicalResultSink
 ----------hashJoin[INNER_JOIN] hashCondition=((part.p_partkey = lineitem.l_partkey)) otherCondition=(((((((part.p_brand = 'Brand#12') AND p_container IN ('SM BOX', 'SM CASE', 'SM PACK', 'SM PKG')) AND ((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00))) AND (part.p_size <= 5)) OR ((((part.p_brand = 'Brand#23') AND p_container IN ('MED BAG', 'MED BOX', 'MED PACK', 'MED PKG')) AND ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) AND (part.p_size <= 10))) OR ((((part.p_brand = 'Brand#34') AND p_container IN ('LG BOX', 'LG CASE', 'LG PACK', 'LG PKG')) AND ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) AND (part.p_size <= 15)))) build RFs:RF0 p_partkey->[l_partkey]
 ------------PhysicalDistribute
 --------------PhysicalProject
-----------------filter(((((lineitem.l_quantity >= 1.00) AND (lineitem.l_quantity <= 11.00)) OR ((lineitem.l_quantity >= 10.00) AND (lineitem.l_quantity <= 20.00))) OR ((lineitem.l_quantity >= 20.00) AND (lineitem.l_quantity <= 30.00))) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
+----------------filter((lineitem.l_quantity <= 30.00) and (lineitem.l_quantity >= 1.00) and (lineitem.l_shipinstruct = 'DELIVER IN PERSON') and l_shipmode IN ('AIR REG', 'AIR'))
 ------------------PhysicalOlapScan[lineitem] apply RFs: RF0
 ------------PhysicalDistribute
 --------------PhysicalProject
diff --git a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out
index 0a7754c55f6b92..e4183539e3201d 100644
--- a/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out
+++ b/regression-test/data/nereids_tpch_shape_sf1000_p0/shape_no_stats/q7.out
@@ -31,10 +31,10 @@ PhysicalResultSink
 ------------------------------PhysicalOlapScan[customer] apply RFs: RF4
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------filter(((n1.n_name = 'FRANCE') OR (n1.n_name = 'GERMANY')))
+--------------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ----------------------------PhysicalOlapScan[nation]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------filter(((n2.n_name = 'GERMANY') OR (n2.n_name = 'FRANCE')))
+----------------------filter(n_name IN ('FRANCE', 'GERMANY'))
 ------------------------PhysicalOlapScan[nation]
 

From 9ff8bd2e9c995760b740800addd42eddfa8e25df Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Wed, 27 Dec 2023 11:51:32 +0800
Subject: [PATCH 054/299] [Enhancement](Wal)Support dynamic wal space limit
 (#27726)

---
 be/src/common/config.cpp                      |   7 +-
 be/src/common/config.h                        |   5 +-
 be/src/http/action/http_stream.cpp            |  55 +++--
 be/src/http/action/stream_load.cpp            |  64 +++--
 be/src/http/utils.cpp                         |   7 +-
 be/src/io/fs/local_file_system.cpp            |  13 +
 be/src/io/fs/local_file_system.h              |   1 +
 be/src/olap/wal_dirs_info.cpp                 | 224 ++++++++++++++++++
 be/src/olap/wal_dirs_info.h                   |  85 +++++++
 be/src/olap/wal_manager.cpp                   | 165 +++++++++++--
 be/src/olap/wal_manager.h                     |  37 ++-
 be/src/olap/wal_writer.cpp                    |  28 +--
 be/src/olap/wal_writer.h                      |  10 +-
 be/src/runtime/exec_env.h                     |   1 +
 be/src/runtime/group_commit_mgr.cpp           |  92 ++++++-
 be/src/runtime/group_commit_mgr.h             |  24 +-
 be/src/vec/sink/group_commit_block_sink.cpp   |  30 ++-
 be/src/vec/sink/group_commit_block_sink.h     |   2 +-
 be/src/vec/sink/writer/vwal_writer.cpp        |  10 +-
 be/src/vec/sink/writer/vwal_writer.h          |   3 +-
 be/test/http/stream_load_test.cpp             |  13 +-
 be/test/olap/wal_manager_test.cpp             | 101 +++++++-
 be/test/olap/wal_reader_writer_test.cpp       |   5 +-
 be/test/vec/exec/vtablet_sink_test.cpp        |   2 +-
 be/test/vec/exec/vwal_scanner_test.cpp        |   7 +-
 ..._group_commit_and_wal_back_pressure.groovy |   6 +-
 .../test_group_commit_wal_limit.groovy        |  76 +-----
 27 files changed, 846 insertions(+), 227 deletions(-)
 create mode 100644 be/src/olap/wal_dirs_info.cpp
 create mode 100644 be/src/olap/wal_dirs_info.h

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 9a886ff19c9207..358db99e592b24 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1099,7 +1099,7 @@ DEFINE_Int32(grace_shutdown_wait_seconds, "120");
 DEFINE_Int16(bitmap_serialize_version, "1");
 
 // group commit insert config
-DEFINE_String(group_commit_wal_path, "./wal");
+DEFINE_String(group_commit_wal_path, "");
 DEFINE_Int32(group_commit_replay_wal_retry_num, "10");
 DEFINE_Int32(group_commit_replay_wal_retry_interval_seconds, "5");
 DEFINE_Int32(group_commit_relay_wal_threads, "10");
@@ -1128,8 +1128,9 @@ DEFINE_String(default_tzfiles_path, "${DORIS_HOME}/zoneinfo");
 // Max size(bytes) of group commit queues, used for mem back pressure, defult 64M.
 DEFINE_Int32(group_commit_max_queue_size, "67108864");
 
-// Max size(bytes) of wal disk using, used for disk space back pressure, default 64M.
-DEFINE_Int32(wal_max_disk_size, "67108864");
+// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
+// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
+DEFINE_String(group_commit_wal_max_disk_limit, "10%");
 
 // Ingest binlog work pool size, -1 is disable, 0 is hardware concurrency
 DEFINE_Int32(ingest_binlog_work_pool_size, "-1");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index bb341002902ef7..a41a3d06141182 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1203,8 +1203,9 @@ DECLARE_String(default_tzfiles_path);
 // Max size(bytes) of group commit queues, used for mem back pressure.
 DECLARE_Int32(group_commit_max_queue_size);
 
-// Max size(bytes) of wal disk using, used for disk space back pressure.
-DECLARE_Int32(wal_max_disk_size);
+// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
+// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
+DECLARE_mString(group_commit_wal_max_disk_limit);
 
 // Ingest binlog work pool size
 DECLARE_Int32(ingest_binlog_work_pool_size);
diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp
index 8c75f6be7402ac..18256237a81107 100644
--- a/be/src/http/action/http_stream.cpp
+++ b/be/src/http/action/http_stream.cpp
@@ -46,6 +46,7 @@
 #include "http/utils.h"
 #include "io/fs/stream_load_pipe.h"
 #include "olap/storage_engine.h"
+#include "olap/wal_manager.h"
 #include "runtime/client_cache.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
@@ -169,31 +170,38 @@ int HttpStreamAction::on_header(HttpRequest* req) {
 
     Status st = Status::OK();
     std::string group_commit_mode = req->header(HTTP_GROUP_COMMIT);
-    if (iequal(group_commit_mode, "off_mode")) {
-        group_commit_mode = "";
-    }
     if (!group_commit_mode.empty() && !iequal(group_commit_mode, "sync_mode") &&
         !iequal(group_commit_mode, "async_mode") && !iequal(group_commit_mode, "off_mode")) {
         st = Status::InternalError("group_commit can only be [async_mode, sync_mode, off_mode]");
-        if (iequal(group_commit_mode, "off_mode")) {
-            group_commit_mode = "";
-        }
+    } else if (group_commit_mode.empty() || iequal(group_commit_mode, "off_mode")) {
+        // off_mode and empty
+        group_commit_mode = "off_mode";
+        ctx->group_commit = false;
+    } else {
+        // sync_mode and async_mode
+        ctx->group_commit = true;
     }
     ctx->two_phase_commit = req->header(HTTP_TWO_PHASE_COMMIT) == "true";
     auto temp_partitions = !req->header(HTTP_TEMP_PARTITIONS).empty();
     auto partitions = !req->header(HTTP_PARTITIONS).empty();
     if (!temp_partitions && !partitions && !ctx->two_phase_commit &&
         (!group_commit_mode.empty() || config::wait_internal_group_commit_finish)) {
-        if (config::wait_internal_group_commit_finish) {
+        if (iequal(group_commit_mode, "async_mode") || config::wait_internal_group_commit_finish) {
             ctx->group_commit = true;
-        } else {
-            ctx->group_commit = load_size_smaller_than_wal_limit(req);
-            if (!ctx->group_commit) {
-                LOG(WARNING) << "The data size for this http load("
-                             << req->header(HttpHeaders::CONTENT_LENGTH)
-                             << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
-                             << config::wal_max_disk_size * 0.8
-                             << " Bytes). Please set this load to \"group commit\"=false.";
+            group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
+            if (iequal(group_commit_mode, "sync_mode")) {
+                size_t max_available_size =
+                        ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
+                LOG(INFO) << "When enable group commit, the data size can't be too large. The data "
+                             "size "
+                             "for this http load("
+                          << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
+                                      ? 0
+                                      : std::stol(req->header(HttpHeaders::CONTENT_LENGTH)))
+                          << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
+                          << max_available_size
+                          << " Bytes). So we set this load to \"group commit\"=sync_mode "
+                             "automatically.";
                 st = Status::Error<EXCEEDED_LIMIT>("Http load size too large.");
             }
         }
@@ -358,6 +366,23 @@ Status HttpStreamAction::_process_put(HttpRequest* http_req,
     ctx->txn_id = ctx->put_result.params.txn_conf.txn_id;
     ctx->label = ctx->put_result.params.import_label;
     ctx->put_result.params.__set_wal_id(ctx->wal_id);
+    if (http_req->header(HTTP_GROUP_COMMIT) == "async mode") {
+        if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
+            size_t content_length = 0;
+            content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
+            if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
+                ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZ4FRAME ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZOP ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZ4BLOCK ||
+                ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
+                content_length *= 3;
+            }
+            RETURN_IF_ERROR(ExecEnv::GetInstance()->group_commit_mgr()->update_load_info(
+                    ctx->id.to_thrift(), content_length));
+        }
+    }
 
     return _exec_env->stream_load_executor()->execute_plan_fragment(ctx);
 }
diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index 04c29a53027823..eddf31856a03d5 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -32,6 +32,7 @@
 #include <thrift/protocol/TDebugProtocol.h>
 #include <time.h>
 
+#include <algorithm>
 #include <future>
 #include <map>
 #include <sstream>
@@ -189,15 +190,16 @@ int StreamLoadAction::on_header(HttpRequest* req) {
     ctx->label = req->header(HTTP_LABEL_KEY);
     Status st = Status::OK();
     std::string group_commit_mode = req->header(HTTP_GROUP_COMMIT);
-    if (iequal(group_commit_mode, "off_mode")) {
-        group_commit_mode = "";
-    }
     if (!group_commit_mode.empty() && !iequal(group_commit_mode, "sync_mode") &&
         !iequal(group_commit_mode, "async_mode") && !iequal(group_commit_mode, "off_mode")) {
         st = Status::InternalError("group_commit can only be [async_mode, sync_mode, off_mode]");
-        if (iequal(group_commit_mode, "off_mode")) {
-            group_commit_mode = "";
-        }
+    } else if (group_commit_mode.empty() || iequal(group_commit_mode, "off_mode")) {
+        // off_mode and empty
+        group_commit_mode = "off_mode";
+        ctx->group_commit = false;
+    } else {
+        // sync_mode and async_mode
+        ctx->group_commit = true;
     }
     auto partial_columns = !req->header(HTTP_PARTIAL_COLUMNS).empty() &&
                            iequal(req->header(HTTP_PARTIAL_COLUMNS), "true");
@@ -206,19 +208,26 @@ int StreamLoadAction::on_header(HttpRequest* req) {
     auto partitions = !req->header(HTTP_PARTITIONS).empty();
     if (!partial_columns && !partitions && !temp_partitions && !ctx->two_phase_commit &&
         (!group_commit_mode.empty() || config::wait_internal_group_commit_finish)) {
-        if (!group_commit_mode.empty() && !ctx->label.empty()) {
+        if (!config::wait_internal_group_commit_finish && ctx->group_commit &&
+            !ctx->label.empty()) {
             st = Status::InternalError("label and group_commit can't be set at the same time");
         }
-        if (config::wait_internal_group_commit_finish) {
+        if (iequal(group_commit_mode, "async_mode") || config::wait_internal_group_commit_finish) {
             ctx->group_commit = true;
-        } else {
-            ctx->group_commit = load_size_smaller_than_wal_limit(req);
-            if (!ctx->group_commit) {
-                LOG(WARNING) << "The data size for this stream load("
-                             << req->header(HttpHeaders::CONTENT_LENGTH)
-                             << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
-                             << config::wal_max_disk_size * 0.8
-                             << " Bytes). Please set this load to \"group commit\"=false.";
+            group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
+            if (iequal(group_commit_mode, "sync_mode")) {
+                size_t max_available_size =
+                        ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
+                LOG(INFO) << "When enable group commit, the data size can't be too large. The data "
+                             "size "
+                             "for this stream load("
+                          << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
+                                      ? 0
+                                      : std::stol(req->header(HttpHeaders::CONTENT_LENGTH)))
+                          << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
+                          << max_available_size
+                          << " Bytes). So we set this load to \"group commit\"=sync_mode "
+                             "automatically.";
                 st = Status::Error<EXCEEDED_LIMIT>("Stream load size too large.");
             }
         }
@@ -640,11 +649,7 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
         request.__set_memtable_on_sink_node(value);
     }
     if (ctx->group_commit) {
-        if (!http_req->header(HTTP_GROUP_COMMIT).empty()) {
-            request.__set_group_commit_mode(http_req->header(HTTP_GROUP_COMMIT));
-        } else {
-            request.__set_group_commit_mode("sync_mode");
-        }
+        request.__set_group_commit_mode(http_req->header(HTTP_GROUP_COMMIT));
     }
 
 #ifndef BE_TEST
@@ -665,6 +670,23 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
         LOG(WARNING) << "plan streaming load failed. errmsg=" << plan_status << ctx->brief();
         return plan_status;
     }
+    if (http_req->header(HTTP_GROUP_COMMIT) == "async_mode") {
+        if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
+            size_t content_length = 0;
+            content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
+            if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
+                ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZ4FRAME ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZOP ||
+                ctx->format == TFileFormatType::FORMAT_CSV_LZ4BLOCK ||
+                ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
+                content_length *= 3;
+            }
+            RETURN_IF_ERROR(ExecEnv::GetInstance()->group_commit_mgr()->update_load_info(
+                    ctx->id.to_thrift(), content_length));
+        }
+    }
 
     VLOG_NOTICE << "params is " << apache::thrift::ThriftDebugString(ctx->put_result.params);
     // if we not use streaming, we must download total content before we begin
diff --git a/be/src/http/utils.cpp b/be/src/http/utils.cpp
index 0d66887663f849..9fbf4c79300dcd 100644
--- a/be/src/http/utils.cpp
+++ b/be/src/http/utils.cpp
@@ -22,6 +22,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
+#include <algorithm>
 #include <memory>
 #include <ostream>
 #include <vector>
@@ -38,6 +39,8 @@
 #include "http/http_status.h"
 #include "io/fs/file_system.h"
 #include "io/fs/local_file_system.h"
+#include "olap/wal_manager.h"
+#include "runtime/exec_env.h"
 #include "util/path_util.h"
 #include "util/url_coding.h"
 
@@ -199,8 +202,8 @@ bool load_size_smaller_than_wal_limit(HttpRequest* req) {
     // these blocks within the limited space. So we need to set group_commit = false to avoid dead lock.
     if (!req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
         size_t body_bytes = std::stol(req->header(HttpHeaders::CONTENT_LENGTH));
-        // TODO(Yukang): change it to WalManager::wal_limit
-        return (body_bytes <= config::wal_max_disk_size * 0.8) && (body_bytes != 0);
+        size_t max_available_size = ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
+        return (body_bytes != 0 && body_bytes < 0.8 * max_available_size);
     } else {
         return false;
     }
diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp
index 83e1554fe6b48f..a71708ac72c824 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -172,6 +172,19 @@ Status LocalFileSystem::file_size_impl(const Path& file, int64_t* file_size) con
     return Status::OK();
 }
 
+Status LocalFileSystem::directory_size(const Path& dir_path, size_t* dir_size) {
+    *dir_size = 0;
+    if (std::filesystem::exists(dir_path) && std::filesystem::is_directory(dir_path)) {
+        for (const auto& entry : std::filesystem::recursive_directory_iterator(dir_path)) {
+            if (std::filesystem::is_regular_file(entry)) {
+                *dir_size += std::filesystem::file_size(entry);
+            }
+        }
+        return Status::OK();
+    }
+    return Status::IOError("faile to get dir size {}", dir_path.native());
+}
+
 Status LocalFileSystem::list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
                                   bool* exists) {
     RETURN_IF_ERROR(exists_impl(dir, exists));
diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h
index 53df0b8e06bf44..4f7e5107fb3eb6 100644
--- a/be/src/io/fs/local_file_system.h
+++ b/be/src/io/fs/local_file_system.h
@@ -78,6 +78,7 @@ class LocalFileSystem final : public FileSystem {
     // "safe" means the path will be concat with the path prefix config::user_files_secure_path,
     // so that it can not list any files outside the config::user_files_secure_path
     Status safe_glob(const std::string& path, std::vector<FileInfo>* res);
+    Status directory_size(const Path& dir_path, size_t* dir_size);
 
 protected:
     Status create_file_impl(const Path& file, FileWriterPtr* writer,
diff --git a/be/src/olap/wal_dirs_info.cpp b/be/src/olap/wal_dirs_info.cpp
new file mode 100644
index 00000000000000..340d896a8c60db
--- /dev/null
+++ b/be/src/olap/wal_dirs_info.cpp
@@ -0,0 +1,224 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/wal_dirs_info.h"
+
+#include <memory>
+#include <mutex>
+#include <shared_mutex>
+
+#include "common/config.h"
+#include "common/status.h"
+#include "io/fs/local_file_system.h"
+#include "util/parse_util.h"
+
+namespace doris {
+
+std::string WalDirInfo::get_wal_dir() {
+    return _wal_dir;
+}
+
+size_t WalDirInfo::get_limit() {
+    std::shared_lock rlock(_lock);
+    return _limit;
+}
+
+size_t WalDirInfo::get_used() {
+    std::shared_lock rlock(_lock);
+    return _used;
+}
+
+size_t WalDirInfo::get_pre_allocated() {
+    std::shared_lock rlock(_lock);
+    return _pre_allocated;
+}
+
+Status WalDirInfo::set_limit(size_t limit) {
+    std::unique_lock wlock(_lock);
+    _limit = limit;
+    return Status::OK();
+}
+
+Status WalDirInfo::set_used(size_t used) {
+    std::unique_lock wlock(_lock);
+    _used = used;
+    return Status::OK();
+}
+
+Status WalDirInfo::set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated) {
+    std::unique_lock wlock(_lock);
+    if (is_add_pre_allocated) {
+        _pre_allocated += pre_allocated;
+    } else {
+        _pre_allocated -= pre_allocated;
+    }
+    return Status::OK();
+}
+
+size_t WalDirInfo::available() {
+    std::unique_lock wlock(_lock);
+    int64_t available = _limit - _used - _pre_allocated;
+    return available > 0 ? available : 0;
+}
+
+Status WalDirInfo::update_wal_dir_limit(size_t limit) {
+    if (limit != static_cast<size_t>(-1)) {
+        RETURN_IF_ERROR(set_limit(limit));
+    } else {
+        size_t available_bytes;
+        size_t disk_capacity_bytes;
+        RETURN_IF_ERROR(io::global_local_filesystem()->get_space_info(
+                _wal_dir, &disk_capacity_bytes, &available_bytes));
+        bool is_percent = true;
+        int64_t wal_disk_limit = ParseUtil::parse_mem_spec(config::group_commit_wal_max_disk_limit,
+                                                           -1, available_bytes, &is_percent);
+        if (wal_disk_limit <= 0) {
+            return Status::InternalError("Disk full! Please check your disk usage!");
+        }
+        size_t wal_dir_size = 0;
+        RETURN_IF_ERROR(io::global_local_filesystem()->directory_size(_wal_dir, &wal_dir_size));
+        RETURN_IF_ERROR(set_limit(wal_disk_limit));
+    }
+    return Status::OK();
+}
+
+Status WalDirInfo::update_wal_dir_used(size_t used) {
+    if (used != static_cast<size_t>(-1)) {
+        RETURN_IF_ERROR(set_used(used));
+    } else {
+        size_t wal_dir_size = 0;
+        RETURN_IF_ERROR(io::global_local_filesystem()->directory_size(_wal_dir, &wal_dir_size));
+        RETURN_IF_ERROR(set_used(wal_dir_size));
+    }
+    return Status::OK();
+}
+
+Status WalDirInfo::update_wal_dir_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated) {
+    RETURN_IF_ERROR(set_pre_allocated(pre_allocated, is_add_pre_allocated));
+    return Status::OK();
+}
+
+Status WalDirsInfo::add(const std::string& wal_dir, size_t limit, size_t used,
+                        size_t pre_allocated) {
+    for (const auto& it : _wal_dirs_info_vec) {
+        if (it->get_wal_dir() == wal_dir) {
+#ifdef BE_TEST
+            return Status::OK();
+#endif
+            return Status::InternalError("wal dir {} exists!", wal_dir);
+        }
+    }
+    std::unique_lock wlock(_lock);
+    _wal_dirs_info_vec.emplace_back(
+            std::make_shared<WalDirInfo>(wal_dir, limit, used, pre_allocated));
+    return Status::OK();
+}
+
+Status WalDirsInfo::clear() {
+    std::unique_lock wlock(_lock);
+    _wal_dirs_info_vec.clear();
+    return Status::OK();
+}
+
+std::string WalDirsInfo::get_available_random_wal_dir() {
+    if (_wal_dirs_info_vec.size() == 1) {
+        return (*_wal_dirs_info_vec.begin())->get_wal_dir();
+    } else {
+        std::vector<std::string> available_wal_dirs;
+        for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+            if (wal_dir_info->available() > wal_dir_info->get_limit() * 0.2) {
+                available_wal_dirs.emplace_back(wal_dir_info->get_wal_dir());
+            }
+        }
+        if (available_wal_dirs.empty()) {
+            return (*std::min_element(_wal_dirs_info_vec.begin(), _wal_dirs_info_vec.end(),
+                                      [](const auto& info1, const auto& info2) {
+                                          return info1->available() < info2->available();
+                                      }))
+                    ->get_wal_dir();
+        } else {
+            return (*std::next(_wal_dirs_info_vec.begin(), rand() % _wal_dirs_info_vec.size()))
+                    ->get_wal_dir();
+        }
+    }
+}
+
+size_t WalDirsInfo::get_max_available_size() {
+    return _wal_dirs_info_vec.size() == 1
+                   ? (*_wal_dirs_info_vec.begin())->available()
+                   : (*std::max_element(_wal_dirs_info_vec.begin(), _wal_dirs_info_vec.end(),
+                                        [](const auto& info1, const auto& info2) {
+                                            return info1->available() < info2->available();
+                                        }))
+                             ->available();
+}
+
+Status WalDirsInfo::update_wal_dir_limit(std::string wal_dir, size_t limit) {
+    for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+        if (wal_dir_info->get_wal_dir() == wal_dir) {
+            return wal_dir_info->update_wal_dir_limit(limit);
+        }
+    }
+    return Status::InternalError("Can not find wal dir in wal disks info.");
+}
+
+Status WalDirsInfo::update_all_wal_dir_limit() {
+    for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+        RETURN_IF_ERROR(wal_dir_info->update_wal_dir_limit(-1));
+    }
+    return Status::OK();
+}
+
+Status WalDirsInfo::update_wal_dir_used(std::string wal_dir, size_t used) {
+    for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+        if (wal_dir_info->get_wal_dir() == wal_dir) {
+            return wal_dir_info->update_wal_dir_used(used);
+        }
+    }
+    return Status::InternalError("Can not find wal dir in wal disks info.");
+}
+
+Status WalDirsInfo::update_all_wal_dir_used() {
+    for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+        RETURN_IF_ERROR(wal_dir_info->update_wal_dir_used(-1));
+    }
+    return Status::OK();
+}
+
+Status WalDirsInfo::update_wal_dir_pre_allocated(std::string wal_dir, size_t pre_allocated,
+                                                 bool is_add_pre_allocated) {
+    for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+        if (wal_dir_info->get_wal_dir() == wal_dir) {
+            return wal_dir_info->update_wal_dir_pre_allocated(pre_allocated, is_add_pre_allocated);
+        }
+    }
+    return Status::InternalError("Can not find wal dir in wal disks info.");
+}
+
+Status WalDirsInfo::get_wal_dir_available_size(const std::string& wal_dir,
+                                               size_t* available_bytes) {
+    std::shared_lock l(_lock);
+    for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+        if (wal_dir_info->get_wal_dir() == wal_dir) {
+            *available_bytes = wal_dir_info->available();
+            return Status::OK();
+        }
+    }
+    return Status::InternalError("can not find wal dir!");
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_dirs_info.h b/be/src/olap/wal_dirs_info.h
new file mode 100644
index 00000000000000..048cd8f9564362
--- /dev/null
+++ b/be/src/olap/wal_dirs_info.h
@@ -0,0 +1,85 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <memory>
+#include <mutex>
+#include <shared_mutex>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "common/factory_creator.h"
+#include "common/status.h"
+
+namespace doris {
+class WalDirInfo {
+    ENABLE_FACTORY_CREATOR(WalDirInfo);
+
+public:
+    WalDirInfo(std::string wal_dir, size_t limit, size_t used, size_t pre_allocated)
+            : _wal_dir(std::move(wal_dir)),
+              _limit(limit),
+              _used(used),
+              _pre_allocated(pre_allocated) {}
+    std::string get_wal_dir();
+    size_t get_limit();
+    size_t get_used();
+    size_t get_pre_allocated();
+    Status set_limit(size_t limit);
+    Status set_used(size_t used);
+    Status set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated);
+    size_t available();
+    Status update_wal_dir_limit(size_t limit = -1);
+    Status update_wal_dir_used(size_t used = -1);
+    Status update_wal_dir_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated = true);
+
+private:
+    std::string _wal_dir;
+    size_t _limit;
+    size_t _used;
+    size_t _pre_allocated;
+    std::shared_mutex _lock;
+};
+
+class WalDirsInfo {
+    ENABLE_FACTORY_CREATOR(WalDirsInfo);
+
+public:
+    WalDirsInfo() = default;
+    ~WalDirsInfo() = default;
+    Status add(const std::string& wal_dir, size_t limit, size_t used, size_t pre_allocated);
+    Status clear();
+    std::string get_available_random_wal_dir();
+    size_t get_max_available_size();
+    Status update_wal_dir_limit(std::string wal_dir, size_t limit = -1);
+    Status update_all_wal_dir_limit();
+    Status update_wal_dir_used(std::string wal_dir, size_t used = -1);
+    Status update_all_wal_dir_used();
+    Status update_wal_dir_pre_allocated(std::string wal_dir, size_t pre_allocated,
+                                        bool is_add_pre_allocated);
+    Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
+
+private:
+    std::vector<std::shared_ptr<WalDirInfo>> _wal_dirs_info_vec;
+    std::shared_mutex _lock;
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_manager.cpp b/be/src/olap/wal_manager.cpp
index 238f88ebd00707..c62996da19b415 100644
--- a/be/src/olap/wal_manager.cpp
+++ b/be/src/olap/wal_manager.cpp
@@ -17,6 +17,7 @@
 
 #include "olap/wal_manager.h"
 
+#include <glog/logging.h>
 #include <thrift/protocol/TDebugProtocol.h>
 
 #include <atomic>
@@ -24,14 +25,24 @@
 #include <cstdint>
 #include <filesystem>
 #include <memory>
+#include <mutex>
+#include <shared_mutex>
+#include <thread>
+#include <unordered_map>
 #include <utility>
+#include <vector>
 
+#include "common/config.h"
+#include "common/status.h"
 #include "io/fs/local_file_system.h"
+#include "olap/wal_dirs_info.h"
 #include "olap/wal_writer.h"
 #include "runtime/client_cache.h"
+#include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/plan_fragment_executor.h"
 #include "runtime/stream_load/stream_load_context.h"
+#include "util/parse_util.h"
 #include "util/path_util.h"
 #include "util/thrift_rpc_helper.h"
 #include "vec/exec/format/wal/wal_reader.h"
@@ -40,12 +51,11 @@ namespace doris {
 WalManager::WalManager(ExecEnv* exec_env, const std::string& wal_dir_list)
         : _exec_env(exec_env), _stop_background_threads_latch(1), _stop(false) {
     doris::vectorized::WalReader::string_split(wal_dir_list, ";", _wal_dirs);
-    _all_wal_disk_bytes = std::make_shared<std::atomic_size_t>(0);
-    _cv = std::make_shared<std::condition_variable>();
     static_cast<void>(ThreadPoolBuilder("GroupCommitReplayWalThreadPool")
                               .set_min_threads(1)
                               .set_max_threads(config::group_commit_relay_wal_threads)
                               .build(&_thread_pool));
+    _wal_dirs_info = WalDirsInfo::create_unique();
 }
 
 WalManager::~WalManager() {
@@ -60,12 +70,46 @@ void WalManager::stop() {
         if (_replay_thread) {
             _replay_thread->join();
         }
+        if (_update_wal_dirs_info_thread) {
+            _update_wal_dirs_info_thread->join();
+        }
         _thread_pool->shutdown();
         LOG(INFO) << "WalManager is stopped";
     }
 }
 
 Status WalManager::init() {
+    RETURN_IF_ERROR(_init_wal_dirs_conf());
+    RETURN_IF_ERROR(_init_wal_dirs());
+    RETURN_IF_ERROR(_init_wal_dirs_info());
+    return Thread::create(
+            "WalMgr", "replay_wal", [this]() { static_cast<void>(this->replay()); },
+            &_replay_thread);
+}
+
+Status WalManager::_init_wal_dirs_conf() {
+    std::vector<std::string> tmp_dirs;
+    if (_wal_dirs.empty()) {
+        // default case.
+        for (const StorePath& path : ExecEnv::GetInstance()->store_paths()) {
+            tmp_dirs.emplace_back(path.path + "/wal");
+        }
+    } else {
+        // user config must be absolute path.
+        for (const std::string& wal_dir : _wal_dirs) {
+            if (std::filesystem::path(wal_dir).is_absolute()) {
+                tmp_dirs.emplace_back(wal_dir);
+            } else {
+                return Status::InternalError(
+                        "BE config group_commit_replay_wal_dir has to be absolute path!");
+            }
+        }
+    }
+    _wal_dirs = tmp_dirs;
+    return Status::OK();
+}
+
+Status WalManager::_init_wal_dirs() {
     bool exists = false;
     for (auto wal_dir : _wal_dirs) {
         std::string tmp_dir = wal_dir + "/tmp";
@@ -80,9 +124,44 @@ Status WalManager::init() {
         }
         RETURN_IF_ERROR(scan_wals(wal_dir));
     }
+    return Status::OK();
+}
+
+Status WalManager::_init_wal_dirs_info() {
+    for (const std::string& wal_dir : _wal_dirs) {
+        size_t available_bytes;
+#ifndef BE_TEST
+        size_t disk_capacity_bytes;
+        RETURN_IF_ERROR(io::global_local_filesystem()->get_space_info(wal_dir, &disk_capacity_bytes,
+                                                                      &available_bytes));
+#else
+        available_bytes = wal_limit_test_bytes;
+#endif
+        bool is_percent = true;
+        int64_t wal_disk_limit = ParseUtil::parse_mem_spec(config::group_commit_wal_max_disk_limit,
+                                                           -1, available_bytes, &is_percent);
+        if (wal_disk_limit < 0) {
+            return Status::InternalError(
+                    "group_commit_wal_max_disk_limit config is wrong, please check your config!");
+        }
+        // if there are some wal files in wal dir, we need to add it to wal disk limit.
+        size_t wal_dir_size = 0;
+#ifndef BE_TEST
+        RETURN_IF_ERROR(io::global_local_filesystem()->directory_size(wal_dir, &wal_dir_size));
+#endif
+        if (is_percent) {
+            wal_disk_limit += wal_dir_size;
+        }
+        RETURN_IF_ERROR(_wal_dirs_info->add(wal_dir, wal_disk_limit, wal_dir_size, 0));
+
+#ifdef BE_TEST
+        wal_limit_test_bytes = wal_disk_limit;
+#endif
+    }
     return Thread::create(
-            "WalMgr", "replay_wal", [this]() { static_cast<void>(this->replay()); },
-            &_replay_thread);
+            "WalMgr", "update_wal_dir_info",
+            [this]() { static_cast<void>(this->_update_wal_dir_info_thread()); },
+            &_update_wal_dirs_info_thread);
 }
 
 void WalManager::add_wal_status_queue(int64_t table_id, int64_t wal_id, WAL_STATUS wal_status) {
@@ -158,9 +237,8 @@ void WalManager::print_wal_status_queue() {
 }
 
 Status WalManager::add_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id,
-                                const std::string& label) {
-    std::string base_path =
-            _wal_dirs.size() == 1 ? _wal_dirs[0] : _wal_dirs[rand() % _wal_dirs.size()];
+                                const std::string& label, std::string& base_path) {
+    base_path = _wal_dirs_info->get_available_random_wal_dir();
     std::stringstream ss;
     ss << base_path << "/" << std::to_string(db_id) << "/" << std::to_string(table_id) << "/"
        << std::to_string(wal_id) << "_" << label;
@@ -209,7 +287,7 @@ Status WalManager::create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>&
         RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(base_path));
     }
     LOG(INFO) << "create wal " << wal_path;
-    wal_writer = std::make_shared<WalWriter>(wal_path, _all_wal_disk_bytes, _cv);
+    wal_writer = std::make_shared<WalWriter>(wal_path);
     RETURN_IF_ERROR(wal_writer->init());
     {
         std::lock_guard<std::shared_mutex> wrlock(_wal_lock);
@@ -250,7 +328,7 @@ Status WalManager::scan_wals(const std::string& wal_path) {
                              << ", st=" << st.to_string();
                 return st;
             }
-            if (wals.size() == 0) {
+            if (wals.empty()) {
                 continue;
             }
             std::vector<std::string> res;
@@ -331,6 +409,12 @@ Status WalManager::add_recover_wal(int64_t db_id, int64_t table_id, std::vector<
         table_ptr = it->second;
     }
     table_ptr->add_wals(wals);
+#ifndef BE_TEST
+    for (auto wal : wals) {
+        RETURN_IF_ERROR(update_wal_dir_limit(_get_base_wal_path(wal)));
+        RETURN_IF_ERROR(update_wal_dir_used(_get_base_wal_path(wal)));
+    }
+#endif
     return Status::OK();
 }
 
@@ -344,31 +428,20 @@ size_t WalManager::get_wal_table_size(int64_t table_id) {
     }
 }
 
-Status WalManager::delete_wal(int64_t wal_id) {
+Status WalManager::delete_wal(int64_t wal_id, size_t block_queue_pre_allocated) {
+    std::string wal_path;
     {
         std::lock_guard<std::shared_mutex> wrlock(_wal_lock);
-        if (_wal_id_to_writer_map.find(wal_id) != _wal_id_to_writer_map.end()) {
-            _all_wal_disk_bytes->fetch_sub(_wal_id_to_writer_map[wal_id]->disk_bytes(),
-                                           std::memory_order_relaxed);
-            _cv->notify_one();
-            std::string wal_path = _wal_path_map[wal_id];
-            LOG(INFO) << "wal delete file=" << wal_path << ", this file disk usage is "
-                      << _wal_id_to_writer_map[wal_id]->disk_bytes()
-                      << " ,after deleting it, all wals disk usage is "
-                      << _all_wal_disk_bytes->load(std::memory_order_relaxed);
-            _wal_id_to_writer_map.erase(wal_id);
-        }
-        if (_wal_id_to_writer_map.empty()) {
-            CHECK_EQ(_all_wal_disk_bytes->load(std::memory_order_relaxed), 0);
-        }
         auto it = _wal_path_map.find(wal_id);
         if (it != _wal_path_map.end()) {
-            std::string wal_path = it->second;
+            wal_path = it->second;
             RETURN_IF_ERROR(io::global_local_filesystem()->delete_file(wal_path));
             LOG(INFO) << "delete file=" << wal_path;
             _wal_path_map.erase(wal_id);
         }
     }
+    RETURN_IF_ERROR(update_wal_dir_pre_allocated(_get_base_wal_path(wal_path),
+                                                 block_queue_pre_allocated, false));
     return Status::OK();
 }
 
@@ -409,4 +482,46 @@ Status WalManager::get_wal_column_index(int64_t wal_id, std::vector<size_t>& col
     return Status::OK();
 }
 
+size_t WalManager::get_max_available_size() {
+    return _wal_dirs_info->get_max_available_size();
+}
+
+Status WalManager::update_wal_dir_limit(const std::string& wal_dir, size_t limit) {
+    return _wal_dirs_info->update_wal_dir_limit(wal_dir, limit);
+}
+
+Status WalManager::update_wal_dir_used(const std::string& wal_dir, size_t used) {
+    return _wal_dirs_info->update_wal_dir_used(wal_dir, used);
+}
+
+Status WalManager::update_wal_dir_pre_allocated(const std::string& wal_dir, size_t pre_allocated,
+                                                bool is_add_pre_allocated) {
+    return _wal_dirs_info->update_wal_dir_pre_allocated(wal_dir, pre_allocated,
+                                                        is_add_pre_allocated);
+}
+
+Status WalManager::_update_wal_dir_info_thread() {
+    while (!_stop.load()) {
+        static_cast<void>(_wal_dirs_info->update_all_wal_dir_limit());
+        static_cast<void>(_wal_dirs_info->update_all_wal_dir_used());
+        std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    }
+    return Status::OK();
+}
+
+Status WalManager::get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes) {
+    return _wal_dirs_info->get_wal_dir_available_size(wal_dir, available_bytes);
+}
+
+std::string WalManager::_get_base_wal_path(const std::string& wal_path_str) {
+    io::Path wal_path = wal_path_str;
+    for (int i = 0; i < 3; ++i) {
+        if (!wal_path.has_parent_path()) {
+            return "";
+        }
+        wal_path = wal_path.parent_path();
+    }
+    return wal_path.string();
+}
+
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_manager.h b/be/src/olap/wal_manager.h
index 83847beabf0b85..8d330a64bb9fc8 100644
--- a/be/src/olap/wal_manager.h
+++ b/be/src/olap/wal_manager.h
@@ -16,15 +16,24 @@
 // under the License.
 
 #pragma once
+
 #include <gen_cpp/PaloInternalService_types.h>
 
+#include <atomic>
 #include <condition_variable>
+#include <cstddef>
+#include <cstdint>
 #include <memory>
+#include <shared_mutex>
+#include <thread>
+#include <unordered_map>
 
 #include "common/config.h"
 #include "gen_cpp/FrontendService.h"
 #include "gen_cpp/FrontendService_types.h"
 #include "gen_cpp/HeartbeatService_types.h"
+#include "gutil/ref_counted.h"
+#include "olap/wal_dirs_info.h"
 #include "olap/wal_reader.h"
 #include "olap/wal_table.h"
 #include "olap/wal_writer.h"
@@ -47,7 +56,7 @@ class WalManager {
 public:
     WalManager(ExecEnv* exec_env, const std::string& wal_dir);
     ~WalManager();
-    Status delete_wal(int64_t wal_id);
+    Status delete_wal(int64_t wal_id, size_t block_queue_pre_allocated = 0);
     Status init();
     Status scan_wals(const std::string& wal_path);
     Status replay();
@@ -56,7 +65,8 @@ class WalManager {
     Status scan();
     size_t get_wal_table_size(int64_t table_id);
     Status add_recover_wal(int64_t db_id, int64_t table_id, std::vector<std::string> wals);
-    Status add_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id, const std::string& label);
+    Status add_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id, const std::string& label,
+                        std::string& base_path);
     Status get_wal_path(int64_t wal_id, std::string& wal_path);
     Status get_wal_status_queue_size(const PGetWalQueueSizeRequest* request,
                                      PGetWalQueueSizeResponse* response);
@@ -70,10 +80,30 @@ class WalManager {
     void erase_wal_column_index(int64_t wal_id);
     Status get_wal_column_index(int64_t wal_id, std::vector<size_t>& column_index);
 
+    Status update_wal_dir_limit(const std::string& wal_dir, size_t limit = -1);
+    Status update_wal_dir_used(const std::string& wal_dir, size_t used = -1);
+    Status update_wal_dir_pre_allocated(const std::string& wal_dir, size_t pre_allocated,
+                                        bool is_add_pre_allocated);
+    Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
+    size_t get_max_available_size();
+
+private:
+    Status _init_wal_dirs_conf();
+    Status _init_wal_dirs();
+    Status _init_wal_dirs_info();
+    std::string _get_base_wal_path(const std::string& wal_path_str);
+    const std::string& _get_available_random_wal_dir();
+    Status _update_wal_dir_info_thread();
+
+public:
+    // used for be ut
+    size_t wal_limit_test_bytes;
+
 private:
     ExecEnv* _exec_env = nullptr;
     std::shared_mutex _lock;
     scoped_refptr<Thread> _replay_thread;
+    scoped_refptr<Thread> _update_wal_dirs_info_thread;
     CountDownLatch _stop_background_threads_latch;
     std::map<int64_t, std::shared_ptr<WalTable>> _table_map;
     std::vector<std::string> _wal_dirs;
@@ -81,12 +111,11 @@ class WalManager {
     std::shared_mutex _wal_status_lock;
     std::unordered_map<int64_t, std::string> _wal_path_map;
     std::unordered_map<int64_t, std::shared_ptr<WalWriter>> _wal_id_to_writer_map;
-    std::shared_ptr<std::atomic_size_t> _all_wal_disk_bytes;
     std::unordered_map<int64_t, std::unordered_map<int64_t, WAL_STATUS>> _wal_status_queues;
     std::atomic<bool> _stop;
     std::shared_mutex _wal_column_id_map_lock;
     std::unordered_map<int64_t, std::vector<size_t>&> _wal_column_id_map;
-    std::shared_ptr<std::condition_variable> _cv;
     std::unique_ptr<doris::ThreadPool> _thread_pool;
+    std::unique_ptr<WalDirsInfo> _wal_dirs_info;
 };
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_writer.cpp b/be/src/olap/wal_writer.cpp
index 9d3da90d887386..beae6ec80b6c23 100644
--- a/be/src/olap/wal_writer.cpp
+++ b/be/src/olap/wal_writer.cpp
@@ -26,6 +26,7 @@
 #include "io/fs/local_file_system.h"
 #include "io/fs/path.h"
 #include "olap/storage_engine.h"
+#include "olap/wal_manager.h"
 #include "util/crc32c.h"
 
 namespace doris {
@@ -33,14 +34,7 @@ namespace doris {
 const char* k_wal_magic = "WAL1";
 const uint32_t k_wal_magic_length = 4;
 
-WalWriter::WalWriter(const std::string& file_name,
-                     const std::shared_ptr<std::atomic_size_t>& all_wal_disk_bytes,
-                     const std::shared_ptr<std::condition_variable>& cv)
-        : cv(cv),
-          _file_name(file_name),
-          _disk_bytes(0),
-          _all_wal_disk_bytes(all_wal_disk_bytes),
-          _is_first_append_blocks(true) {}
+WalWriter::WalWriter(const std::string& file_name) : _file_name(file_name) {}
 
 WalWriter::~WalWriter() {}
 
@@ -58,22 +52,6 @@ Status WalWriter::finalize() {
 }
 
 Status WalWriter::append_blocks(const PBlockArray& blocks) {
-    {
-        if (_is_first_append_blocks) {
-            _is_first_append_blocks = false;
-            std::unique_lock l(_mutex);
-            while (_all_wal_disk_bytes->load(std::memory_order_relaxed) >
-                   config::wal_max_disk_size) {
-                LOG(INFO) << "First time to append blocks to wal file " << _file_name
-                          << ". Currently, all wal disk space usage is "
-                          << _all_wal_disk_bytes->load(std::memory_order_relaxed)
-                          << ", larger than the maximum limit " << config::wal_max_disk_size
-                          << ", so we need to wait. When any other load finished, that wal will be "
-                             "removed, the space used by that wal will be free.";
-                cv->wait_for(l, std::chrono::milliseconds(WalWriter::MAX_WAL_WRITE_WAIT_TIME));
-            }
-        }
-    }
     size_t total_size = 0;
     for (const auto& block : blocks) {
         total_size += LENGTH_SIZE + block->ByteSizeLong() + CHECKSUM_SIZE;
@@ -99,8 +77,6 @@ Status WalWriter::append_blocks(const PBlockArray& blocks) {
                 "failed to write block to wal expected= " + std::to_string(total_size) +
                 ",actually=" + std::to_string(offset));
     }
-    _disk_bytes.fetch_add(total_size, std::memory_order_relaxed);
-    _all_wal_disk_bytes->fetch_add(total_size, std::memory_order_relaxed);
     return Status::OK();
 }
 
diff --git a/be/src/olap/wal_writer.h b/be/src/olap/wal_writer.h
index 88ff4659769b1c..ea8bba4f021e54 100644
--- a/be/src/olap/wal_writer.h
+++ b/be/src/olap/wal_writer.h
@@ -34,16 +34,13 @@ extern const uint32_t k_wal_magic_length;
 
 class WalWriter {
 public:
-    explicit WalWriter(const std::string& file_name,
-                       const std::shared_ptr<std::atomic_size_t>& all_wal_disk_bytes,
-                       const std::shared_ptr<std::condition_variable>& cv);
+    explicit WalWriter(const std::string& file_name);
     ~WalWriter();
 
     Status init();
     Status finalize();
 
     Status append_blocks(const PBlockArray& blocks);
-    size_t disk_bytes() const { return _disk_bytes.load(std::memory_order_relaxed); };
     Status append_header(uint32_t version, std::string col_ids);
 
     std::string file_name() { return _file_name; };
@@ -51,17 +48,12 @@ class WalWriter {
 public:
     static const int64_t LENGTH_SIZE = 8;
     static const int64_t CHECKSUM_SIZE = 4;
-    std::shared_ptr<std::condition_variable> cv;
     static const int64_t VERSION_SIZE = 4;
 
 private:
     static constexpr size_t MAX_WAL_WRITE_WAIT_TIME = 1000;
     std::string _file_name;
     io::FileWriterPtr _file_writer;
-    std::atomic_size_t _disk_bytes;
-    std::shared_ptr<std::atomic_size_t> _all_wal_disk_bytes;
-    std::mutex _mutex;
-    bool _is_first_append_blocks;
 };
 
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 4f4325a79c7bda..4c1060891f5ffb 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -229,6 +229,7 @@ class ExecEnv {
     void set_routine_load_task_executor(RoutineLoadTaskExecutor* r) {
         this->_routine_load_task_executor = r;
     }
+    void set_wal_mgr(std::shared_ptr<WalManager> wm) { this->_wal_manager = wm; }
 
 #endif
     stream_load::LoadStreamStubPool* load_stream_stub_pool() {
diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index 201a290a6e8878..28df650f13c7b8 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -21,14 +21,22 @@
 #include <glog/logging.h>
 
 #include <atomic>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <shared_mutex>
+#include <vector>
 
 #include "client_cache.h"
 #include "common/config.h"
+#include "common/status.h"
 #include "olap/wal_manager.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/runtime_state.h"
 #include "util/thrift_rpc_helper.h"
+#include "vec/core/block.h"
 
 namespace doris {
 
@@ -46,7 +54,6 @@ Status LoadBlockQueue::add_block(std::shared_ptr<vectorized::Block> block, bool
             RETURN_IF_ERROR(_v_wal_writer->write_wal(block.get()));
         }
         _all_block_queues_bytes->fetch_add(block->bytes(), std::memory_order_relaxed);
-        _single_block_queue_bytes->fetch_add(block->bytes(), std::memory_order_relaxed);
     }
     _get_cond.notify_all();
     return Status::OK();
@@ -68,7 +75,6 @@ Status LoadBlockQueue::get_block(RuntimeState* runtime_state, vectorized::Block*
     }
     while (!runtime_state->is_cancelled() && status.ok() && _block_queue.empty() &&
            (!need_commit || (need_commit && !_load_ids.empty()))) {
-        CHECK_EQ(_single_block_queue_bytes->load(), 0);
         auto left_milliseconds = _group_commit_interval_ms;
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
                                 std::chrono::steady_clock::now() - _start_time)
@@ -106,10 +112,8 @@ Status LoadBlockQueue::get_block(RuntimeState* runtime_state, vectorized::Block*
         *find_block = true;
         _block_queue.pop_front();
         _all_block_queues_bytes->fetch_sub(block->bytes(), std::memory_order_relaxed);
-        _single_block_queue_bytes->fetch_sub(block->bytes(), std::memory_order_relaxed);
     }
     if (_block_queue.empty() && need_commit && _load_ids.empty()) {
-        CHECK_EQ(_single_block_queue_bytes->load(), 0);
         *eos = true;
     } else {
         *eos = false;
@@ -150,7 +154,6 @@ void LoadBlockQueue::_cancel_without_lock(const Status& st) {
         {
             auto& future_block = _block_queue.front();
             _all_block_queues_bytes->fetch_sub(future_block->bytes(), std::memory_order_relaxed);
-            _single_block_queue_bytes->fetch_sub(future_block->bytes(), std::memory_order_relaxed);
         }
         _block_queue.pop_front();
     }
@@ -333,11 +336,12 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_
                 10000L);
         result_status = Status::create(result.status);
     }
+    std::shared_ptr<LoadBlockQueue> load_block_queue;
     {
         std::lock_guard<std::mutex> l(_lock);
         auto it = _load_block_queues.find(instance_id);
         if (it != _load_block_queues.end()) {
-            auto& load_block_queue = it->second;
+            load_block_queue = it->second;
             if (!status.ok()) {
                 load_block_queue->cancel(status);
             }
@@ -357,7 +361,8 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_
     // result_status: commit txn result
     if (status.ok() && st.ok() &&
         (result_status.ok() || result_status.is<ErrorCode::PUBLISH_TIMEOUT>())) {
-        RETURN_IF_ERROR(_exec_env->wal_mgr()->delete_wal(txn_id));
+        RETURN_IF_ERROR(_exec_env->wal_mgr()->delete_wal(
+                txn_id, load_block_queue->block_queue_pre_allocated.load()));
         RETURN_IF_ERROR(_exec_env->wal_mgr()->erase_wal_status_queue(table_id, txn_id));
     } else {
         std::string wal_path;
@@ -443,8 +448,9 @@ Status GroupCommitMgr::get_first_block_load_queue(int64_t db_id, int64_t table_i
         }
         group_commit_table = _table_map[table_id];
     }
-    return group_commit_table->get_first_block_load_queue(table_id, base_schema_version, load_id,
-                                                          load_block_queue, be_exe_version);
+    RETURN_IF_ERROR(group_commit_table->get_first_block_load_queue(
+            table_id, base_schema_version, load_id, load_block_queue, be_exe_version));
+    return Status::OK();
 }
 
 Status GroupCommitMgr::get_load_block_queue(int64_t table_id, const TUniqueId& instance_id,
@@ -461,11 +467,14 @@ Status GroupCommitMgr::get_load_block_queue(int64_t table_id, const TUniqueId& i
     }
     return group_commit_table->get_load_block_queue(instance_id, load_block_queue);
 }
+
 Status LoadBlockQueue::create_wal(int64_t db_id, int64_t tb_id, int64_t wal_id,
                                   const std::string& import_label, WalManager* wal_manager,
                                   std::vector<TSlotDescriptor>& slot_desc, int be_exe_version) {
+    RETURN_IF_ERROR(ExecEnv::GetInstance()->wal_mgr()->add_wal_path(db_id, tb_id, wal_id,
+                                                                    import_label, wal_base_path));
     _v_wal_writer = std::make_shared<vectorized::VWalWriter>(
-            db_id, tb_id, wal_id, import_label, wal_manager, slot_desc, be_exe_version);
+            tb_id, wal_id, import_label, wal_manager, slot_desc, be_exe_version);
     return _v_wal_writer->init();
 }
 
@@ -475,4 +484,67 @@ Status LoadBlockQueue::close_wal() {
     }
     return Status::OK();
 }
+
+bool LoadBlockQueue::has_enough_wal_disk_space(
+        const std::vector<std::shared_ptr<vectorized::Block>>& blocks, const TUniqueId& load_id,
+        bool is_blocks_contain_all_load_data) {
+    size_t blocks_size = 0;
+    for (auto block : blocks) {
+        blocks_size += block->bytes();
+    }
+    size_t content_length = 0;
+    Status st = ExecEnv::GetInstance()->group_commit_mgr()->get_load_info(load_id, &content_length);
+    if (st.ok()) {
+        RETURN_IF_ERROR(ExecEnv::GetInstance()->group_commit_mgr()->remove_load_info(load_id));
+    } else {
+        return Status::InternalError("can not find load id.");
+    }
+    size_t pre_allocated = is_blocks_contain_all_load_data
+                                   ? blocks_size
+                                   : (blocks_size > content_length ? blocks_size : content_length);
+    auto* wal_mgr = ExecEnv::GetInstance()->wal_mgr();
+    size_t available_bytes = 0;
+    {
+        st = wal_mgr->get_wal_dir_available_size(wal_base_path, &available_bytes);
+        if (!st.ok()) {
+            LOG(WARNING) << "get wal disk available size filed!";
+        }
+    }
+    if (pre_allocated < available_bytes) {
+        st = wal_mgr->update_wal_dir_pre_allocated(wal_base_path, pre_allocated, true);
+        if (!st.ok()) {
+            LOG(WARNING) << "update wal dir pre_allocated failed, reason: " << st.to_string();
+        }
+        block_queue_pre_allocated.fetch_add(pre_allocated);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+Status GroupCommitMgr::update_load_info(TUniqueId load_id, size_t content_length) {
+    std::unique_lock l(_load_info_lock);
+    if (_load_id_to_content_length_map.find(load_id) == _load_id_to_content_length_map.end()) {
+        _load_id_to_content_length_map.insert(std::make_pair(load_id, content_length));
+    }
+    return Status::OK();
+}
+
+Status GroupCommitMgr::get_load_info(TUniqueId load_id, size_t* content_length) {
+    std::shared_lock l(_load_info_lock);
+    if (_load_id_to_content_length_map.find(load_id) != _load_id_to_content_length_map.end()) {
+        *content_length = _load_id_to_content_length_map[load_id];
+        return Status::OK();
+    }
+    return Status::InternalError("can not find load id!");
+}
+
+Status GroupCommitMgr::remove_load_info(TUniqueId load_id) {
+    std::unique_lock l(_load_info_lock);
+    if (_load_id_to_content_length_map.find(load_id) == _load_id_to_content_length_map.end()) {
+        return Status::InternalError("can not remove load id!");
+    }
+    _load_id_to_content_length_map.erase(load_id);
+    return Status::OK();
+}
 } // namespace doris
diff --git a/be/src/runtime/group_commit_mgr.h b/be/src/runtime/group_commit_mgr.h
index 6f625eda6aec04..35afcc46249b7a 100644
--- a/be/src/runtime/group_commit_mgr.h
+++ b/be/src/runtime/group_commit_mgr.h
@@ -20,9 +20,17 @@
 #include <gen_cpp/PaloInternalService_types.h>
 
 #include <atomic>
+#include <condition_variable>
+#include <cstdint>
 #include <memory>
+#include <mutex>
+#include <shared_mutex>
+#include <unordered_map>
+#include <utility>
 
 #include "common/status.h"
+#include "olap/wal_manager.h"
+#include "runtime/exec_env.h"
 #include "util/threadpool.h"
 #include "vec/core/block.h"
 #include "vec/sink/writer/vwal_writer.h"
@@ -45,9 +53,7 @@ class LoadBlockQueue {
               wait_internal_group_commit_finish(wait_internal_group_commit_finish),
               _start_time(std::chrono::steady_clock::now()),
               _all_block_queues_bytes(all_block_queues_bytes),
-              _group_commit_interval_ms(group_commit_interval_ms) {
-        _single_block_queue_bytes = std::make_shared<std::atomic_size_t>(0);
-    };
+              _group_commit_interval_ms(group_commit_interval_ms) {};
 
     Status add_block(std::shared_ptr<vectorized::Block> block, bool write_wal);
     Status get_block(RuntimeState* runtime_state, vectorized::Block* block, bool* find_block,
@@ -59,6 +65,8 @@ class LoadBlockQueue {
                       WalManager* wal_manager, std::vector<TSlotDescriptor>& slot_desc,
                       int be_exe_version);
     Status close_wal();
+    bool has_enough_wal_disk_space(const std::vector<std::shared_ptr<vectorized::Block>>& blocks,
+                                   const TUniqueId& load_id, bool is_blocks_contain_all_load_data);
 
     static constexpr size_t MAX_BLOCK_QUEUE_ADD_WAIT_TIME = 1000;
     UniqueId load_instance_id;
@@ -71,6 +79,8 @@ class LoadBlockQueue {
     bool process_finish = false;
     std::condition_variable internal_group_commit_finish_cv;
     Status status = Status::OK();
+    std::string wal_base_path;
+    std::atomic_size_t block_queue_pre_allocated = 0;
 
 private:
     void _cancel_without_lock(const Status& st);
@@ -84,8 +94,6 @@ class LoadBlockQueue {
 
     // memory consumption of all tables' load block queues, used for back pressure.
     std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
-    // memory consumption of one load block queue, used for correctness check.
-    std::shared_ptr<std::atomic_size_t> _single_block_queue_bytes;
     // group commit interval in ms, can be changed by 'ALTER TABLE my_table SET ("group_commit_interval_ms"="1000");'
     int64_t _group_commit_interval_ms;
     std::shared_ptr<vectorized::VWalWriter> _v_wal_writer;
@@ -145,6 +153,10 @@ class GroupCommitMgr {
                                       const UniqueId& load_id,
                                       std::shared_ptr<LoadBlockQueue>& load_block_queue,
                                       int be_exe_version);
+    Status update_load_info(TUniqueId load_id, size_t content_length);
+    Status get_load_info(TUniqueId load_id, size_t* content_length);
+    Status remove_load_info(TUniqueId load_id);
+    std::condition_variable cv;
 
 private:
     ExecEnv* _exec_env = nullptr;
@@ -155,6 +167,8 @@ class GroupCommitMgr {
     std::unique_ptr<doris::ThreadPool> _thread_pool;
     // memory consumption of all tables' load block queues, used for back pressure.
     std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
+    std::shared_mutex _load_info_lock;
+    std::unordered_map<TUniqueId, size_t> _load_id_to_content_length_map;
 };
 
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/sink/group_commit_block_sink.cpp b/be/src/vec/sink/group_commit_block_sink.cpp
index d4ede45868fe5d..3e38f9c42fc456 100644
--- a/be/src/vec/sink/group_commit_block_sink.cpp
+++ b/be/src/vec/sink/group_commit_block_sink.cpp
@@ -17,6 +17,14 @@
 
 #include "vec/sink/group_commit_block_sink.h"
 
+#include <gen_cpp/DataSinks_types.h>
+
+#include <chrono>
+#include <mutex>
+#include <shared_mutex>
+
+#include "common/exception.h"
+#include "runtime/exec_env.h"
 #include "runtime/group_commit_mgr.h"
 #include "runtime/runtime_state.h"
 #include "util/doris_metrics.h"
@@ -61,6 +69,8 @@ Status GroupCommitBlockSink::init(const TDataSink& t_sink) {
 
 Status GroupCommitBlockSink::prepare(RuntimeState* state) {
     RETURN_IF_ERROR(DataSink::prepare(state));
+    RETURN_IF_ERROR(
+            ExecEnv::GetInstance()->group_commit_mgr()->update_load_info(_load_id.to_thrift(), 0));
     _state = state;
 
     // profile must add to state's object pool
@@ -107,7 +117,7 @@ Status GroupCommitBlockSink::close(RuntimeState* state, Status close_status) {
             (double)state->num_rows_load_filtered() / num_selected_rows > _max_filter_ratio) {
             return Status::DataQualityError("too many filtered rows");
         }
-        RETURN_IF_ERROR(_add_blocks());
+        RETURN_IF_ERROR(_add_blocks(_group_commit_mode != TGroupCommitMode::SYNC_MODE, true));
     }
     if (_load_block_queue) {
         _load_block_queue->remove_load_id(_load_id);
@@ -213,7 +223,7 @@ Status GroupCommitBlockSink::_add_block(RuntimeState* state,
         _blocks.emplace_back(output_block);
     } else {
         if (!_is_block_appended) {
-            RETURN_IF_ERROR(_add_blocks());
+            RETURN_IF_ERROR(_add_blocks(_group_commit_mode != TGroupCommitMode::SYNC_MODE, false));
         }
         RETURN_IF_ERROR(_load_block_queue->add_block(
                 output_block, _group_commit_mode != TGroupCommitMode::SYNC_MODE));
@@ -221,7 +231,7 @@ Status GroupCommitBlockSink::_add_block(RuntimeState* state,
     return Status::OK();
 }
 
-Status GroupCommitBlockSink::_add_blocks() {
+Status GroupCommitBlockSink::_add_blocks(bool write_wal, bool is_blocks_contain_all_load_data) {
     DCHECK(_is_block_appended == false);
     TUniqueId load_id;
     load_id.__set_hi(_load_id.hi);
@@ -231,6 +241,20 @@ Status GroupCommitBlockSink::_add_blocks() {
             RETURN_IF_ERROR(_state->exec_env()->group_commit_mgr()->get_first_block_load_queue(
                     _db_id, _table_id, _base_schema_version, load_id, _load_block_queue,
                     _state->be_exec_version()));
+            if (write_wal) {
+                _group_commit_mode = _load_block_queue->has_enough_wal_disk_space(
+                                             _blocks, load_id, is_blocks_contain_all_load_data)
+                                             ? TGroupCommitMode::ASYNC_MODE
+                                             : TGroupCommitMode::SYNC_MODE;
+                if (_group_commit_mode == TGroupCommitMode::SYNC_MODE) {
+                    LOG(INFO)
+                            << "Load label " << _load_block_queue->label
+                            << " will not write wal because wal disk space usage reachs max limit.";
+                } else {
+                    LOG(INFO) << "Load label " << _load_block_queue->label << " will write wal to "
+                              << _load_block_queue->wal_base_path << ".";
+                }
+            }
             _state->set_import_label(_load_block_queue->label);
             _state->set_wal_id(_load_block_queue->txn_id);
         } else {
diff --git a/be/src/vec/sink/group_commit_block_sink.h b/be/src/vec/sink/group_commit_block_sink.h
index d7c8ed70d6dafc..c0971f4801a5ac 100644
--- a/be/src/vec/sink/group_commit_block_sink.h
+++ b/be/src/vec/sink/group_commit_block_sink.h
@@ -47,7 +47,7 @@ class GroupCommitBlockSink : public DataSink {
 
 private:
     Status _add_block(RuntimeState* state, std::shared_ptr<vectorized::Block> block);
-    Status _add_blocks();
+    Status _add_blocks(bool write_wal, bool is_blocks_contain_all_load_data);
 
     vectorized::VExprContextSPtrs _output_vexpr_ctxs;
 
diff --git a/be/src/vec/sink/writer/vwal_writer.cpp b/be/src/vec/sink/writer/vwal_writer.cpp
index d929207e9a9b3d..2dc945a2a2f5b9 100644
--- a/be/src/vec/sink/writer/vwal_writer.cpp
+++ b/be/src/vec/sink/writer/vwal_writer.cpp
@@ -42,11 +42,10 @@
 namespace doris {
 namespace vectorized {
 
-VWalWriter::VWalWriter(int64_t db_id, int64_t tb_id, int64_t wal_id,
-                       const std::string& import_label, WalManager* wal_manager,
-                       std::vector<TSlotDescriptor>& slot_desc, int be_exe_version)
-        : _db_id(db_id),
-          _tb_id(tb_id),
+VWalWriter::VWalWriter(int64_t tb_id, int64_t wal_id, const std::string& import_label,
+                       WalManager* wal_manager, std::vector<TSlotDescriptor>& slot_desc,
+                       int be_exe_version)
+        : _tb_id(tb_id),
           _wal_id(wal_id),
           _label(import_label),
           _wal_manager(wal_manager),
@@ -56,7 +55,6 @@ VWalWriter::VWalWriter(int64_t db_id, int64_t tb_id, int64_t wal_id,
 VWalWriter::~VWalWriter() {}
 
 Status VWalWriter::init() {
-    RETURN_IF_ERROR(_wal_manager->add_wal_path(_db_id, _tb_id, _wal_id, _label));
     RETURN_IF_ERROR(_wal_manager->create_wal_writer(_wal_id, _wal_writer));
     _wal_manager->add_wal_status_queue(_tb_id, _wal_id, WalManager::WAL_STATUS::CREATE);
     std::stringstream ss;
diff --git a/be/src/vec/sink/writer/vwal_writer.h b/be/src/vec/sink/writer/vwal_writer.h
index 17c9dc979a1c47..324409e9d469a1 100644
--- a/be/src/vec/sink/writer/vwal_writer.h
+++ b/be/src/vec/sink/writer/vwal_writer.h
@@ -83,7 +83,7 @@ namespace vectorized {
 
 class VWalWriter {
 public:
-    VWalWriter(int64_t db_id, int64_t tb_id, int64_t wal_id, const std::string& import_label,
+    VWalWriter(int64_t tb_id, int64_t wal_id, const std::string& import_label,
                WalManager* wal_manager, std::vector<TSlotDescriptor>& slot_desc,
                int be_exe_version);
     ~VWalWriter();
@@ -92,7 +92,6 @@ class VWalWriter {
     Status close();
 
 private:
-    int64_t _db_id;
     int64_t _tb_id;
     int64_t _wal_id;
     uint32_t _version = 0;
diff --git a/be/test/http/stream_load_test.cpp b/be/test/http/stream_load_test.cpp
index 75588f4190b1ef..74dcf9f1f3ff67 100644
--- a/be/test/http/stream_load_test.cpp
+++ b/be/test/http/stream_load_test.cpp
@@ -19,6 +19,8 @@
 
 #include <gtest/gtest.h>
 
+#include <memory>
+
 #include "common/config.h"
 #include "event2/buffer.h"
 #include "event2/event.h"
@@ -34,6 +36,7 @@
 #include "http/http_request.h"
 #include "http/utils.h"
 #include "olap/wal_manager.h"
+#include "runtime/exec_env.h"
 
 namespace doris {
 
@@ -51,10 +54,14 @@ void http_request_done_cb(struct evhttp_request* req, void* arg) {
 
 TEST_F(StreamLoadTest, TestHeader) {
     // 1G
-    config::wal_max_disk_size = 1073741824;
+    auto wal_mgr = WalManager::create_shared(ExecEnv::GetInstance(), config::group_commit_wal_path);
+    static_cast<void>(wal_mgr->_wal_dirs_info->add("test_path_1", 1000, 0, 0));
+    static_cast<void>(wal_mgr->_wal_dirs_info->add("test_path_2", 10000, 0, 0));
+    static_cast<void>(wal_mgr->_wal_dirs_info->add("test_path_3", 100000, 0, 0));
+    ExecEnv::GetInstance()->set_wal_mgr(wal_mgr);
     // 1. empty info
     {
-        auto evhttp_req = evhttp_request_new(nullptr, nullptr);
+        auto* evhttp_req = evhttp_request_new(nullptr, nullptr);
         HttpRequest req(evhttp_req);
         EXPECT_EQ(load_size_smaller_than_wal_limit(&req), false);
         evhttp_request_free(evhttp_req);
@@ -86,7 +93,7 @@ TEST_F(StreamLoadTest, TestHeader) {
         evhttp_req->uri = uri;
         evhttp_req->uri_elems = evhttp_uri_parse(uri);
         evhttp_add_header(evhttp_req->input_headers, HTTP_GROUP_COMMIT.c_str(), "true");
-        evhttp_add_header(evhttp_req->input_headers, HttpHeaders::CONTENT_LENGTH, "1000");
+        evhttp_add_header(evhttp_req->input_headers, HttpHeaders::CONTENT_LENGTH, "20000");
         HttpRequest req(evhttp_req);
         req.init_from_evhttp();
         EXPECT_EQ(load_size_smaller_than_wal_limit(&req), true);
diff --git a/be/test/olap/wal_manager_test.cpp b/be/test/olap/wal_manager_test.cpp
index 17fb3e3e6f43c7..93d8636eb225ec 100644
--- a/be/test/olap/wal_manager_test.cpp
+++ b/be/test/olap/wal_manager_test.cpp
@@ -18,6 +18,7 @@
 
 #include <gtest/gtest.h>
 
+#include <cstddef>
 #include <filesystem>
 #include <map>
 #include <string>
@@ -27,6 +28,7 @@
 #include "gen_cpp/HeartbeatService_types.h"
 #include "gen_cpp/internal_service.pb.h"
 #include "io/fs/local_file_system.h"
+#include "olap/options.h"
 #include "runtime/decimalv2_value.h"
 #include "runtime/exec_env.h"
 #include "runtime/result_queue_mgr.h"
@@ -46,8 +48,7 @@ extern Status k_stream_load_plan_status;
 extern std::string k_request_line;
 
 ExecEnv* _env = nullptr;
-std::string wal_dir = "./wal_test";
-std::string tmp_dir = "./wal_test/tmp";
+std::string wal_dir = std::string(getenv("DORIS_HOME")) + "/wal_test";
 
 class WalManagerTest : public testing::Test {
 public:
@@ -63,6 +64,7 @@ class WalManagerTest : public testing::Test {
         _env->_internal_client_cache = new BrpcClientCache<PBackendService_Stub>();
         _env->_function_client_cache = new BrpcClientCache<PFunctionService_Stub>();
         _env->_stream_load_executor = StreamLoadExecutor::create_shared(_env);
+        _env->_store_paths = {StorePath(std::filesystem::current_path(), 0)};
         _env->_wal_manager = WalManager::create_shared(_env, wal_dir);
         k_stream_load_begin_result = TLoadTxnBeginResult();
         k_stream_load_plan_status = Status::OK();
@@ -78,16 +80,14 @@ class WalManagerTest : public testing::Test {
     void prepare() { static_cast<void>(io::global_local_filesystem()->create_directory(wal_dir)); }
 
     void createWal(const std::string& wal_path) {
-        std::shared_ptr<std::atomic_size_t> _all_wal_disk_bytes =
-                std::make_shared<std::atomic_size_t>(0);
-        std::shared_ptr<std::condition_variable> cv = std::make_shared<std::condition_variable>();
-        auto wal_writer = WalWriter(wal_path, _all_wal_disk_bytes, cv);
+        auto wal_writer = WalWriter(wal_path);
         static_cast<void>(wal_writer.init());
         static_cast<void>(wal_writer.finalize());
     }
 };
 
 TEST_F(WalManagerTest, recovery_normal) {
+    _env->wal_mgr()->wal_limit_test_bytes = 1099511627776;
     k_request_line = "{\"Status\": \"Success\",    \"Message\": \"Test\"}";
 
     std::string db_id = "1";
@@ -123,4 +123,93 @@ TEST_F(WalManagerTest, recovery_normal) {
     ASSERT_TRUE(!std::filesystem::exists(wal_200));
     ASSERT_TRUE(!std::filesystem::exists(wal_201));
 }
+
+TEST_F(WalManagerTest, TestDynamicWalSpaceLimt) {
+    auto wal_mgr = WalManager::create_shared(_env, config::group_commit_wal_path);
+    static_cast<void>(wal_mgr->init());
+    _env->set_wal_mgr(wal_mgr);
+
+    // 1T
+    size_t available_bytes = 1099511627776;
+    size_t wal_limit_bytes;
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "0%";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 0);
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "5%";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    wal_limit_bytes = available_bytes * 0.05;
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "50%";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    wal_limit_bytes = available_bytes * 0.5;
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "200%";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    wal_limit_bytes = available_bytes * 2;
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, wal_limit_bytes);
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "-10%";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::InternalError(""));
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, available_bytes);
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "0";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 0);
+
+    // 1M
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "1048576";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1048576);
+
+    // 1G
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "1073741824";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1073741824);
+
+    // 100G
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "107374182400";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 107374182400);
+
+    // 1M
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "1M";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1048576);
+
+    // 1G
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "1G";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 1073741824);
+
+    // 100G
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "100G";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::OK());
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, 107374182400);
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "-1024";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::InternalError(""));
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, available_bytes);
+
+    _env->wal_mgr()->wal_limit_test_bytes = available_bytes;
+    config::group_commit_wal_max_disk_limit = "-1M";
+    EXPECT_EQ(_env->wal_mgr()->_init_wal_dirs_info(), Status::InternalError(""));
+    EXPECT_EQ(_env->wal_mgr()->wal_limit_test_bytes, available_bytes);
+}
 } // namespace doris
\ No newline at end of file
diff --git a/be/test/olap/wal_reader_writer_test.cpp b/be/test/olap/wal_reader_writer_test.cpp
index 1d1102f350fa09..d24db7286802c7 100644
--- a/be/test/olap/wal_reader_writer_test.cpp
+++ b/be/test/olap/wal_reader_writer_test.cpp
@@ -90,10 +90,7 @@ void generate_block(PBlock& pblock, int row_index) {
 
 TEST_F(WalReaderWriterTest, TestWriteAndRead1) {
     std::string file_name = _s_test_data_path + "/abcd123.txt";
-    std::shared_ptr<std::atomic_size_t> _all_wal_disk_bytes =
-            std::make_shared<std::atomic_size_t>(0);
-    std::shared_ptr<std::condition_variable> cv = std::make_shared<std::condition_variable>();
-    auto wal_writer = WalWriter(file_name, _all_wal_disk_bytes, cv);
+    auto wal_writer = WalWriter(file_name);
     static_cast<void>(wal_writer.init());
     size_t file_len = 0;
     int64_t file_size = -1;
diff --git a/be/test/vec/exec/vtablet_sink_test.cpp b/be/test/vec/exec/vtablet_sink_test.cpp
index 81c2bdc4b072e3..f0b8e8c96433ef 100644
--- a/be/test/vec/exec/vtablet_sink_test.cpp
+++ b/be/test/vec/exec/vtablet_sink_test.cpp
@@ -496,7 +496,7 @@ class VOlapTableSinkTest : public testing::Test {
 private:
     ExecEnv* _env = nullptr;
     brpc::Server* _server = nullptr;
-    std::string wal_dir = "./wal_test";
+    std::string wal_dir = std::string(getenv("DORIS_HOME")) + "/wal_test";
 };
 
 TEST_F(VOlapTableSinkTest, normal) {
diff --git a/be/test/vec/exec/vwal_scanner_test.cpp b/be/test/vec/exec/vwal_scanner_test.cpp
index a239741a5cb341..ac066aceb98955 100644
--- a/be/test/vec/exec/vwal_scanner_test.cpp
+++ b/be/test/vec/exec/vwal_scanner_test.cpp
@@ -57,7 +57,7 @@ class VWalScannerTest : public testing::Test {
     void init_desc_table();
 
     ExecEnv* _env = nullptr;
-    std::string wal_dir = "./wal_test";
+    std::string wal_dir = std::string(getenv("DORIS_HOME")) + "/wal_test";
     int64_t db_id = 1;
     int64_t tb_id = 2;
     int64_t txn_id = 789;
@@ -195,6 +195,7 @@ void VWalScannerTest::init_desc_table() {
 }
 
 void VWalScannerTest::init() {
+    config::group_commit_wal_max_disk_limit = "100M";
     init_desc_table();
     static_cast<void>(io::global_local_filesystem()->create_directory(
             wal_dir + "/" + std::to_string(db_id) + "/" + std::to_string(tb_id)));
@@ -215,7 +216,9 @@ void VWalScannerTest::init() {
 
     _env = ExecEnv::GetInstance();
     _env->_wal_manager = WalManager::create_shared(_env, wal_dir);
-    auto st = _env->_wal_manager->add_wal_path(db_id, tb_id, txn_id, label);
+    std::string base_path;
+    auto st = _env->_wal_manager->_init_wal_dirs_info();
+    st = _env->_wal_manager->add_wal_path(db_id, tb_id, txn_id, label, base_path);
 }
 
 TEST_F(VWalScannerTest, normal) {
diff --git a/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy b/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy
index f5cb97fbae8aef..9a081aa2d59792 100644
--- a/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy
@@ -70,7 +70,7 @@ suite("test_group_commit_and_wal_back_pressure", "p2") {
                 unset 'label'
 
                 file 'test_group_commit_and_wal_back_pressure.csv.gz'
-                time 100000 
+                time 600000 
             }
         })
     }
@@ -88,7 +88,7 @@ suite("test_group_commit_and_wal_back_pressure", "p2") {
                 unset 'label'
 
                 file 'test_group_commit_and_wal_back_pressure.csv.gz'
-                time 100000 
+                time 600000 
             }
         })
     }
@@ -106,7 +106,7 @@ suite("test_group_commit_and_wal_back_pressure", "p2") {
                 unset 'label'
 
                 file 'test_group_commit_and_wal_back_pressure.csv.gz'
-                time 100000 
+                time 600000 
             }
         })
     }
diff --git a/regression-test/suites/load_p0/stream_load/test_group_commit_wal_limit.groovy b/regression-test/suites/load_p0/stream_load/test_group_commit_wal_limit.groovy
index 24b66de04b86c6..7361faf72df16a 100644
--- a/regression-test/suites/load_p0/stream_load/test_group_commit_wal_limit.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_group_commit_wal_limit.groovy
@@ -41,7 +41,7 @@ suite("test_group_commit_wal_limit") {
     // normal case
     StringBuilder strBuilder = new StringBuilder()
     strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
-    strBuilder.append(" -H \"group_commit:true\" -H \"column_separator:,\" " )
+    strBuilder.append(" -H \"group_commit:async_mode\" -H \"column_separator:,\" " )
     strBuilder.append(" -H \"compress_type:gz\" -H \"format:csv\" " )
     strBuilder.append(" -T " + context.config.dataPath + "/load_p0/stream_load/test_group_commit_wal_limit.csv.gz")
     strBuilder.append(" http://" + context.config.feHttpAddress + "/api/${db}/${tableName}/_stream_load")
@@ -55,49 +55,13 @@ suite("test_group_commit_wal_limit") {
     logger.info("out is " + out )
     assertTrue(out.contains('group_commit'))
 
-    // chunked data case
-    strBuilder = new StringBuilder()
-    strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
-    strBuilder.append(" -H \"group_commit:true\" -H \"column_separator:,\" " )
-    strBuilder.append(" -H \"compress_type:gz\" -H \"format:csv\" " )
-    strBuilder.append(" -H \"Content-Length:0\" " )
-    strBuilder.append(" -T " + context.config.dataPath + "/load_p0/stream_load/test_group_commit_wal_limit.csv.gz")
-    strBuilder.append(" http://" + context.config.feHttpAddress + "/api/${db}/${tableName}/_stream_load")
-
-    command = strBuilder.toString()
-    logger.info("command is " + command)
-    process = ['bash','-c',command].execute() 
-    code = process.waitFor()
-    assertEquals(code, 0)
-    out = process.text
-    logger.info("out is " + out )
-    assertTrue(out.contains('Stream load size too large'))
-
-    // too lagre data case 1TB
-    strBuilder = new StringBuilder()
-    strBuilder.append("curl --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
-    strBuilder.append(" -H \"group_commit:true\" -H \"column_separator:,\" " )
-    strBuilder.append(" -H \"compress_type:gz\" -H \"format:csv\" " )
-    strBuilder.append(" -H \"Content-Length:1099511627776\" " )
-    strBuilder.append(" -T " + context.config.dataPath + "/load_p0/stream_load/test_group_commit_wal_limit.csv.gz")
-    strBuilder.append(" http://" + context.config.feHttpAddress + "/api/${db}/${tableName}/_stream_load")
-
-    command = strBuilder.toString()
-    logger.info("command is " + command)
-    process = ['bash','-c',command].execute() 
-    code = process.waitFor()
-    assertEquals(code, 0)
-    out = process.text
-    logger.info("out is " + out )
-    assertTrue(out.contains('Stream load size too large'))
-
     // httpload 
     // normal case
     strBuilder = new StringBuilder()
     strBuilder.append("curl -v --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
     String sql = " -H \"sql:insert into " + db + "." + tableName + " (k,v) select c1, c2 from http_stream(\\\"format\\\" = \\\"csv\\\", \\\"column_separator\\\" = \\\",\\\", \\\"compress_type\\\" = \\\"gz\\\" ) \" "
     strBuilder.append(sql)
-    strBuilder.append(" -H \"group_commit:true\"") 
+    strBuilder.append(" -H \"group_commit:async_mode\"") 
     strBuilder.append(" -T " + context.config.dataPath + "/load_p0/stream_load/test_group_commit_wal_limit.csv.gz")
     strBuilder.append(" http://" + context.config.feHttpAddress + "/api/_http_stream")
 
@@ -109,40 +73,4 @@ suite("test_group_commit_wal_limit") {
     out = process.text
     logger.info("out is " + out )
     assertTrue(out.contains('group_commit'))
-
-    // chunked data case
-    strBuilder = new StringBuilder()
-    strBuilder.append("curl -v --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
-    sql = " -H \"sql:insert into " + db + "." + tableName + " (k,v) select c1, c2 from http_stream(\\\"format\\\" = \\\"csv\\\", \\\"column_separator\\\" = \\\",\\\", \\\"compress_type\\\" = \\\"gz\\\" ) \" "
-    strBuilder.append(sql)
-    strBuilder.append(" -H \"group_commit:true\" -H \"Content-Length:0\"") 
-    strBuilder.append(" -T " + context.config.dataPath + "/load_p0/stream_load/test_group_commit_wal_limit.csv.gz")
-    strBuilder.append(" http://" + context.config.feHttpAddress + "/api/_http_stream")
-
-    command = strBuilder.toString()
-    logger.info("command is " + command)
-    process = ['bash','-c',command].execute() 
-    code = process.waitFor()
-    assertEquals(code, 0)
-    out = process.text
-    logger.info("out is " + out )
-    assertTrue(out.contains('Http load size too large'))
-
-    // too lagre data case 1TB
-    strBuilder = new StringBuilder()
-    strBuilder.append("curl -v --location-trusted -u " + context.config.jdbcUser + ":" + context.config.jdbcPassword)
-    sql = " -H \"sql:insert into " + db + "." + tableName + " (k,v) select c1, c2 from http_stream(\\\"format\\\" = \\\"csv\\\", \\\"column_separator\\\" = \\\",\\\", \\\"compress_type\\\" = \\\"gz\\\" ) \" "
-    strBuilder.append(sql)
-    strBuilder.append(" -H \"group_commit:true\" -H \"Content-Length:1099511627776\"") 
-    strBuilder.append(" -T " + context.config.dataPath + "/load_p0/stream_load/test_group_commit_wal_limit.csv.gz")
-    strBuilder.append(" http://" + context.config.feHttpAddress + "/api/_http_stream")
-
-    command = strBuilder.toString()
-    logger.info("command is " + command)
-    process = ['bash','-c',command].execute() 
-    code = process.waitFor()
-    assertEquals(code, 0)
-    out = process.text
-    logger.info("out is " + out )
-    assertTrue(out.contains('Http load size too large'))
 }

From 86a0eb934455d68ad61ae958e13c8ef7fe74cb70 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 11:19:01 +0700
Subject: [PATCH 055/299] [fix](doc) spell errors fixes query profile docs
 (#28437)

---
 docs/en/docs/admin-manual/query-profile.md    | 8 ++++----
 docs/zh-CN/docs/admin-manual/query-profile.md | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/en/docs/admin-manual/query-profile.md b/docs/en/docs/admin-manual/query-profile.md
index 9bfd2610239ee8..f46061419c669b 100644
--- a/docs/en/docs/admin-manual/query-profile.md
+++ b/docs/en/docs/admin-manual/query-profile.md
@@ -26,7 +26,7 @@ under the License.
 
 # Statistics of query execution
 
-This document focuses on introducing the **Running Profile** which recorded runtime status of Doris in query execution. Using these statistical information, we can understand the execution of frgment to become a expert of Doris's **debugging and tuning**.
+This document focuses on introducing the **Running Profile** which recorded runtime status of Doris in query execution. Using these statistical information, we can understand the execution of fragment to become a expert of Doris's **debugging and tuning**.
 
 You can also refer to following statements to view profile in command line:
 
@@ -109,7 +109,7 @@ There are many statistical information collected at BE.  so we list the correspo
 #### `BlockMgr`
   - BlocksCreated: Number of Block be created by BlockMgr
   - BlocksRecycled: Number of Block be recycled by BlockMgr
-  - BytesWritten: How many bytes be writen to spill to disk
+  - BytesWritten: How many bytes be written to spill to disk
   - MaxBlockSize: Max size of one Block
   - TotalReadBlockTime: Total time read block from disk
 
@@ -154,7 +154,7 @@ There are many statistical information collected at BE.  so we list the correspo
   - HTResizeTime: Time spent in resizing hashtable
   - HTResize: Number of times hashtable resizes
   - HashBuckets: Number of buckets in hashtable
-  - HashBucketsWithDuplicate: Number of buckets with duplicatenode in hashtable
+  - HashBucketsWithDuplicate: Number of buckets with duplicate node in hashtable
   - HashCollisions: Number of hash conflicts generated 
   - HashDuplicateNodes: Number of duplicate nodes with the same buckets in hashtable
   - HashFailedProbe: Number of failed probe operations
@@ -210,7 +210,7 @@ OLAP_SCAN_NODE (id=0):(Active: 1.2ms,% non-child: 0.00%)
   - RowsReturnedRate: 6.979K /sec       # RowsReturned/ActiveTime
   - TabletCount: 20                     # The number of Tablets involved in this ScanNode.
   - TotalReadThroughput: 74.70 KB/sec   # BytesRead divided by the total time spent in this node (from Open to Close). For IO bounded queries, this should be very close to the total throughput of all the disks
-  - ScannerBatchWaitTime: 426.886us     # To count the time the transfer thread waits for the scaner thread to return rowbatch. In pipeline, this value is always 0.
+  - ScannerBatchWaitTime: 426.886us     # To count the time the transfer thread waits for the scanner thread to return rowbatch. In pipeline, this value is always 0.
   - ScannerWorkerWaitTime: 17.745us     # To count the time that the scanner thread waits for the available worker threads in the thread pool.
   OlapScanner:
     - BlockConvertTime: 8.941us         # The time it takes to convert a vectorized Block into a RowBlock with a row structure. The vectorized Block is VectorizedRowBatch in V1 and RowBlockV2 in V2.
diff --git a/docs/zh-CN/docs/admin-manual/query-profile.md b/docs/zh-CN/docs/admin-manual/query-profile.md
index 9960bc18a9dcf7..7c86a736ef10c1 100644
--- a/docs/zh-CN/docs/admin-manual/query-profile.md
+++ b/docs/zh-CN/docs/admin-manual/query-profile.md
@@ -209,7 +209,7 @@ OLAP_SCAN_NODE (id=0):(Active: 1.2ms, % non-child: 0.00%)
   - RowsReturnedRate: 6.979K /sec       # RowsReturned/ActiveTime
   - TabletCount : 20                    # 该 ScanNode 涉及的 Tablet 数量。
   - TotalReadThroughput: 74.70 KB/sec   # BytesRead除以该节点运行的总时间（从Open到Close），对于IO受限的查询，接近磁盘的总吞吐量。
-  - ScannerBatchWaitTime: 426.886us     # 用于统计transfer 线程等待scaner 线程返回rowbatch的时间。在Pipeline调度中，此值无意义。
+  - ScannerBatchWaitTime: 426.886us     # 用于统计transfer 线程等待scanner 线程返回rowbatch的时间。在Pipeline调度中，此值无意义。
   - ScannerWorkerWaitTime: 17.745us     # 用于统计scanner thread 等待线程池中可用工作线程的时间。
   OlapScanner:
     - BlockConvertTime: 8.941us         # 将向量化Block转换为行结构的 RowBlock 的耗时。向量化 Block 在 V1 中为 VectorizedRowBatch，V2中为 RowBlockV2。

From 17f92155d7899b0b5a08f5d388a4440d7e1fd891 Mon Sep 17 00:00:00 2001
From: seawinde <149132972+seawinde@users.noreply.github.com>
Date: Wed, 27 Dec 2023 12:21:25 +0800
Subject: [PATCH 056/299] [opt](Nerieds) add infer props to expression (#28953)

---
 .../rules/rewrite/PredicatePropagation.java   |   5 +-
 .../trees/expressions/BinaryOperator.java     |   6 +-
 .../expressions/ComparisonPredicate.java      |   6 +-
 .../trees/expressions/EqualPredicate.java     |   6 +-
 .../nereids/trees/expressions/EqualTo.java    |  19 ++-
 .../nereids/trees/expressions/Expression.java |  24 ++++
 .../trees/expressions/GreaterThan.java        |  19 ++-
 .../trees/expressions/GreaterThanEqual.java   |  19 ++-
 .../nereids/trees/expressions/LessThan.java   |  19 ++-
 .../trees/expressions/LessThanEqual.java      |  19 ++-
 .../trees/expressions/NullSafeEqual.java      |  19 ++-
 .../doris/nereids/util/ExpressionUtils.java   |  22 ++++
 .../rules/rewrite/InferPredicatesTest.java    | 113 ++++++++++++------
 13 files changed, 237 insertions(+), 59 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PredicatePropagation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PredicatePropagation.java
index ecb1c5499bde77..72e9023dc45df5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PredicatePropagation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PredicatePropagation.java
@@ -88,6 +88,9 @@ public ComparisonInferInfo(InferType inferType,
     public Set<Expression> infer(Set<Expression> predicates) {
         Set<Expression> inferred = Sets.newHashSet();
         for (Expression predicate : predicates) {
+            // if we support more infer predicate expression type, we should impl withInferred() method.
+            // And should add inferred props in withChildren() method just like ComparisonPredicate,
+            // and it's subclass, to mark the predicate is from infer.
             if (!(predicate instanceof ComparisonPredicate)) {
                 continue;
             }
@@ -130,7 +133,7 @@ private Expression doInfer(ComparisonInferInfo equalInfo, ComparisonInferInfo pr
                 .comparisonPredicate.withChildren(newLeft, newRight);
         Expression expr = SimplifyComparisonPredicate.INSTANCE
                 .rewrite(TypeCoercionUtils.processComparisonPredicate(newPredicate), null);
-        return DateFunctionRewrite.INSTANCE.rewrite(expr, null);
+        return DateFunctionRewrite.INSTANCE.rewrite(expr, null).withInferred(true);
     }
 
     private Expression inferOneSide(Expression predicateOneSide, Expression equalLeft, Expression equalRight) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java
index c3006fa927b154..2d06456d0a46fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/BinaryOperator.java
@@ -34,7 +34,11 @@ public abstract class BinaryOperator extends Expression implements BinaryExpress
     protected final String symbol;
 
     public BinaryOperator(List<Expression> children, String symbol) {
-        super(children);
+        this(children, symbol, false);
+    }
+
+    public BinaryOperator(List<Expression> children, String symbol, boolean inferred) {
+        super(children, inferred);
         this.symbol = symbol;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java
index bba834967f03c1..a22852ca31c09d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java
@@ -33,7 +33,11 @@
 public abstract class ComparisonPredicate extends BinaryOperator {
 
     public ComparisonPredicate(List<Expression> children, String symbol) {
-        super(children, symbol);
+        this(children, symbol, false);
+    }
+
+    public ComparisonPredicate(List<Expression> children, String symbol, boolean inferred) {
+        super(children, symbol, inferred);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualPredicate.java
index 3f61bd3cf621a5..5691f7de94c2a2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualPredicate.java
@@ -25,7 +25,11 @@
 public abstract class EqualPredicate extends ComparisonPredicate {
 
     protected EqualPredicate(List<Expression> children, String symbol) {
-        super(children, symbol);
+        this(children, symbol, false);
+    }
+
+    protected EqualPredicate(List<Expression> children, String symbol, boolean inferred) {
+        super(children, symbol, inferred);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java
index 3faccff6d99651..2704d446555867 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java
@@ -32,11 +32,19 @@
 public class EqualTo extends EqualPredicate implements PropagateNullable {
 
     public EqualTo(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), "=");
+        this(left, right, false);
+    }
+
+    public EqualTo(Expression left, Expression right, boolean inferred) {
+        super(ImmutableList.of(left, right), "=", inferred);
     }
 
     private EqualTo(List<Expression> children) {
-        super(children, "=");
+        this(children, false);
+    }
+
+    private EqualTo(List<Expression> children, boolean inferred) {
+        super(children, "=", inferred);
     }
 
     @Override
@@ -47,7 +55,12 @@ public boolean nullable() throws UnboundException {
     @Override
     public EqualTo withChildren(List<Expression> children) {
         Preconditions.checkArgument(children.size() == 2);
-        return new EqualTo(children);
+        return new EqualTo(children, this.isInferred());
+    }
+
+    @Override
+    public Expression withInferred(boolean inferred) {
+        return new EqualTo(this.children, inferred);
     }
 
     public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
index bdd776ffe97cc5..1ce12fc0efac03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -59,6 +59,8 @@ public abstract class Expression extends AbstractTreeNode<Expression> implements
     protected Optional<String> exprName = Optional.empty();
     private final int depth;
     private final int width;
+    // Mark this expression is from predicate infer or something else infer
+    private final boolean inferred;
 
     protected Expression(Expression... children) {
         super(children);
@@ -69,6 +71,7 @@ protected Expression(Expression... children) {
                 .mapToInt(e -> e.width)
                 .sum() + (children.length == 0 ? 1 : 0);
         checkLimit();
+        this.inferred = false;
     }
 
     protected Expression(List<Expression> children) {
@@ -80,6 +83,19 @@ protected Expression(List<Expression> children) {
                 .mapToInt(e -> e.width)
                 .sum() + (children.isEmpty() ? 1 : 0);
         checkLimit();
+        this.inferred = false;
+    }
+
+    protected Expression(List<Expression> children, boolean inferred) {
+        super(children);
+        depth = children.stream()
+                .mapToInt(e -> e.depth)
+                .max().orElse(0) + 1;
+        width = children.stream()
+                .mapToInt(e -> e.width)
+                .sum() + (children.isEmpty() ? 1 : 0);
+        checkLimit();
+        this.inferred = inferred;
     }
 
     private void checkLimit() {
@@ -216,11 +232,19 @@ public int getDepth() {
         return depth;
     }
 
+    public boolean isInferred() {
+        return inferred;
+    }
+
     @Override
     public Expression withChildren(List<Expression> children) {
         throw new RuntimeException();
     }
 
+    public Expression withInferred(boolean inferred) {
+        throw new RuntimeException("current expression has not impl the withInferred method");
+    }
+
     /**
      * Whether the expression is a constant.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java
index 1871781ca9cd10..ec4606eda5539b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java
@@ -37,11 +37,19 @@ public class GreaterThan extends ComparisonPredicate implements PropagateNullabl
      * @param right right child of greater than
      */
     public GreaterThan(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), ">");
+        this(left, right, false);
+    }
+
+    public GreaterThan(Expression left, Expression right, boolean inferred) {
+        super(ImmutableList.of(left, right), ">", inferred);
     }
 
     private GreaterThan(List<Expression> children) {
-        super(children, ">");
+        this(children, false);
+    }
+
+    private GreaterThan(List<Expression> children, boolean inferred) {
+        super(children, ">", inferred);
     }
 
     @Override
@@ -57,7 +65,12 @@ public String toString() {
     @Override
     public GreaterThan withChildren(List<Expression> children) {
         Preconditions.checkArgument(children.size() == 2);
-        return new GreaterThan(children);
+        return new GreaterThan(children, this.isInferred());
+    }
+
+    @Override
+    public Expression withInferred(boolean inferred) {
+        return new GreaterThan(this.children, inferred);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java
index e2995298e3734b..693097f1825c8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java
@@ -32,11 +32,19 @@
 public class GreaterThanEqual extends ComparisonPredicate implements PropagateNullable {
 
     public GreaterThanEqual(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), ">=");
+        this(left, right, false);
+    }
+
+    public GreaterThanEqual(Expression left, Expression right, boolean inferred) {
+        super(ImmutableList.of(left, right), ">=", inferred);
     }
 
     private GreaterThanEqual(List<Expression> children) {
-        super(children, ">=");
+        this(children, false);
+    }
+
+    private GreaterThanEqual(List<Expression> children, boolean inferred) {
+        super(children, ">=", inferred);
     }
 
     @Override
@@ -52,7 +60,12 @@ public String toString() {
     @Override
     public GreaterThanEqual withChildren(List<Expression> children) {
         Preconditions.checkArgument(children.size() == 2);
-        return new GreaterThanEqual(children);
+        return new GreaterThanEqual(children, this.isInferred());
+    }
+
+    @Override
+    public Expression withInferred(boolean inferred) {
+        return new GreaterThanEqual(this.children, inferred);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java
index 4d34b50bb9c7b8..e1ba9f69aad731 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java
@@ -31,11 +31,19 @@
  */
 public class LessThan extends ComparisonPredicate implements PropagateNullable {
     public LessThan(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), "<");
+        this(left, right, false);
+    }
+
+    public LessThan(Expression left, Expression right, boolean inferred) {
+        super(ImmutableList.of(left, right), "<", inferred);
     }
 
     private LessThan(List<Expression> children) {
-        super(children, "<");
+        this(children, false);
+    }
+
+    private LessThan(List<Expression> children, boolean inferred) {
+        super(children, "<", inferred);
     }
 
     @Override
@@ -51,7 +59,12 @@ public String toString() {
     @Override
     public LessThan withChildren(List<Expression> children) {
         Preconditions.checkArgument(children.size() == 2);
-        return new LessThan(children);
+        return new LessThan(children, this.isInferred());
+    }
+
+    @Override
+    public Expression withInferred(boolean inferred) {
+        return new LessThan(this.children, inferred);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java
index 4ac997d3ab7b4e..cc8c64ff106b18 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java
@@ -37,11 +37,19 @@ public class LessThanEqual extends ComparisonPredicate implements PropagateNulla
      * @param right right child of Less Than And Equal
      */
     public LessThanEqual(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), "<=");
+        this(left, right, false);
+    }
+
+    public LessThanEqual(Expression left, Expression right, boolean inferred) {
+        super(ImmutableList.of(left, right), "<=", inferred);
     }
 
     private LessThanEqual(List<Expression> children) {
-        super(children, "<=");
+        this(children, false);
+    }
+
+    private LessThanEqual(List<Expression> children, boolean inferred) {
+        super(children, "<=", inferred);
     }
 
     @Override
@@ -57,7 +65,12 @@ public String toString() {
     @Override
     public LessThanEqual withChildren(List<Expression> children) {
         Preconditions.checkArgument(children.size() == 2);
-        return new LessThanEqual(children);
+        return new LessThanEqual(children, this.isInferred());
+    }
+
+    @Override
+    public Expression withInferred(boolean inferred) {
+        return new LessThanEqual(this.children, inferred);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
index 48d05364fa3441..6eda6c738b761c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
@@ -31,11 +31,19 @@
  */
 public class NullSafeEqual extends EqualPredicate implements AlwaysNotNullable {
     public NullSafeEqual(Expression left, Expression right) {
-        super(ImmutableList.of(left, right), "<=>");
+        this(left, right, false);
+    }
+
+    public NullSafeEqual(Expression left, Expression right, boolean inferred) {
+        super(ImmutableList.of(left, right), "<=>", inferred);
     }
 
     private NullSafeEqual(List<Expression> children) {
-        super(children, "<=>");
+        this(children, false);
+    }
+
+    private NullSafeEqual(List<Expression> children, boolean inferred) {
+        super(children, "<=>", inferred);
     }
 
     @Override
@@ -51,7 +59,12 @@ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
     @Override
     public NullSafeEqual withChildren(List<Expression> children) {
         Preconditions.checkArgument(children.size() == 2);
-        return new NullSafeEqual(children);
+        return new NullSafeEqual(children, this.isInferred());
+    }
+
+    @Override
+    public Expression withInferred(boolean inferred) {
+        return new NullSafeEqual(this.children, inferred);
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 70b91dfe1028c1..b9c240df415f49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -48,6 +48,7 @@
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
 import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
 import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
+import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.visitor.ExpressionLineageReplacer;
 
@@ -639,4 +640,25 @@ public static boolean checkSlotConstant(Slot slot, Set<Expression> predicates) {
                 }
         );
     }
+
+    /**
+     * Check the expression is inferred or not, if inferred return true, nor return false
+     */
+    public static boolean isInferred(Expression expression) {
+        return expression.accept(new DefaultExpressionVisitor<Boolean, Void>() {
+
+            @Override
+            public Boolean visit(Expression expr, Void context) {
+                boolean inferred = expr.isInferred();
+                if (expr.isInferred() || expr.children().isEmpty()) {
+                    return inferred;
+                }
+                inferred = true;
+                for (Expression child : expr.children()) {
+                    inferred = inferred && child.accept(this, context);
+                }
+                return inferred;
+            }
+        }, null);
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java
index 1421a912a028de..c910e98fcd50c5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/InferPredicatesTest.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.rules.rewrite;
 
 import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.util.ExpressionUtils;
 import org.apache.doris.nereids.util.MemoPatternMatchSupported;
 import org.apache.doris.nereids.util.PlanChecker;
 import org.apache.doris.utframe.TestWithFeService;
@@ -87,10 +88,12 @@ public void inferPredicatesTest01() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("sid > 1"))
                         )
                     )
                 );
@@ -125,7 +128,8 @@ public void inferPredicatesTest03() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id IN (1, 2, 3)")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id IN (1, 2, 3)")),
                             logicalOlapScan()
                         )
                     )
@@ -144,7 +148,8 @@ public void inferPredicatesTest04() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id IN (1, 2, 3)")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id IN (1, 2, 3)")),
                             logicalOlapScan()
                         )
                     )
@@ -164,10 +169,12 @@ public void inferPredicatesTest05() {
                             logicalJoin(
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                                ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("id > 1")),
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("sid > 1"))
                             ),
                             logicalFilter(
                                 logicalOlapScan()
@@ -190,10 +197,12 @@ public void inferPredicatesTest06() {
                             logicalJoin(
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                                    ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("id > 1")),
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                    ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("sid > 1"))
                             ),
                             logicalFilter(
                                     logicalOlapScan()
@@ -215,10 +224,12 @@ public void inferPredicatesTest07() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("sid > 1"))
                         )
                     )
                 );
@@ -237,7 +248,8 @@ public void inferPredicatesTest08() {
                             logicalOlapScan(),
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("sid > 1"))
                         )
                     )
                 );
@@ -256,10 +268,12 @@ public void inferPredicatesTest09() {
                         logicalJoin(
                             logicalFilter(
                                 logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalFilter(
                                 logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("sid > 1"))
                         )
                     )
                 );
@@ -278,11 +292,13 @@ public void inferPredicatesTest10() {
                             logicalProject(
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("id > 1"))
+                                ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("id > 1"))
                             ),
                             logicalFilter(
                                 logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("sid > 1"))
                         )
                     )
                 );
@@ -303,7 +319,8 @@ public void inferPredicatesTest11() {
                             ),
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("sid > 1"))
                         )
                     )
                 );
@@ -321,13 +338,15 @@ public void inferPredicatesTest12() {
                         logicalJoin(
                             logicalFilter(
                                 logicalOlapScan()
-                            ).when(filer -> filer.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalProject(
                                 logicalAggregate(
                                     logicalProject(
                                         logicalFilter(
                                             logicalOlapScan()
-                                        ).when(filer -> filer.getPredicate().toSql().contains("sid > 1"))
+                                        ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                                & filter.getPredicate().toSql().contains("sid > 1"))
                                    )
                                 )
                             )
@@ -349,11 +368,13 @@ public void inferPredicatesTest13() {
                             logicalProject(
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("id = 1"))
+                                    ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("id = 1"))
                             ),
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("sid = 1"))
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("sid = 1"))
                         )
                     )
                 );
@@ -371,11 +392,13 @@ public void inferPredicatesTest14() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalProject(
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                    ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("sid > 1"))
                             )
                         )
                     )
@@ -394,11 +417,13 @@ public void inferPredicatesTest15() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalProject(
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                    ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("sid > 1"))
                             )
                         )
                     )
@@ -419,7 +444,8 @@ public void inferPredicatesTest16() {
                             logicalProject(
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("sid > 1"))
                             )
                         )
                     )
@@ -440,7 +466,8 @@ public void inferPredicatesTest17() {
                             logicalProject(
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                    ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("sid > 1"))
                             )
                         )
                     )
@@ -459,11 +486,13 @@ public void inferPredicatesTest18() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalProject(
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                    ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("sid > 1"))
                             )
                         )
                     )
@@ -504,19 +533,22 @@ public void inferPredicatesTest19() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("k1 = 3")),
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("k1 = 3")),
                             logicalProject(
                                 logicalJoin(
                                     logicalJoin(
                                        logicalProject(
                                                logicalFilter(
                                                        logicalOlapScan()
-                                               ).when(filter -> filter.getPredicate().toSql().contains("k3 = 3"))
+                                               ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                                       & filter.getPredicate().toSql().contains("k3 = 3"))
                                        ),
                                        logicalProject(
                                                logicalFilter(
                                                        logicalOlapScan()
-                                               ).when(filter -> filter.getPredicate().toSql().contains("k1 = 3"))
+                                               ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                                       & filter.getPredicate().toSql().contains("k1 = 3"))
                                        )
                                     ),
                                     logicalAggregate(
@@ -544,10 +576,12 @@ public void inferPredicatesTest20() {
                             innerLogicalJoin(
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                                ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("id > 1")),
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("sid > 1"))
                             ),
                             logicalFilter(
                                 logicalOlapScan()
@@ -570,10 +604,12 @@ public void inferPredicatesTest21() {
                             logicalJoin(
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                                ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("id > 1")),
                                 logicalFilter(
                                     logicalOlapScan()
-                                ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                        & filter.getPredicate().toSql().contains("sid > 1"))
                             ),
                             logicalFilter(
                                 logicalOlapScan()
@@ -598,11 +634,13 @@ public void inferPredicatesTest22() {
                         logicalJoin(
                             logicalFilter(
                                     logicalOlapScan()
-                            ).when(filter -> filter.getPredicate().toSql().contains("id > 1")),
+                            ).when(filter -> ExpressionUtils.isInferred(filter.getPredicate())
+                                    & filter.getPredicate().toSql().contains("id > 1")),
                             logicalProject(
                                     logicalFilter(
                                             logicalOlapScan()
-                                    ).when(filter -> filter.getPredicate().toSql().contains("sid > 1"))
+                                    ).when(filter -> !ExpressionUtils.isInferred(filter.getPredicate())
+                                            & filter.getPredicate().toSql().contains("sid > 1"))
                             )
                         )
                     )
@@ -626,6 +664,7 @@ public void innerJoinShouldNotInferUnderLeftJoinOnClausePredicates() {
                                 logicalFilter(
                                         logicalOlapScan()
                                 ).when(filter -> filter.getConjuncts().size() == 1
+                                        && !ExpressionUtils.isInferred(filter.getPredicate())
                                         && filter.getPredicate().toSql().contains("id = 2")),
                                 any()
                         ).when(join -> join.getJoinType() == JoinType.LEFT_OUTER_JOIN)

From 3e5c8d9949fb3ee92470a97e08b0c3c8a2400679 Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Wed, 27 Dec 2023 12:25:14 +0800
Subject: [PATCH 057/299] [fix](read) remove logic of estimating count of rows
 to read in segment iterator to avoid wrong result of unique key. (#29109)

---
 .../rowset/segment_v2/segment_iterator.cpp    | 25 -------------------
 .../olap/rowset/segment_v2/segment_iterator.h |  5 ----
 2 files changed, 30 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 3bbbe0bcbc6d6a..135e576825fb72 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -263,8 +263,6 @@ SegmentIterator::SegmentIterator(std::shared_ptr<Segment> segment, SchemaSPtr sc
           _lazy_materialization_read(false),
           _lazy_inited(false),
           _inited(false),
-          _estimate_row_size(true),
-          _wait_times_estimate_row_size(10),
           _pool(new ObjectPool) {}
 
 Status SegmentIterator::init(const StorageReadOptions& opts) {
@@ -2167,13 +2165,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
 
     _current_batch_rows_read = 0;
     uint32_t nrows_read_limit = _opts.block_row_max;
-    if (_wait_times_estimate_row_size > 0) {
-        // first time, read 100 rows to estimate average row size, to avoid oom caused by a single batch being too large.
-        // If no valid data is read for the first time, block_row_max is read each time thereafter.
-        // Avoid low performance when valid data cannot be read all the time
-        nrows_read_limit = std::min(nrows_read_limit, (uint32_t)100);
-        _wait_times_estimate_row_size--;
-    }
     RETURN_IF_ERROR(_read_columns_by_index(
             nrows_read_limit, _current_batch_rows_read,
             _lazy_materialization_read || _opts.record_rowids || _is_need_expr_eval));
@@ -2320,9 +2311,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
             // shrink char_type suffix zero data
             block->shrink_char_type_column_suffix_zero(_char_type_idx);
 
-            if (UNLIKELY(_estimate_row_size) && block->rows() > 0) {
-                _update_max_row(block);
-            }
             return Status::OK();
         }
         // step4: read non_predicate column
@@ -2358,10 +2346,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
     }
 #endif
 
-    if (UNLIKELY(_estimate_row_size) && block->rows() > 0) {
-        _update_max_row(block);
-    }
-
     // reverse block row order
     if (_opts.read_orderby_key_reverse) {
         size_t num_rows = block->rows();
@@ -2515,15 +2499,6 @@ void SegmentIterator::_convert_dict_code_for_predicate_if_necessary_impl(
     }
 }
 
-void SegmentIterator::_update_max_row(const vectorized::Block* block) {
-    _estimate_row_size = false;
-    auto avg_row_size = block->bytes() / block->rows();
-    if (avg_row_size > 0) {
-        int block_row_max = config::doris_scan_block_max_mb / avg_row_size;
-        _opts.block_row_max = std::min(block_row_max, _opts.block_row_max);
-    }
-}
-
 Status SegmentIterator::current_block_row_locations(std::vector<RowLocation>* block_row_locations) {
     DCHECK(_opts.record_rowids);
     DCHECK_GE(_block_rowids.size(), _current_batch_rows_read);
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index c953e0b9bcc8ba..45548c987c71b9 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -286,8 +286,6 @@ class SegmentIterator : public RowwiseIterator {
 
     void _convert_dict_code_for_predicate_if_necessary_impl(ColumnPredicate* predicate);
 
-    void _update_max_row(const vectorized::Block* block);
-
     bool _check_apply_by_bitmap_index(ColumnPredicate* pred);
     bool _check_apply_by_inverted_index(ColumnPredicate* pred, bool pred_in_compound = false);
 
@@ -442,9 +440,6 @@ class SegmentIterator : public RowwiseIterator {
     // the actual init process is delayed to the first call to next_batch()
     bool _lazy_inited;
     bool _inited;
-    bool _estimate_row_size;
-    // Read up to 100 rows at a time while waiting for the estimated row size.
-    int _wait_times_estimate_row_size;
 
     StorageReadOptions _opts;
     // make a copy of `_opts.column_predicates` in order to make local changes

From 0b5fe681e462eea7d45df2916995414b262110b9 Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchihdu@hotmail.com>
Date: Wed, 27 Dec 2023 12:35:17 +0800
Subject: [PATCH 058/299] [fix](paimon) read batch by doris' batch size
 (#29039)

---
 .../apache/doris/paimon/PaimonJniScanner.java    | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 237e06ad9c51e4..849f6fb67bb504 100644
--- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -57,6 +57,7 @@ public class PaimonJniScanner extends JniScanner {
     private long dbId;
     private long tblId;
     private long lastUpdateTime;
+    private RecordReader.RecordIterator<InternalRow> recordIterator = null;
 
     public PaimonJniScanner(int batchSize, Map<String, String> params) {
         LOG.debug("params:{}", params);
@@ -133,18 +134,25 @@ public void close() throws IOException {
     protected int getNext() throws IOException {
         int rows = 0;
         try {
-            RecordReader.RecordIterator<InternalRow> batch;
-            while ((batch = reader.readBatch()) != null) {
+            if (recordIterator == null) {
+                recordIterator = reader.readBatch();
+            }
+
+            while (recordIterator != null) {
                 InternalRow record;
-                while ((record = batch.next()) != null) {
+                while ((record = recordIterator.next()) != null) {
                     columnValue.setOffsetRow(record);
                     for (int i = 0; i < fields.length; i++) {
                         columnValue.setIdx(i, types[i]);
                         appendData(i, columnValue);
                     }
                     rows++;
+                    if (rows >= batchSize) {
+                        return rows;
+                    }
                 }
-                batch.releaseBatch();
+                recordIterator.releaseBatch();
+                recordIterator = reader.readBatch();
             }
         } catch (IOException e) {
             LOG.warn("failed to getNext columnValue ", e);

From 6ed09c3f46a2ba3e75452b3369e1bdcfc71d9d4b Mon Sep 17 00:00:00 2001
From: zhangdong <493738387@qq.com>
Date: Wed, 27 Dec 2023 12:36:17 +0800
Subject: [PATCH 059/299] [fix](mtmv)add log for resolve pending task (#29078)

---
 .../org/apache/doris/job/scheduler/JobScheduler.java     | 9 ++++++---
 .../org/apache/doris/regression/suite/Suite.groovy       | 4 ++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
index 08bbbb6dbaba9a..03a7792cc502f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
@@ -161,9 +161,12 @@ public void schedulerInstantJob(T job, TaskType taskType, C context) {
             }
             return;
         }
-        tasks.forEach(task -> taskDisruptorGroupManager.dispatchInstantTask(task, job.getJobType(),
-                job.getJobConfig()));
-
+        tasks.forEach(task -> {
+            taskDisruptorGroupManager.dispatchInstantTask(task, job.getJobType(),
+                    job.getJobConfig());
+            log.info("dispatch instant job, job id is {}, job name is {}, task id is {}", job.getJobId(),
+                    job.getJobName(), task.getTaskId());
+        });
     }
 
     /**
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 29f60841f31cfd..a299f0a01d49f0 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -845,7 +845,7 @@ class Suite implements GroovyInterceptable {
 
     void waitingMTMVTaskFinished(String jobName) {
         Thread.sleep(2000);
-        String showTasks = "select Status from tasks('type'='mv') where JobName = '${jobName}'"
+        String showTasks = "select TaskId,JobId,JobName,MvId,Status from tasks('type'='mv') where JobName = '${jobName}'"
         String status = "NULL"
         List<List<Object>> result
         long startTime = System.currentTimeMillis()
@@ -854,7 +854,7 @@ class Suite implements GroovyInterceptable {
             result = sql(showTasks)
             logger.info("result: " + result.toString())
             if (!result.isEmpty()) {
-                status = result.last().get(0)
+                status = result.last().get(4)
             }
             logger.info("The state of ${showTasks} is ${status}")
             Thread.sleep(1000);

From 6f5672f3184ee71e63537f9e16c3810248f77039 Mon Sep 17 00:00:00 2001
From: airborne12 <airborne08@gmail.com>
Date: Wed, 27 Dec 2023 12:49:26 +0800
Subject: [PATCH 060/299] [Refact](inverted index) refactor inverted index
 writer init (#29072)

---
 .../segment_v2/inverted_index_writer.cpp      | 319 ++++++++----------
 1 file changed, 137 insertions(+), 182 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index c2cc0bbbefa728..13a31e24768fc6 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -72,7 +72,6 @@ const int32_t MERGE_FACTOR = 100000000;
 const int32_t MAX_LEAF_COUNT = 1024;
 const float MAXMBSortInHeap = 512.0 * 8;
 const int DIMS = 1;
-const std::string empty_value;
 
 template <FieldType field_type>
 class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
@@ -139,9 +138,38 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
         return Status::OK();
     }
 
-    Status init_fulltext_index() {
-        bool create = true;
+    std::unique_ptr<lucene::analysis::Analyzer> create_chinese_analyzer() {
+        auto chinese_analyzer = std::make_unique<lucene::analysis::LanguageBasedAnalyzer>();
+        chinese_analyzer->setLanguage(L"chinese");
+        chinese_analyzer->initDict(config::inverted_index_dict_path);
+
+        auto mode = get_parser_mode_string_from_properties(_index_meta->properties());
+        if (mode == INVERTED_INDEX_PARSER_FINE_GRANULARITY) {
+            chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::All);
+        } else {
+            chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::Default);
+        }
+
+        return chinese_analyzer;
+    }
+
+    Status create_char_string_reader(std::unique_ptr<lucene::util::Reader>& string_reader) {
+        CharFilterMap char_filter_map =
+                get_parser_char_filter_map_from_properties(_index_meta->properties());
+        if (!char_filter_map.empty()) {
+            string_reader = std::unique_ptr<lucene::util::Reader>(CharFilterFactory::create(
+                    char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE],
+                    new lucene::util::SStringReader<char>(),
+                    char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN],
+                    char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT]));
+        } else {
+            string_reader = std::make_unique<lucene::util::SStringReader<char>>();
+        }
+        return Status::OK();
+    }
 
+    Status create_index_directory(std::unique_ptr<DorisCompoundDirectory>& dir) {
+        bool create = true;
         auto index_path = InvertedIndexDescriptor::get_temporary_index_path(
                 _directory + "/" + _segment_file_name, _index_meta->index_id(),
                 _index_meta->get_index_suffix());
@@ -149,8 +177,7 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
         bool exists = false;
         auto st = _fs->exists(index_path.c_str(), &exists);
         if (!st.ok()) {
-            LOG(ERROR) << "index_path:"
-                       << " exists error:" << st;
+            LOG(ERROR) << "index_path: exists error:" << st;
             return st;
         }
         if (exists) {
@@ -158,69 +185,76 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
             return Status::InternalError("init_fulltext_index directory already exists");
         }
 
-        _char_string_reader = std::make_unique<lucene::util::SStringReader<char>>();
-        CharFilterMap char_filter_map =
-                get_parser_char_filter_map_from_properties(_index_meta->properties());
-        if (!char_filter_map.empty()) {
-            _char_string_reader.reset(CharFilterFactory::create(
-                    char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE],
-                    _char_string_reader.release(),
-                    char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN],
-                    char_filter_map[INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT]));
-        }
+        dir = std::unique_ptr<DorisCompoundDirectory>(
+                DorisCompoundDirectory::getDirectory(_fs, index_path.c_str(), create));
+        return Status::OK();
+    }
 
-        _doc = std::make_unique<lucene::document::Document>();
-        _dir.reset(DorisCompoundDirectory::getDirectory(_fs, index_path.c_str(), true));
-
-        if (_parser_type == InvertedIndexParserType::PARSER_STANDARD ||
-            _parser_type == InvertedIndexParserType::PARSER_UNICODE) {
-            _analyzer = std::make_unique<lucene::analysis::standard95::StandardAnalyzer>();
-        } else if (_parser_type == InvertedIndexParserType::PARSER_ENGLISH) {
-            _analyzer = std::make_unique<lucene::analysis::SimpleAnalyzer<char>>();
-        } else if (_parser_type == InvertedIndexParserType::PARSER_CHINESE) {
-            auto chinese_analyzer = _CLNEW lucene::analysis::LanguageBasedAnalyzer();
-            chinese_analyzer->setLanguage(L"chinese");
-            chinese_analyzer->initDict(config::inverted_index_dict_path);
-            auto mode = get_parser_mode_string_from_properties(_index_meta->properties());
-            if (mode == INVERTED_INDEX_PARSER_FINE_GRANULARITY) {
-                chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::All);
-            } else {
-                chinese_analyzer->setMode(lucene::analysis::AnalyzerMode::Default);
-            }
-            _analyzer.reset(chinese_analyzer);
-        } else {
-            // ANALYSER_NOT_SET, ANALYSER_NONE use default SimpleAnalyzer
-            _analyzer = std::make_unique<lucene::analysis::SimpleAnalyzer<char>>();
+    Status create_index_writer(std::unique_ptr<lucene::index::IndexWriter>& index_writer) {
+        bool create_index = true;
+        bool close_dir_on_shutdown = true;
+        index_writer = std::make_unique<lucene::index::IndexWriter>(
+                _dir.get(), _analyzer.get(), create_index, close_dir_on_shutdown);
+        index_writer->setMaxBufferedDocs(MAX_BUFFER_DOCS);
+        index_writer->setRAMBufferSizeMB(config::inverted_index_ram_buffer_size);
+        index_writer->setMaxFieldLength(MAX_FIELD_LEN);
+        index_writer->setMergeFactor(MERGE_FACTOR);
+        index_writer->setUseCompoundFile(false);
+
+        return Status::OK();
+    }
+
+    Status create_field(lucene::document::Field** field) {
+        int field_config = int(lucene::document::Field::STORE_NO) |
+                           int(lucene::document::Field::INDEX_NONORMS);
+        field_config |= (_parser_type == InvertedIndexParserType::PARSER_NONE)
+                                ? int(lucene::document::Field::INDEX_UNTOKENIZED)
+                                : int(lucene::document::Field::INDEX_TOKENIZED);
+        *field = new lucene::document::Field(_field_name.c_str(), field_config);
+        (*field)->setOmitTermFreqAndPositions(
+                get_parser_phrase_support_string_from_properties(_index_meta->properties()) ==
+                                INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES
+                        ? false
+                        : true);
+        return Status::OK();
+    }
+
+    Status create_analyzer(std::unique_ptr<lucene::analysis::Analyzer>& analyzer) {
+        switch (_parser_type) {
+        case InvertedIndexParserType::PARSER_STANDARD:
+        case InvertedIndexParserType::PARSER_UNICODE:
+            analyzer = std::make_unique<lucene::analysis::standard95::StandardAnalyzer>();
+            break;
+        case InvertedIndexParserType::PARSER_ENGLISH:
+            analyzer = std::make_unique<lucene::analysis::SimpleAnalyzer<char>>();
+            break;
+        case InvertedIndexParserType::PARSER_CHINESE:
+            analyzer = create_chinese_analyzer();
+            break;
+        default:
+            analyzer = std::make_unique<lucene::analysis::SimpleAnalyzer<char>>();
+            break;
         }
+        setup_analyzer_lowercase(analyzer);
+        return Status::OK();
+    }
+
+    void setup_analyzer_lowercase(std::unique_ptr<lucene::analysis::Analyzer>& analyzer) {
         auto lowercase = get_parser_lowercase_from_properties(_index_meta->properties());
         if (lowercase == "true") {
-            _analyzer->set_lowercase(true);
+            analyzer->set_lowercase(true);
         } else if (lowercase == "false") {
-            _analyzer->set_lowercase(false);
+            analyzer->set_lowercase(false);
         }
-        _index_writer = std::make_unique<lucene::index::IndexWriter>(_dir.get(), _analyzer.get(),
-                                                                     create, true);
-        _index_writer->setMaxBufferedDocs(MAX_BUFFER_DOCS);
-        _index_writer->setRAMBufferSizeMB(config::inverted_index_ram_buffer_size);
-        _index_writer->setMaxFieldLength(MAX_FIELD_LEN);
-        _index_writer->setMergeFactor(MERGE_FACTOR);
-        _index_writer->setUseCompoundFile(false);
-        _doc->clear();
+    }
 
-        int field_config = int(lucene::document::Field::STORE_NO) |
-                           int(lucene::document::Field::INDEX_NONORMS);
-        if (_parser_type == InvertedIndexParserType::PARSER_NONE) {
-            field_config |= int(lucene::document::Field::INDEX_UNTOKENIZED);
-        } else {
-            field_config |= int(lucene::document::Field::INDEX_TOKENIZED);
-        }
-        _field = new lucene::document::Field(_field_name.c_str(), field_config);
-        if (get_parser_phrase_support_string_from_properties(_index_meta->properties()) ==
-            INVERTED_INDEX_PARSER_PHRASE_SUPPORT_YES) {
-            _field->setOmitTermFreqAndPositions(false);
-        } else {
-            _field->setOmitTermFreqAndPositions(true);
-        }
+    Status init_fulltext_index() {
+        RETURN_IF_ERROR(create_index_directory(_dir));
+        RETURN_IF_ERROR(create_char_string_reader(_char_string_reader));
+        RETURN_IF_ERROR(create_analyzer(_analyzer));
+        RETURN_IF_ERROR(create_index_writer(_index_writer));
+        RETURN_IF_ERROR(create_field(&_field));
+        _doc = std::make_unique<lucene::document::Document>();
         _doc->add(*_field);
         return Status::OK();
     }
@@ -555,12 +589,13 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
     roaring::Roaring _null_bitmap;
     uint64_t _reverted_index_size;
 
-    std::unique_ptr<lucene::document::Document> _doc {};
-    lucene::document::Field* _field {};
-    std::unique_ptr<lucene::index::IndexWriter> _index_writer {};
-    std::unique_ptr<lucene::analysis::Analyzer> _analyzer {};
-    std::unique_ptr<lucene::util::Reader> _char_string_reader {};
-    std::shared_ptr<lucene::util::bkd::bkd_writer> _bkd_writer;
+    std::unique_ptr<lucene::document::Document> _doc = nullptr;
+    lucene::document::Field* _field = nullptr;
+    std::unique_ptr<lucene::index::IndexWriter> _index_writer = nullptr;
+    std::unique_ptr<lucene::analysis::Analyzer> _analyzer = nullptr;
+    std::unique_ptr<lucene::util::Reader> _char_string_reader = nullptr;
+    std::shared_ptr<lucene::util::bkd::bkd_writer> _bkd_writer = nullptr;
+    std::unique_ptr<DorisCompoundDirectory> _dir = nullptr;
     std::string _segment_file_name;
     std::string _directory;
     io::FileSystemSPtr _fs;
@@ -568,7 +603,6 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
     const TabletIndex* _index_meta;
     InvertedIndexParserType _parser_type;
     std::wstring _field_name;
-    std::unique_ptr<DorisCompoundDirectory> _dir;
 };
 
 Status InvertedIndexColumnWriter::create(const Field* field,
@@ -576,127 +610,48 @@ Status InvertedIndexColumnWriter::create(const Field* field,
                                          const std::string& segment_file_name,
                                          const std::string& dir, const TabletIndex* index_meta,
                                          const io::FileSystemSPtr& fs) {
-    auto typeinfo = field->type_info();
+    const auto* typeinfo = field->type_info();
     FieldType type = typeinfo->type();
     std::string field_name = field->name();
     if (type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
-        const auto array_typeinfo = dynamic_cast<const ArrayTypeInfo*>(typeinfo);
-        typeinfo = array_typeinfo->item_type_info();
-        type = typeinfo->type();
+        const auto* array_typeinfo = dynamic_cast<const ArrayTypeInfo*>(typeinfo);
+        if (array_typeinfo != nullptr) {
+            typeinfo = array_typeinfo->item_type_info();
+            type = typeinfo->type();
+        } else {
+            return Status::NotSupported("unsupported array type for inverted index: " +
+                                        std::to_string(int(type)));
+        }
     }
 
     switch (type) {
-    case FieldType::OLAP_FIELD_TYPE_CHAR: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_CHAR>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_VARCHAR: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_VARCHAR>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_STRING: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_STRING>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DATETIME: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DATETIME>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DATE: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DATE>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: {
-        *res = std::make_unique<
-                InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DATETIMEV2>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DATEV2: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DATEV2>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_TINYINT: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_TINYINT>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_SMALLINT: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_SMALLINT>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT: {
-        *res = std::make_unique<
-                InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT>>(
-                field_name, segment_file_name, dir, fs, index_meta);
+#define M(TYPE)                                                       \
+    case TYPE:                                                        \
+        *res = std::make_unique<InvertedIndexColumnWriterImpl<TYPE>>( \
+                field_name, segment_file_name, dir, fs, index_meta);  \
         break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_INT: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_INT>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_LARGEINT: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_LARGEINT>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DECIMAL: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DECIMAL32: {
-        *res = std::make_unique<
-                InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL32>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DECIMAL64: {
-        *res = std::make_unique<
-                InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL64>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: {
-        *res = std::make_unique<
-                InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL128I>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DECIMAL256: {
-        *res = std::make_unique<
-                InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DECIMAL256>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_BOOL: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_BOOL>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_DOUBLE: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_DOUBLE>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_FLOAT: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_FLOAT>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
-    case FieldType::OLAP_FIELD_TYPE_BIGINT: {
-        *res = std::make_unique<InvertedIndexColumnWriterImpl<FieldType::OLAP_FIELD_TYPE_BIGINT>>(
-                field_name, segment_file_name, dir, fs, index_meta);
-        break;
-    }
+        M(FieldType::OLAP_FIELD_TYPE_TINYINT)
+        M(FieldType::OLAP_FIELD_TYPE_SMALLINT)
+        M(FieldType::OLAP_FIELD_TYPE_INT)
+        M(FieldType::OLAP_FIELD_TYPE_UNSIGNED_INT)
+        M(FieldType::OLAP_FIELD_TYPE_BIGINT)
+        M(FieldType::OLAP_FIELD_TYPE_LARGEINT)
+        M(FieldType::OLAP_FIELD_TYPE_CHAR)
+        M(FieldType::OLAP_FIELD_TYPE_VARCHAR)
+        M(FieldType::OLAP_FIELD_TYPE_STRING)
+        M(FieldType::OLAP_FIELD_TYPE_DATE)
+        M(FieldType::OLAP_FIELD_TYPE_DATETIME)
+        M(FieldType::OLAP_FIELD_TYPE_DECIMAL)
+        M(FieldType::OLAP_FIELD_TYPE_DATEV2)
+        M(FieldType::OLAP_FIELD_TYPE_DATETIMEV2)
+        M(FieldType::OLAP_FIELD_TYPE_DECIMAL32)
+        M(FieldType::OLAP_FIELD_TYPE_DECIMAL64)
+        M(FieldType::OLAP_FIELD_TYPE_DECIMAL128I)
+        M(FieldType::OLAP_FIELD_TYPE_DECIMAL256)
+        M(FieldType::OLAP_FIELD_TYPE_BOOL)
+        M(FieldType::OLAP_FIELD_TYPE_DOUBLE)
+        M(FieldType::OLAP_FIELD_TYPE_FLOAT)
+#undef M
     default:
         return Status::NotSupported("unsupported type for inverted index: " +
                                     std::to_string(int(type)));

From e33395a0ee4fb1a17ec9f8a3610d785139325bfc Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchihdu@hotmail.com>
Date: Wed, 27 Dec 2023 12:57:39 +0800
Subject: [PATCH 061/299] [fix](paimon)Remove the static attribute of the
 source for paimon (#29032)

---
 .../planner/external/FileQueryScanNode.java   | 23 +++++--------------
 .../planner/external/MaxComputeScanNode.java  |  9 +++++++-
 .../planner/external/hudi/HudiScanNode.java   |  9 +++++++-
 .../external/iceberg/IcebergScanNode.java     |  9 +++++++-
 .../external/paimon/PaimonScanNode.java       | 13 ++++++++---
 .../paimon/test_paimon_catalog.groovy         | 18 +++++++++++++++
 6 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
index 658ac49659ba3e..9bae707c2f3ed8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -38,12 +38,6 @@
 import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
 import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
 import org.apache.doris.planner.PlanNodeId;
-import org.apache.doris.planner.external.hudi.HudiScanNode;
-import org.apache.doris.planner.external.hudi.HudiSplit;
-import org.apache.doris.planner.external.iceberg.IcebergScanNode;
-import org.apache.doris.planner.external.iceberg.IcebergSplit;
-import org.apache.doris.planner.external.paimon.PaimonScanNode;
-import org.apache.doris.planner.external.paimon.PaimonSplit;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
@@ -260,6 +254,10 @@ public TFileScanRangeParams getFileScanRangeParams() {
         return params;
     }
 
+    // Set some parameters of scan to support different types of file data sources
+    protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+    }
+
     @Override
     public void createScanRangeLocations() throws UserException {
         long start = System.currentTimeMillis();
@@ -353,17 +351,7 @@ public void createScanRangeLocations() throws UserException {
                 rangeDesc.setTableFormatParams(tableFormatFileDesc);
             }
 
-            // external data lake table
-            if (fileSplit instanceof IcebergSplit) {
-                // TODO: extract all data lake split to factory
-                IcebergScanNode.setIcebergParams(rangeDesc, (IcebergSplit) fileSplit);
-            } else if (fileSplit instanceof PaimonSplit) {
-                PaimonScanNode.setPaimonParams(rangeDesc, (PaimonSplit) fileSplit);
-            } else if (fileSplit instanceof HudiSplit) {
-                HudiScanNode.setHudiParams(rangeDesc, (HudiSplit) fileSplit);
-            } else if (fileSplit instanceof MaxComputeSplit) {
-                MaxComputeScanNode.setScanParams(rangeDesc, (MaxComputeSplit) fileSplit);
-            }
+            setScanParams(rangeDesc, fileSplit);
 
             curLocations.getScanRange().getExtScanRange().getFileScanRange().addToRanges(rangeDesc);
             TScanRangeLocation location = new TScanRangeLocation();
@@ -473,6 +461,7 @@ private TFileRangeDesc createFileRangeDesc(FileSplit fileSplit, List<String> col
 
     protected abstract TFileFormatType getFileFormatType() throws UserException;
 
+
     protected TFileCompressType getFileCompressType(FileSplit fileSplit) throws UserException {
         return Util.inferFileCompressTypeByPath(fileSplit.getPath().toString());
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
index ae0b424ad815af..8fe7c8dff2283d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
@@ -63,7 +63,14 @@ public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeNa
         catalog = (MaxComputeExternalCatalog) table.getCatalog();
     }
 
-    public static void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) {
+    @Override
+    protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+        if (split instanceof MaxComputeSplit) {
+            setScanParams(rangeDesc, (MaxComputeSplit) split);
+        }
+    }
+
+    public void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeSplit) {
         TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
         tableFormatFileDesc.setTableFormatType(TableFormatType.MAX_COMPUTE.value());
         TMaxComputeFileDesc fileDesc = new TMaxComputeFileDesc();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
index f73947262e48c0..9d601e71daa951 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java
@@ -132,7 +132,14 @@ protected Map<String, String> getLocationProperties() throws UserException {
         }
     }
 
-    public static void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) {
+    @Override
+    protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+        if (split instanceof HudiSplit) {
+            setHudiParams(rangeDesc, (HudiSplit) split);
+        }
+    }
+
+    public void setHudiParams(TFileRangeDesc rangeDesc, HudiSplit hudiSplit) {
         TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
         tableFormatFileDesc.setTableFormatType(hudiSplit.getTableFormatType().value());
         THudiFileDesc fileDesc = new THudiFileDesc();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
index e8e90d5096a8c9..ae3efccadc5ff1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanNode.java
@@ -129,7 +129,14 @@ protected void doInitialize() throws UserException {
         super.doInitialize();
     }
 
-    public static void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) {
+    @Override
+    protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+        if (split instanceof IcebergSplit) {
+            setIcebergParams(rangeDesc, (IcebergSplit) split);
+        }
+    }
+
+    public void setIcebergParams(TFileRangeDesc rangeDesc, IcebergSplit icebergSplit) {
         TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
         tableFormatFileDesc.setTableFormatType(icebergSplit.getTableFormatType().value());
         TIcebergFileDesc fileDesc = new TIcebergFileDesc();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
index 7e8a4b78773c80..1e23d2f781df20 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonScanNode.java
@@ -54,8 +54,8 @@
 import java.util.stream.Collectors;
 
 public class PaimonScanNode extends FileQueryScanNode {
-    private static PaimonSource source = null;
-    private static List<Predicate> predicates;
+    private PaimonSource source = null;
+    private List<Predicate> predicates;
 
     public PaimonScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) {
         super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, needCheckColumnPriv);
@@ -91,7 +91,14 @@ public static <T> String encodeObjectToString(T t) {
         }
     }
 
-    public static void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) {
+    @Override
+    protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
+        if (split instanceof PaimonSplit) {
+            setPaimonParams(rangeDesc, (PaimonSplit) split);
+        }
+    }
+
+    public void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit) {
         TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc();
         tableFormatFileDesc.setTableFormatType(paimonSplit.getTableFormatType().value());
         TPaimonFileDesc fileDesc = new TPaimonFileDesc();
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index c00ef1cf1be6a1..ecf3263f82506a 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -284,5 +284,23 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_
             qt_c98 c98
             qt_c99 c99
             qt_c100 c100
+
+            // test view from jion paimon
+            sql """ switch internal """
+            String view_db = "test_view_for_paimon"
+            sql """ drop database if exists ${view_db}"""
+            sql """ create database if not exists ${view_db}"""
+            sql """use ${view_db}"""
+            sql """ create view test_tst_1 as select * from ${catalog_name}.`db1`.all_table; """
+            sql """ create view test_tst_2 as select * from ${catalog_name}.`db1`.all_table_with_parquet; """
+            sql """ create view test_tst_5 as select * from ${catalog_name}.`db1`.array_nested; """
+            sql """ create table test_tst_6 properties ("replication_num" = "1") as 
+                        select f.c2,f.c3,c.c1 from 
+                            (select a.c2,b.c3 from test_tst_1 a inner join test_tst_2 b on a.c2=b.c2) f
+                    inner join test_tst_5 c on f.c2=c.c1;
+                """
+            def view1 = """select * from test_tst_6 order by c1"""
+
+            // qt_view1 view1
         }
 }

From 51bd402b5884ff599c3bf3cb214b4b0eeeb12434 Mon Sep 17 00:00:00 2001
From: bobhan1 <bh2444151092@outlook.com>
Date: Wed, 27 Dec 2023 12:59:15 +0800
Subject: [PATCH 062/299] [fix](planner) Fix delete statement on MOR unique
 table (#28968)

add conditions for mor table removed in #26776
---
 .../org/apache/doris/analysis/DeleteStmt.java |   2 +-
 .../commands/DeleteFromUsingCommand.java      |   2 +-
 .../data/delete_p0/test_delete_on_mor.out     |  51 +++++++++
 .../delete_p0/test_delete_on_mor.groovy       | 106 ++++++++++++++++++
 4 files changed, 159 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/data/delete_p0/test_delete_on_mor.out
 create mode 100644 regression-test/suites/delete_p0/test_delete_on_mor.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
index 24c54ae8bcf7d6..3ab422bdf34cea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DeleteStmt.java
@@ -157,7 +157,7 @@ private void constructInsertStmt() throws AnalysisException {
                 expr = new BoolLiteral(true);
             } else if (column.isKey()) {
                 expr = new SlotRef(targetTableRef.getAliasAsName(), column.getName());
-            } else if (!isMow && !column.isVisible()) {
+            } else if (!isMow && (!column.isVisible() || (!column.isAllowNull() && !column.hasDefaultValue()))) {
                 expr = new SlotRef(targetTableRef.getAliasAsName(), column.getName());
             } else {
                 continue;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java
index 3022578ed60a74..b8ac38939a842f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DeleteFromUsingCommand.java
@@ -96,7 +96,7 @@ public LogicalPlan completeQueryPlan(ConnectContext ctx, LogicalPlan logicalQuer
                 selectLists.add(new UnboundSlot(tableName, targetTable.getSequenceMapCol()));
             } else if (column.isKey()) {
                 selectLists.add(new UnboundSlot(tableName, column.getName()));
-            } else if (!isMow && !column.isVisible()) {
+            } else if (!isMow && (!column.isVisible() || (!column.isAllowNull() && !column.hasDefaultValue()))) {
                 selectLists.add(new UnboundSlot(tableName, column.getName()));
             } else {
                 continue;
diff --git a/regression-test/data/delete_p0/test_delete_on_mor.out b/regression-test/data/delete_p0/test_delete_on_mor.out
new file mode 100644
index 00000000000000..69029c568889c1
--- /dev/null
+++ b/regression-test/data/delete_p0/test_delete_on_mor.out
@@ -0,0 +1,51 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+10000	u1	北京	20	1
+10001	u3	北京	30	0
+10002	u4	上海	20	1
+10003	u5	重庆	32	0
+10004	u7	重庆	35	1
+
+-- !sql --
+10000	u1	北京	20	1
+10001	u3	北京	30	0
+10002	u4	上海	20	1
+10003	u5	广州	32	0
+10004	u7	深圳	35	1
+
+-- !sql --
+10001	u3	北京	30	0
+10002	u4	上海	20	1
+10003	u5	重庆	32	0
+10004	u7	重庆	35	1
+
+-- !sql --
+10001	u3	北京	30	0
+10003	u5	重庆	32	0
+10004	u7	重庆	35	1
+
+-- !sql --
+10000	u1	北京	20	1
+10001	u3	北京	30	0
+10002	u4	上海	20	1
+10003	u5	重庆	32	0
+10004	u7	重庆	35	1
+
+-- !sql --
+10000	u1	北京	20	1
+10001	u3	北京	30	0
+10002	u4	上海	20	1
+10003	u5	广州	32	0
+10004	u7	深圳	35	1
+
+-- !sql --
+10001	u3	北京	30	0
+10002	u4	上海	20	1
+10003	u5	重庆	32	0
+10004	u7	重庆	35	1
+
+-- !sql --
+10001	u3	北京	30	0
+10003	u5	重庆	32	0
+10004	u7	重庆	35	1
+
diff --git a/regression-test/suites/delete_p0/test_delete_on_mor.groovy b/regression-test/suites/delete_p0/test_delete_on_mor.groovy
new file mode 100644
index 00000000000000..3388356e6e304f
--- /dev/null
+++ b/regression-test/suites/delete_p0/test_delete_on_mor.groovy
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_delete_on_mor") {
+    String db = context.config.getDbNameByFile(context.file)
+    sql "select 1;" // to create database
+
+    for (def use_nereids_planner : [false, true]) {
+        logger.info("current params: use_nereids_planner: ${use_nereids_planner}")
+        connect(user = context.config.jdbcUser, password = context.config.jdbcPassword, url = context.config.jdbcUrl) {
+            sql "use ${db};"
+            if (use_nereids_planner) {
+                sql """ set enable_nereids_dml = true; """
+                sql """ set enable_nereids_planner=true; """
+                sql """ set enable_fallback_to_original_planner=false; """
+            } else {
+                sql """ set enable_nereids_dml = false; """
+                sql """ set enable_nereids_planner = false; """
+            }
+
+            def tableA = "test_delete_on_mor1"
+            sql """ DROP TABLE IF EXISTS ${tableA} """
+            sql """ CREATE TABLE IF NOT EXISTS ${tableA} (
+                `user_id` LARGEINT NOT NULL COMMENT "用户id",
+                `username` VARCHAR(50) NOT NULL COMMENT "用户昵称",
+                `city` VARCHAR(20) COMMENT "用户所在城市",
+                `age` SMALLINT COMMENT "用户年龄",
+                `sex` TINYINT COMMENT "用户性别")
+                UNIQUE KEY(`user_id`)
+                DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
+                PROPERTIES (
+                    "enable_unique_key_merge_on_write" = "false",
+                    "replication_allocation" = "tag.location.default: 1"
+                );"""
+
+            def tableB = "test_delete_on_mor2"
+            sql """ DROP TABLE IF EXISTS ${tableB} """
+            sql """ CREATE TABLE IF NOT EXISTS ${tableB} (
+                `user_id` LARGEINT NOT NULL COMMENT "用户id",
+                `username` VARCHAR(50) NOT NULL COMMENT "用户昵称",
+                `city` VARCHAR(20) COMMENT "用户所在城市",
+                `age` SMALLINT COMMENT "用户年龄",
+                `sex` TINYINT COMMENT "用户性别")
+                UNIQUE KEY(`user_id`)
+                DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
+                PROPERTIES (
+                    "enable_unique_key_merge_on_write" = "false",
+                    "replication_allocation" = "tag.location.default: 1"
+                );"""
+
+            sql """insert into ${tableA} values
+                (10000,"u1","北京",19,1),
+                (10000,"u1","北京",20,1),
+                (10001,"u3","北京",30,0),
+                (10002,"u4","上海",20,1),
+                (10003,"u5","重庆",32,0),
+                (10004,"u6","重庆",35,1),
+                (10004,"u7","重庆",35,1);  """
+
+            sql """insert into ${tableB} values
+                (10000,"u1","北京",18,1),
+                (10000,"u1","北京",20,1),
+                (10001,"u3","北京",30,0),
+                (10002,"u4","上海",20,1),
+                (10003,"u5","广州",32,0),
+                (10004,"u6","深圳",35,1),
+                (10004,"u7","深圳",35,1); """
+            
+            sql "sync;"
+            qt_sql "select * from ${tableA} order by user_id;"
+            qt_sql "select * from ${tableB} order by user_id;"
+            sql """ DELETE FROM ${tableA} a USING ${tableB} b
+                WHERE a.user_id = b.user_id AND a.city = b.city
+                and b.city = '北京' AND b.age = 20;"""
+            sql "sync;"
+            qt_sql "select * from ${tableA} order by user_id;"
+
+            sql """DELETE from ${tableA} USING ${tableA}  a
+            JOIN (
+                SELECT a.user_id, a.city
+                FROM ${tableA} a
+                JOIN ${tableB} ON a.user_id = ${tableB}.user_id AND a.city = ${tableB}.city
+                WHERE ${tableB}.city = '上海' AND ${tableB}.age = 20
+            ) AS matched_rows
+            ON ${tableA}.user_id = matched_rows.user_id AND ${tableA}.city = matched_rows.city; """
+            qt_sql "select * from ${tableA} order by user_id;"
+
+            sql "DROP TABLE IF EXISTS ${tableA};"
+            sql "DROP TABLE IF EXISTS ${tableB};"
+        }
+    }
+}

From 05f185ff44dc2b07029bfac57ac3b56099db1013 Mon Sep 17 00:00:00 2001
From: lw112 <131352377+felixwluo@users.noreply.github.com>
Date: Wed, 27 Dec 2023 13:00:43 +0800
Subject: [PATCH 063/299] [fix][compile]unused variable (#28992)


From 0ac9b3d1132f9521b213bce9f58e6cb6e99822ec Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Wed, 27 Dec 2023 13:59:12 +0800
Subject: [PATCH 064/299] [Fix](Job)cancel task is not cleared from running
 task (#29114)

---
 .../apache/doris/job/base/AbstractJob.java    | 19 +++--
 .../job/extensions/insert/InsertJob.java      | 51 ++++++++-----
 .../apache/doris/job/manager/JobManager.java  |  1 +
 .../doris/job/scheduler/JobScheduler.java     |  2 +-
 .../suites/job_p0/test_base_insert_job.groovy | 72 +++++++++++++------
 5 files changed, 98 insertions(+), 47 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
index 6e9cb48da1ca44..bc8f921d841dbe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
@@ -89,7 +89,8 @@ public abstract class AbstractJob<T extends AbstractTask, C> implements Job<T, C
     @SerializedName(value = "sql")
     String executeSql;
 
-    public AbstractJob() {}
+    public AbstractJob() {
+    }
 
     public AbstractJob(Long id) {
         jobId = id;
@@ -99,10 +100,10 @@ public AbstractJob(Long id) {
      * executeSql and runningTasks is not required for load.
      */
     public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
-                            String currentDbName,
-                            String comment,
-                            UserIdentity createUser,
-                            JobExecutionConfiguration jobConfig) {
+                       String currentDbName,
+                       String comment,
+                       UserIdentity createUser,
+                       JobExecutionConfiguration jobConfig) {
         this(jobId, jobName, jobStatus, currentDbName, comment,
                 createUser, jobConfig, System.currentTimeMillis(), null, null);
     }
@@ -137,6 +138,7 @@ public void cancelAllTasks() throws JobException {
         for (T task : runningTasks) {
             task.cancel();
         }
+        runningTasks = new ArrayList<>();
     }
 
     private static final ImmutableList<String> TITLE_NAMES =
@@ -163,6 +165,7 @@ public void cancelTaskById(long taskId) throws JobException {
         }
         runningTasks.stream().filter(task -> task.getTaskId().equals(taskId)).findFirst()
                 .orElseThrow(() -> new JobException("no task id: " + taskId)).cancel();
+        runningTasks.removeIf(task -> task.getTaskId().equals(taskId));
         if (jobConfig.getExecuteType().equals(JobExecuteType.ONE_TIME)) {
             updateJobStatus(JobStatus.FINISHED);
         }
@@ -364,10 +367,12 @@ public ShowResultSetMetaData getJobMetaData() {
     }
 
     @Override
-    public void onRegister() throws JobException {}
+    public void onRegister() throws JobException {
+    }
 
     @Override
-    public void onUnRegister() throws JobException {}
+    public void onUnRegister() throws JobException {
+    }
 
     @Override
     public void onReplayCreate() throws JobException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
index 9256864efcacc1..90fc126b723c27 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
@@ -46,6 +46,7 @@
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.mysql.privilege.Privilege;
 import org.apache.doris.nereids.trees.plans.commands.InsertIntoTableCommand;
+import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.qe.ShowResultSetMetaData;
@@ -67,7 +68,6 @@
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
-import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -84,7 +84,7 @@
 @EqualsAndHashCode(callSuper = true)
 @Data
 @Slf4j
-public class InsertJob extends AbstractJob<InsertTask, Map<Object, Object>> {
+public class InsertJob extends AbstractJob<InsertTask, Map<Object, Object>> implements GsonPostProcessable {
 
     public static final ImmutableList<Column> SCHEMA = ImmutableList.of(
             new Column("Id", ScalarType.createStringType()),
@@ -156,6 +156,31 @@ public class InsertJob extends AbstractJob<InsertTask, Map<Object, Object>> {
     // max save task num, do we need to config it?
     private static final int MAX_SAVE_TASK_NUM = 100;
 
+    @Override
+    public void gsonPostProcess() throws IOException {
+        if (null == plans) {
+            plans = new ArrayList<>();
+        }
+        if (null == idToTasks) {
+            idToTasks = new ConcurrentHashMap<>();
+        }
+        if (null == loadStatistic) {
+            loadStatistic = new LoadStatistic();
+        }
+        if (null == finishedTaskIds) {
+            finishedTaskIds = new HashSet<>();
+        }
+        if (null == errorTabletInfos) {
+            errorTabletInfos = new ArrayList<>();
+        }
+        if (null == commitInfos) {
+            commitInfos = new ArrayList<>();
+        }
+        if (null == historyTaskIdList) {
+            historyTaskIdList = new ConcurrentLinkedQueue<>();
+        }
+    }
+
     /**
      * load job type
      */
@@ -197,13 +222,13 @@ public InsertJob(String jobName,
     }
 
     public InsertJob(ConnectContext ctx,
-                      StmtExecutor executor,
-                      String labelName,
-                      List<InsertIntoTableCommand> plans,
-                      Set<String> sinkTableNames,
-                      Map<String, String> properties,
-                      String comment,
-                      JobExecutionConfiguration jobConfig) {
+                     StmtExecutor executor,
+                     String labelName,
+                     List<InsertIntoTableCommand> plans,
+                     Set<String> sinkTableNames,
+                     Map<String, String> properties,
+                     String comment,
+                     JobExecutionConfiguration jobConfig) {
         super(getNextJobId(), labelName, JobStatus.RUNNING, null,
                 comment, ctx.getCurrentUserIdentity(), jobConfig);
         this.ctx = ctx;
@@ -460,14 +485,6 @@ public long getTimeout() {
         return Config.broker_load_default_timeout_second;
     }
 
-
-    public static InsertJob readFields(DataInput in) throws IOException {
-        String jsonJob = Text.readString(in);
-        InsertJob job = GsonUtils.GSON.fromJson(jsonJob, InsertJob.class);
-        job.setRunningTasks(new ArrayList<>());
-        return job;
-    }
-
     @Override
     public void write(DataOutput out) throws IOException {
         Text.writeString(out, GsonUtils.GSON.toJson(this));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
index 814d6b773ad1ef..dae75b2b43ec3e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/manager/JobManager.java
@@ -220,6 +220,7 @@ public List<? extends AbstractTask> queryTasks(Long jobId) throws JobException {
     }
 
     public void triggerJob(long jobId, C context) throws JobException {
+        log.info("trigger job, job id is {}", jobId);
         checkJobExist(jobId);
         jobScheduler.schedulerInstantJob(jobMap.get(jobId), TaskType.MANUAL, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
index 03a7792cc502f1..597e39d96ed1a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/scheduler/JobScheduler.java
@@ -154,7 +154,7 @@ private void cycleTimerJobScheduler(T job, long startTimeWindowMs) {
     public void schedulerInstantJob(T job, TaskType taskType, C context) {
         List<? extends AbstractTask> tasks = job.commonCreateTasks(taskType, context);
         if (CollectionUtils.isEmpty(tasks)) {
-            log.info("job create task is empty, skip scheduler, job id is {},job name is {}", job.getJobId(),
+            log.info("job create task is empty, skip scheduler, job id is {}, job name is {}", job.getJobId(),
                     job.getJobName());
             if (job.getJobConfig().getExecuteType().equals(JobExecuteType.INSTANT)) {
                 job.setJobStatus(JobStatus.FINISHED);
diff --git a/regression-test/suites/job_p0/test_base_insert_job.groovy b/regression-test/suites/job_p0/test_base_insert_job.groovy
index b41bf50738b150..3ef2f86b8ebf5f 100644
--- a/regression-test/suites/job_p0/test_base_insert_job.groovy
+++ b/regression-test/suites/job_p0/test_base_insert_job.groovy
@@ -50,6 +50,10 @@ suite("test_base_insert_job") {
         DROP JOB where jobname =  '${jobMixedName}'
     """
 
+    sql """
+        DROP JOB where jobname =  '${jobName}'
+    """
+
     sql """
         CREATE TABLE IF NOT EXISTS `${tableName}`
         (
@@ -69,7 +73,7 @@ suite("test_base_insert_job") {
     Thread.sleep(2500)
     def jobs = sql """select * from ${tableName}"""
     println jobs
-    assert 3>=jobs.size() >= (2 as Boolean) //at least 2 records, some times 3 records
+    assert 3 >= jobs.size() >= (2 as Boolean) //at least 2 records, some times 3 records
     sql """
        CREATE JOB ${jobMixedName}  ON SCHEDULE every 1 second  DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
     """
@@ -83,7 +87,7 @@ suite("test_base_insert_job") {
     sql """
         DROP JOB where jobname =  '${jobMixedName}'
     """
-    
+
     sql """drop table if exists `${tableName}` force """
     sql """
         CREATE TABLE IF NOT EXISTS `${tableName}`
@@ -99,11 +103,11 @@ suite("test_base_insert_job") {
         );
         """
     // Enlarge this parameter to avoid other factors that cause time verification to fail when submitting.
-    def currentMs=System.currentTimeMillis()+20000;
-    def   dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(currentMs), ZoneId.systemDefault());
+    def currentMs = System.currentTimeMillis() + 20000;
+    def dateTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(currentMs), ZoneId.systemDefault());
 
     def formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
-    def startTime= dateTime.format(formatter);
+    def startTime = dateTime.format(formatter);
     def dataCount = sql """select count(*) from ${tableName}"""
     assert dataCount.get(0).get(0) == 0
     sql """
@@ -113,8 +117,8 @@ suite("test_base_insert_job") {
     Thread.sleep(25000)
     def onceJob = sql """ select id,ExecuteSql from jobs("type"="insert") where Name like '%${jobName}%' and ExecuteType='ONE_TIME' """
     assert onceJob.size() == 1
-    def onceJobId= onceJob.get(0).get(0);
-    def onceJobSql= onceJob.get(0).get(1);
+    def onceJobId = onceJob.get(0).get(0);
+    def onceJobSql = onceJob.get(0).get(1);
     println onceJobSql
     def assertSql = "insert into ${tableName}  values  (\'2023-07-19\', sleep(10000), 1001);"
     println assertSql
@@ -126,7 +130,7 @@ suite("test_base_insert_job") {
     assert datas.get(0).get(0) == "RUNNING"
     def taskId = datas.get(0).get(1)
     sql """cancel  task where jobName='${jobName}' and taskId= ${taskId}"""
-    def cancelTask = sql """ select status from tasks("type"="insert") where jobid= ${onceJobId}""" 
+    def cancelTask = sql """ select status from tasks("type"="insert") where jobid= ${onceJobId}"""
     println cancelTask
     //check task status
     assert cancelTask.size() == 1
@@ -135,32 +139,56 @@ suite("test_base_insert_job") {
     def dataCount1 = sql """select count(1) from ${tableName}"""
     assert dataCount1.get(0).get(0) == 0
     // check job status
-    def oncejob=sql """select status,comment from jobs("type"="insert") where Name='${jobName}' """
+    def oncejob = sql """select status,comment from jobs("type"="insert") where Name='${jobName}' """
     println oncejob
     assert oncejob.get(0).get(0) == "FINISHED"
     //assert comment
     assert oncejob.get(0).get(1) == "test for test&68686781jbjbhj//ncsa"
-    
+    sql """
+        DROP JOB where jobname =  '${jobName}'
+    """
+
+    sql """
+          CREATE JOB ${jobName}  ON SCHEDULE every 1 second   comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName}  values  ('2023-07-19', sleep(10000), 1001);
+     """
+
+    Thread.sleep(2500)
+
+    sql """
+        PAUSE JOB where jobname =  '${jobName}'
+    """
+    def job = sql """ select id,ExecuteSql from jobs("type"="insert") where Name like '%${jobName}%'  """
+    assert job.size() == 1
+    def jobId = job.get(0).get(0);
+    def tasks = sql """ select status from tasks("type"="insert") where jobid= ${jobId}  """
+    assert tasks.size() == 1
+    sql """
+        RESUME JOB where jobname =  '${jobName}'
+    """
+    Thread.sleep(2500)
+    def resumeTasks = sql """ select status from tasks("type"="insert") where jobid= ${jobId}  """
+    println resumeTasks
+    assert resumeTasks.size() == 2
     // assert same job name
     try {
         sql """
           CREATE JOB ${jobName}  ON SCHEDULE EVERY 10 second   comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${tableName}  values  ('2023-07-19', sleep(10000), 1001);
      """
-    }catch (Exception e) {
+    } catch (Exception e) {
         assert e.getMessage().contains("job name exist, jobName:insert_recovery_test_base_insert_job")
     }
-    def errorTblName="${tableName}qwertyuioppoiuyte"
+    def errorTblName = "${tableName}qwertyuioppoiuyte"
     sql """drop table if exists `${errorTblName}` force"""
     // assert error table name
     try {
         sql """
           CREATE JOB ${jobName}  ON SCHEDULE EVERY 10 second   comment 'test for test&68686781jbjbhj//ncsa' DO insert into ${errorTblName}  values  ('2023-07-19', sleep(10000), 1001);
      """
-    }catch (Exception e) {
+    } catch (Exception e) {
         assert e.getMessage().contains("Unknown table 't_test_BASE_inSert_jobqwertyuioppoiuyte'")
     }
     // assert not support stmt
-    try{
+    try {
         sql """
             CREATE JOB ${jobName}  ON SCHEDULE at '${startTime}'   comment 'test' DO update ${tableName} set type=2 where type=1;
         """
@@ -168,7 +196,7 @@ suite("test_base_insert_job") {
         assert e.getMessage().contains("Not support UpdateStmt type in job")
     }
     // assert start time greater than current time
-    try{
+    try {
         sql """
             CREATE JOB ${jobName}  ON SCHEDULE at '${startTime}'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
         """
@@ -176,14 +204,14 @@ suite("test_base_insert_job") {
         assert e.getMessage().contains("startTimeMs must be greater than current time")
     }
     // assert end time less than start time
-    try{
+    try {
         sql """
             CREATE JOB test_one_time_error_starts  ON SCHEDULE at '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
         """
     } catch (Exception e) {
         assert e.getMessage().contains("startTimeMs must be greater than current time")
     }
-    try{
+    try {
         sql """
             CREATE JOB inner_test  ON SCHEDULE at '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
         """
@@ -191,7 +219,7 @@ suite("test_base_insert_job") {
         assert e.getMessage().contains("job name can not start with inner_")
     }
     // assert end time less than start time
-    try{
+    try {
         sql """
             CREATE JOB test_error_starts  ON SCHEDULE every 1 second ends '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
         """
@@ -199,7 +227,7 @@ suite("test_base_insert_job") {
         assert e.getMessage().contains("end time cannot be less than start time")
     }
     // assert interval time unit can not be years
-    try{
+    try {
         sql """
             CREATE JOB test_error_starts  ON SCHEDULE every 1 years ends '2023-11-13 14:18:07'   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
         """
@@ -228,8 +256,8 @@ suite("test_base_insert_job") {
     sql """
         CREATE JOB ENDS  ON SCHEDULE every 20 second   comment 'test' DO insert into ${tableName} (timestamp, type, user_id) values ('2023-03-18','1','12213');
     """
-    
-    def jobCountRsp = sql"""select count(1) from jobs("type"="insert") where name in ('JOB','DO','SCHEDULE','AT','STARTS','ENDS')"""
+
+    def jobCountRsp = sql """select count(1) from jobs("type"="insert") where name in ('JOB','DO','SCHEDULE','AT','STARTS','ENDS')"""
     assert jobCountRsp.get(0).get(0) == 6
-    
+
 }

From fc6a587352337ff6f3ad56b720b60f24e6be4066 Mon Sep 17 00:00:00 2001
From: xzj7019 <131111794+xzj7019@users.noreply.github.com>
Date: Wed, 27 Dec 2023 14:03:53 +0800
Subject: [PATCH 065/299] [test](Nereids) add subquery pull up rule group ut
 (#26725)

add subquery pull up rule group ut:
single table part
(1) subquery_basic_pullup_basic: basic part
(2) subquery_basic_pullup_uk: using unique key
(3) subquery_basic_pullup_and: with and
(4) subquery_basic_pullup_or: with or
(5) subquery_basic_pullup_and_subquery: with and subquery
(6) subquery_basic_pullup_or_subquery: with or subquery

multi table part
(1) subquery_multitable_pullup_basic: basic part
(2) subquery_multitable_pullup_and: with and
(3) subquery_multitable_pullup_or: with or
(4) subquery_multitable_pullup_and_subquery: with and subquery
(5) subquery_multitable_pullup_or_subquery: with or subquery

top op related, such as group by/having, order by, win func, select list:
(1) subquery_topop_pullup_groupby: group by
(2) subquery_topop_pullup_having: having
(3) subquery_topop_pullup_orderby: order by
(4) subquery_topop_pullup_winfunc: window function
(5) subquery_topop_pullup_selectlist: select list

misc, such as group by correlated, view, dml, etc:
(1) subquery_misc_pullup_misc: group by correlated, view, etc
(2) subquery_misc_pullup_dml: insert/update/delete
---
 .../pipeline/p0/conf/regression-conf.groovy   |   2 +-
 .../subquery_basic_pullup_and_subquery.groovy | 157 ++++++++++
 .../basic/subquery_basic_pullup_basic.groovy  | 234 +++++++++++++++
 .../basic/subquery_basic_pullup_or.groovy     | 180 ++++++++++++
 .../subquery_basic_pullup_or_subquery.groovy  | 157 ++++++++++
 .../basic/subquery_basic_pullup_uk.groovy     | 237 +++++++++++++++
 .../basic/subquery_basic_ullup_and.groovy     | 180 ++++++++++++
 .../misc/subquery_misc_pullup_dml.groovy      | 126 ++++++++
 .../misc/subquery_misc_pullup_misc.groovy     | 144 +++++++++
 .../subquery_multitable_pullup_and.groovy     | 191 ++++++++++++
 ...uery_multitable_pullup_and_subquery.groovy | 270 +++++++++++++++++
 .../subquery_multitable_pullup_basic.groovy   | 191 ++++++++++++
 .../subquery_multitable_pullup_or.groovy      | 191 ++++++++++++
 ...query_multitable_pullup_or_subquery.groovy | 270 +++++++++++++++++
 .../subquery_topop_pullup_groupby.groovy      | 273 ++++++++++++++++++
 .../topop/subquery_topop_pullup_having.groovy | 273 ++++++++++++++++++
 .../subquery_topop_pullup_orderby.groovy      | 273 ++++++++++++++++++
 .../subquery_topop_pullup_selectlist.groovy   | 138 +++++++++
 .../subquery_topop_pullup_winfunc.groovy      | 185 ++++++++++++
 19 files changed, 3671 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_and_subquery.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_basic.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or_subquery.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_uk.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_ullup_and.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_dml.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_misc.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and_subquery.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_basic.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or_subquery.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_groupby.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_having.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_orderby.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_selectlist.groovy
 create mode 100644 regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_winfunc.groovy

diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy
index db3b31637a4cbb..b28b2e64004148 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -58,7 +58,7 @@ excludeGroups = ""
 excludeSuites = "test_profile,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_information_schema_external"
 
 // this directories will not be executed
-excludeDirectories = "workload_manager_p1,fault_injection_p0"
+excludeDirectories = "workload_manager_p1,fault_injection_p0,nereids_rules_p0/subquery"
 
 customConf1 = "test_custom_conf_value"
 
diff --git a/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_and_subquery.groovy b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_and_subquery.groovy
new file mode 100644
index 00000000000000..b78c6e21f7fbbe
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_and_subquery.groovy
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_and_subquery") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + filter + and + correlated + equal
+    qt_1_1 """select * from t1 where t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_2 """select * from t1 where t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_3 """select * from t1 where t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3);"""
+
+    qt_1_4 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_5 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_9 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 2. single table + olap table + filter + in + and + exists + correlated + equal
+    qt_2_1 """select * from t1 where t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_2 """select * from t1 where t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_3 """select * from t1 where t1.c1 in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_4 """select * from t1 where t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_5 """select * from t1 where t1.c1 in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_6 """select * from t1 where t1.c1 not in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_7 """select * from t1 where t1.c1 in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_8 """select * from t1 where t1.c1 not in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_9 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_10 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_11 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_12 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 3. single table + olap table + filter + and + correlated + non-equal
+    qt_3_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_3 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_4 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1) and exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_5 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1) and exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1) and not exists (select c1 from t3 where t1.c2 < t3.c1);"""
+
+    // 4. single table + olap table + filter + and + non-correlated
+    qt_4_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_3 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_4 """select * from t1 where exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_5 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1) and not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    // 5. single table + olap table + filter + correlated + and + non-correlated
+    qt_5_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_3 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_4 """select * from t1 where exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_5 """select * from t1 where not exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_6 """select * from t1 where not exists (select c1 from t2 where t2.c1 = 1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_basic.groovy b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_basic.groovy
new file mode 100644
index 00000000000000..8621440a529c36
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_basic.groovy
@@ -0,0 +1,234 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_basic") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + filter + basic + correlated + equal
+    qt_1_1 """select * from t1 where t1.c1 in (select c1 from t2);"""
+    qt_1_2 """select * from t1 where t1.c1 not in (select c1 from t2);"""
+
+    qt_1_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2);"""
+    qt_1_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_1_5 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 + 1);"""
+    qt_1_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 + 1);"""
+
+    qt_1_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1);"""
+    qt_1_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1);"""
+
+    qt_1_9 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 + 1);"""
+    qt_1_10 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 + 1);"""
+
+    // 2. single table + olap table + filter + basic + correlated + non-equal
+    qt_2_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2);"""
+    qt_2_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2);"""
+
+    qt_2_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2 + 1);"""
+    qt_2_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2 + 1);"""
+
+    qt_2_5 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1);"""
+    qt_2_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1);"""
+
+    qt_2_7 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1 + 1);"""
+    qt_2_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1 + 1);"""
+
+    // 3. single table + olap table + filter + basic + correlated + non-equal
+    qt_3_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <= t2.c2);"""
+    qt_3_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <= t2.c2);"""
+
+    qt_3_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <= t2.c2 + 1);"""
+    qt_3_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <= t2.c2 + 1);"""
+
+    qt_3_5 """select * from t1 where exists (select c1 from t2 where t1.c1 <= t2.c1);"""
+    qt_3_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 <= t2.c1);"""
+
+    qt_3_7 """select * from t1 where exists (select c1 from t2 where t1.c1 <= t2.c1 + 1);"""
+    qt_3_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 <= t2.c1 + 1);"""
+
+    // 4. single table + olap table + filter + basic + correlated + non-equal
+    qt_4_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 != t2.c2);"""
+    qt_4_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 != t2.c2);"""
+
+    qt_4_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 != t2.c2 + 1);"""
+    qt_4_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 != t2.c2 + 1);"""
+
+    qt_4_5 """select * from t1 where exists (select c1 from t2 where t1.c1 != t2.c1);"""
+    qt_4_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 != t2.c1);"""
+
+    qt_4_7 """select * from t1 where exists (select c1 from t2 where t1.c1 != t2.c1 + 1);"""
+    qt_4_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 != t2.c1 + 1);"""
+
+    // 5. single table + olap table + filter + basic + correlated + non-equal
+    qt_5_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <=> t2.c2);"""
+    qt_5_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <=> t2.c2);"""
+
+    qt_5_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <=> t2.c2 + 1);"""
+    qt_5_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <=> t2.c2 + 1);"""
+
+    qt_5_5 """select * from t1 where exists (select c1 from t2 where t1.c1 <=> t2.c1);"""
+    qt_5_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 <=> t2.c1);"""
+
+    qt_5_7 """select * from t1 where exists (select c1 from t2 where t1.c1 <=> t2.c1 + 1);"""
+    qt_5_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 <=> t2.c1 + 1);"""
+
+    // 6. single table + olap table + filter + basic + correlated + (mixed equal & non-equal)
+    qt_6_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+    qt_6_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+
+    qt_6_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 + 1 and t1.c3 < t2.c3);"""
+    qt_6_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 + 1 and t1.c3 < t2.c3);"""
+
+    qt_6_5 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3);"""
+    qt_6_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3);"""
+
+    qt_6_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 + 1 and t1.c3 < t2.c3);"""
+    qt_6_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 + 1 and t1.c3 < t2.c3);"""
+
+    // 7. single table + olap table + filter + basic + non-correlated
+    qt_7_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1);"""
+    qt_7_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1);"""
+
+    qt_7_3 """select * from t1 where exists (select c1 from t2 where t1.c1 = 1);"""
+    qt_7_4 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1);"""
+
+    // 8. single table + olap table + filter + basic + scalar
+    qt_8_1 """select * from t1 where t1.c1 in (select max(c1) from t2);"""
+    qt_8_2 """select * from t1 where t1.c1 not in (select max(c1) from t2);"""
+
+    qt_8_3 """select * from t1 where t1.c1 in (select max(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_8_4 """select * from t1 where t1.c1 not in (select max(c1) from t2 where t1.c2 = t2.c2);"""
+
+    qt_8_5 """select * from t1 where exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+    qt_8_6 """select * from t1 where not exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+
+    qt_8_7 """select * from t1 where exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1);"""
+    qt_8_8 """select * from t1 where not exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1);"""
+
+    // 9. single table + olap table + filter + basic + correlated + limit
+    qt_9_1 """select * from t1 where t1.c1 in (select c1 from t2 limit 1);"""
+    qt_9_2 """select * from t1 where t1.c1 not in (select c1 from t2 limit 1);"""
+
+    qt_9_3 """select * from t1 where t1.c1 in (select c1 from t2 limit 0);"""
+    qt_9_4 """select * from t1 where t1.c1 not in (select c1 from t2 limit 0);"""
+
+    qt_9_5 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1);"""
+    qt_9_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 1);"""
+
+    qt_9_7 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 0);"""
+    qt_9_8 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0);"""
+
+    qt_9_9 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 1);"""
+    qt_9_10 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1);"""
+
+    qt_9_11 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 0);"""
+    qt_9_12 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 0);"""
+
+    // 10. single table + olap table + filter + basic + group by / having
+    qt_10_1 """select * from t1 where t1.c1 in (select c1 from t2 group by c1 having c1 > 1);"""
+    qt_10_2 """select * from t1 where t1.c1 not in (select c1 from t2 group by c1 having c1 > 1);"""
+
+    qt_10_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1);"""
+    qt_10_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1);"""
+
+    qt_10_5 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1);"""
+    qt_10_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1);"""
+
+    qt_10_7 """select * from t1 where exists (select c1, max(c2) from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1);"""
+    qt_10_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or.groovy b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or.groovy
new file mode 100644
index 00000000000000..6b3cf46e484254
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or.groovy
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_or") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + filter + or + correlated + equal
+    qt_1_1 """select * from t1 where t1.c1 in (select c1 from t2) or t1.c2 > 1;"""
+    qt_1_2 """select * from t1 where t1.c1 not in (select c1 from t2) or t1.c2 > 1;"""
+
+    qt_1_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 > 1;"""
+    qt_1_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 > 1;"""
+
+    qt_1_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 > 1;"""
+    qt_1_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 > 1;"""
+
+    qt_1_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1) or t1.c2 > 1;"""
+    qt_1_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1) or t1.c2 > 1;"""
+
+    // 2. single table + olap table + filter + or + correlated + non-equal
+    qt_2_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) or t1.c2 > 1;"""
+    qt_2_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c2 > 1;"""
+
+    qt_2_3 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1) or t1.c2 > 1;"""
+    qt_2_4 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1) or t1.c2 > 1;"""
+
+    // 3. single table + olap table + filter + or + correlated + (mixed equal & non-equal)
+    qt_3_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c2 > 1;"""
+    qt_3_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c2 > 1;"""
+
+    qt_3_3 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3) or t1.c2 > 1;"""
+    qt_3_4 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3) or t1.c2 > 1;"""
+
+    // 4. single table + olap table + filter + or + non-correlated
+    qt_4_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 > 1;"""
+    qt_4_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 > 1;"""
+
+    qt_4_3 """select * from t1 where exists (select c1 from t2 where t1.c1 = 1) or t1.c2 > 1;"""
+    qt_4_4 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1) or t1.c2 > 1;"""
+
+    // 5. single table + olap table + filter + or + scalar
+    qt_5_1 """select * from t1 where t1.c1 in (select max(c1) from t2) or t1.c2 > 1;"""
+    qt_5_2 """select * from t1 where t1.c1 not in (select max(c1) from t2) or t1.c2 > 1;"""
+
+    qt_5_3 """select * from t1 where t1.c1 in (select max(c1) from t2 where t1.c2 = t2.c2) or t1.c2 > 1;"""
+    qt_5_4 """select * from t1 where t1.c1 not in (select max(c1) from t2 where t1.c2 = t2.c2) or t1.c2 > 1;"""
+
+    qt_5_5 """select * from t1 where exists (select max(c1) from t2 where t1.c1 = t2.c1) or t1.c2 > 1;"""
+    qt_5_6 """select * from t1 where not exists (select max(c1) from t2 where t1.c1 = t2.c1) or t1.c2 > 1;"""
+
+    qt_5_7 """select * from t1 where exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1) or t1.c2 > 1;"""
+    qt_5_8 """select * from t1 where not exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1) or t1.c2 > 1;"""
+
+    // 6. single table + olap table + filter + or + correlated + limit
+    qt_6_1 """select * from t1 where t1.c1 in (select c1 from t2 limit 1) or t1.c2 > 1;"""
+    qt_6_2 """select * from t1 where t1.c1 not in (select c1 from t2 limit 1) or t1.c2 > 1;"""
+
+    qt_6_3 """select * from t1 where t1.c1 in (select c1 from t2 limit 0) or t1.c2 > 1;"""
+    qt_6_4 """select * from t1 where t1.c1 not in (select c1 from t2 limit 0) or t1.c2 > 1;"""
+
+    qt_6_5 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1) or t1.c2 > 1;"""
+    qt_6_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 1) or t1.c2 > 1;"""
+
+    qt_6_7 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 0) or t1.c2 > 1;"""
+    qt_6_8 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0) or t1.c2 > 1;"""
+
+    qt_6_9 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) or t1.c2 > 1;"""
+    qt_6_10 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) or t1.c2 > 1;"""
+
+    qt_6_11 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) or t1.c2 > 1;"""
+    qt_6_12 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) or t1.c2 > 1;"""
+
+    // 7. single table + olap table + filter + or + group by / having
+    qt_7_1 """select * from t1 where t1.c1 in (select c1 from t2 group by c1 having c1 > 1) or t1.c2 > 1;"""
+    qt_7_2 """select * from t1 where t1.c1 not in (select c1 from t2 group by c1 having c1 > 1) or t1.c2 > 1;"""
+
+    qt_7_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1) or t1.c2 > 1;"""
+    qt_7_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1) or t1.c2 > 1;"""
+
+    qt_7_5 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1) or t1.c2 > 1;"""
+    qt_10_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1) or t1.c2 > 1;"""
+
+    qt_7_7 """select * from t1 where exists (select c1, max(c2) from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1) or t1.c2 > 1;"""
+    qt_7_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1) or t1.c2 > 1;"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or_subquery.groovy b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or_subquery.groovy
new file mode 100644
index 00000000000000..76f2b7ce7abd13
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_or_subquery.groovy
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_or_subquery") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + filter + or + correlated(=)
+    qt_1_1 """select * from t1 where t1.c1 in (select c1 from t2) or t1.c2 in (select c1 from t3);"""
+    qt_1_2 """select * from t1 where t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3);"""
+    qt_1_3 """select * from t1 where t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3);"""
+
+    qt_1_4 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_5 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_9 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 2. single table + olap table + filter + in + or + exists + correlated(=)
+    qt_2_1 """select * from t1 where t1.c1 in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_2 """select * from t1 where t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_3 """select * from t1 where t1.c1 in (select c1 from t2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_4 """select * from t1 where t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_5 """select * from t1 where t1.c1 in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_6 """select * from t1 where t1.c1 not in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_7 """select * from t1 where t1.c1 in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_8 """select * from t1 where t1.c1 not in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_9 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_10 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_11 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_12 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 3. single table + olap table + filter + or + correlated(<)
+    qt_3_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_3 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_4 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1) or exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_5 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1) or exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1) or not exists (select c1 from t3 where t1.c2 < t3.c1);"""
+
+    // 4. single table + olap table + filter + or + non-correlated
+    qt_4_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_3 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_4 """select * from t1 where exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_5 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1) or not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    // 5. single table + olap table + filter + correlated + or + non-correlated
+    qt_5_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_3 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_4 """select * from t1 where exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_5 """select * from t1 where not exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_6 """select * from t1 where not exists (select c1 from t2 where t2.c1 = 1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_uk.groovy b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_uk.groovy
new file mode 100644
index 00000000000000..c3eeb0f090d42b
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_pullup_uk.groovy
@@ -0,0 +1,237 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_uk") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    UNIQUE KEY (c1)
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    UNIQUE KEY (c1)
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    UNIQUE KEY (c1)
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/non-uk
+     */
+    // 1. single table + olap table + filter + uk + correlated + equal
+    qt_1_1 """select * from t1 where t1.c1 in (select c1 from t2);"""
+    qt_1_2 """select * from t1 where t1.c1 not in (select c1 from t2);"""
+
+    qt_1_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2);"""
+    qt_1_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_1_5 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 + 1);"""
+    qt_1_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 + 1);"""
+
+    qt_1_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1);"""
+    qt_1_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1);"""
+
+    qt_1_9 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 + 1);"""
+    qt_1_10 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 + 1);"""
+
+    // 2. single table + olap table + filter + uk + correlated + non-eqau
+    qt_2_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2);"""
+    qt_2_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2);"""
+
+    qt_2_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2 + 1);"""
+    qt_2_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2 + 1);"""
+
+    qt_2_5 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1);"""
+    qt_2_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1);"""
+
+    qt_2_7 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1 + 1);"""
+    qt_2_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1 + 1);"""
+
+    // 3. single table + olap table + filter + uk + correlated + non-equal
+    qt_3_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <= t2.c2);"""
+    qt_3_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <= t2.c2);"""
+
+    qt_3_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <= t2.c2 + 1);"""
+    qt_3_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <= t2.c2 + 1);"""
+
+    qt_3_5 """select * from t1 where exists (select c1 from t2 where t1.c1 <= t2.c1);"""
+    qt_3_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 <= t2.c1);"""
+
+    qt_3_7 """select * from t1 where exists (select c1 from t2 where t1.c1 <= t2.c1 + 1);"""
+    qt_3_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 <= t2.c1 + 1);"""
+
+    // 4. single table + olap table + filter + uk + correlated + non-equal
+    qt_4_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 != t2.c2);"""
+    qt_4_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 != t2.c2);"""
+
+    qt_4_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 != t2.c2 + 1);"""
+    qt_4_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 != t2.c2 + 1);"""
+
+    qt_4_5 """select * from t1 where exists (select c1 from t2 where t1.c1 != t2.c1);"""
+    qt_4_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 != t2.c1);"""
+
+    qt_4_7 """select * from t1 where exists (select c1 from t2 where t1.c1 != t2.c1 + 1);"""
+    qt_4_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 != t2.c1 + 1);"""
+
+    // 5. single table + olap table + filter + uk + correlated + non-equal
+    qt_5_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <=> t2.c2);"""
+    qt_5_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <=> t2.c2);"""
+
+    qt_5_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 <=> t2.c2 + 1);"""
+    qt_5_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 <=> t2.c2 + 1);"""
+
+    qt_5_5 """select * from t1 where exists (select c1 from t2 where t1.c1 <=> t2.c1);"""
+    qt_5_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 <=> t2.c1);"""
+
+    qt_5_7 """select * from t1 where exists (select c1 from t2 where t1.c1 <=> t2.c1 + 1);"""
+    qt_5_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 <=> t2.c1 + 1);"""
+
+    // 6. single table + olap table + filter + uk + correlated + (mixed equal & non-equal)
+    qt_6_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+    qt_6_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+
+    qt_6_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 + 1 and t1.c3 < t2.c3);"""
+    qt_6_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 + 1 and t1.c3 < t2.c3);"""
+
+    qt_6_5 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3);"""
+    qt_6_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3);"""
+
+    qt_6_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 + 1 and t1.c3 < t2.c3);"""
+    qt_6_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 + 1 and t1.c3 < t2.c3);"""
+
+    // 7. single table + olap table + filter + uk + non-correlated
+    qt_7_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1);"""
+    qt_7_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1);"""
+
+    qt_7_3 """select * from t1 where exists (select c1 from t2 where t1.c1 = 1);"""
+    qt_7_4 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1);"""
+
+    // 8. single table + olap table + filter + uk + scalar
+    qt_8_1 """select * from t1 where t1.c1 in (select max(c1) from t2);"""
+    qt_8_2 """select * from t1 where t1.c1 not in (select max(c1) from t2);"""
+
+    qt_8_3 """select * from t1 where t1.c1 in (select max(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_8_4 """select * from t1 where t1.c1 not in (select max(c1) from t2 where t1.c2 = t2.c2);"""
+
+    qt_8_5 """select * from t1 where exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+    qt_8_6 """select * from t1 where not exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+
+    qt_8_7 """select * from t1 where exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1);"""
+    qt_8_8 """select * from t1 where not exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1);"""
+
+    // 9. single table + olap table + filter + uk + correlated + limit
+    qt_9_1 """select * from t1 where t1.c1 in (select c1 from t2 limit 1);"""
+    qt_9_2 """select * from t1 where t1.c1 not in (select c1 from t2 limit 1);"""
+
+    qt_9_3 """select * from t1 where t1.c1 in (select c1 from t2 limit 0);"""
+    qt_9_4 """select * from t1 where t1.c1 not in (select c1 from t2 limit 0);"""
+
+    qt_9_5 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1);"""
+    qt_9_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 1);"""
+
+    qt_9_7 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 0);"""
+    qt_9_8 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0);"""
+
+    qt_9_9 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 1);"""
+    qt_9_10 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1);"""
+
+    qt_9_11 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 0);"""
+    qt_9_12 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 0);"""
+
+    // 10. single table + olap table + filter + uk + group by / having
+    qt_10_1 """select * from t1 where t1.c1 in (select c1 from t2 group by c1 having c1 > 1);"""
+    qt_10_2 """select * from t1 where t1.c1 not in (select c1 from t2 group by c1 having c1 > 1);"""
+
+    qt_10_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1);"""
+    qt_10_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1);"""
+
+    qt_10_5 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1);"""
+    qt_10_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1);"""
+
+    qt_10_7 """select * from t1 where exists (select c1, max(c2) from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1);"""
+    qt_10_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_ullup_and.groovy b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_ullup_and.groovy
new file mode 100644
index 00000000000000..f054e58fa6f740
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/basic/subquery_basic_ullup_and.groovy
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_and") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + filter + andor + correlated + equal
+    qt_1_1 """select * from t1 where t1.c1 in (select c1 from t2) and t1.c2 > 1;"""
+    qt_1_2 """select * from t1 where t1.c1 not in (select c1 from t2) and t1.c2 > 1;"""
+
+    qt_1_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 > 1;"""
+    qt_1_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 > 1;"""
+
+    qt_1_5 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 > 1;"""
+    qt_1_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 > 1;"""
+
+    qt_1_7 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1) and t1.c2 > 1;"""
+    qt_1_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1) and t1.c2 > 1;"""
+
+    // 2. single table + olap table + filter + andor + correlated + non-equal
+    qt_2_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) and t1.c2 > 1;"""
+    qt_2_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c2 > 1;"""
+
+    qt_2_3 """select * from t1 where exists (select c1 from t2 where t1.c1 < t2.c1) and t1.c2 > 1;"""
+    qt_2_4 """select * from t1 where not exists (select c1 from t2 where t1.c1 < t2.c1) and t1.c2 > 1;"""
+
+    // 6. single table + olap table + filter + andor + correlated + (mixed equal & non-equal)
+    qt_6_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c2 > 1;"""
+    qt_6_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c2 > 1;"""
+
+    qt_6_5 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3) and t1.c2 > 1;"""
+    qt_6_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 and t1.c3 < t2.c3) and t1.c2 > 1;"""
+
+    // 7. single table + olap table + filter + andor + non-correlated
+    qt_7_1 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 > 1;"""
+    qt_7_2 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 > 1;"""
+
+    qt_7_3 """select * from t1 where exists (select c1 from t2 where t1.c1 = 1) and t1.c2 > 1;"""
+    qt_7_4 """select * from t1 where not exists (select c1 from t2 where t1.c1 = 1) and t1.c2 > 1;"""
+
+    // 8. single table + olap table + filter + andor + scalar
+    qt_8_1 """select * from t1 where t1.c1 in (select max(c1) from t2) and t1.c2 > 1;"""
+    qt_8_2 """select * from t1 where t1.c1 not in (select max(c1) from t2) and t1.c2 > 1;"""
+
+    qt_8_3 """select * from t1 where t1.c1 in (select max(c1) from t2 where t1.c2 = t2.c2) and t1.c2 > 1;"""
+    qt_8_4 """select * from t1 where t1.c1 not in (select max(c1) from t2 where t1.c2 = t2.c2) and t1.c2 > 1;"""
+
+    qt_8_5 """select * from t1 where exists (select max(c1) from t2 where t1.c1 = t2.c1) and t1.c2 > 1;"""
+    qt_8_6 """select * from t1 where not exists (select max(c1) from t2 where t1.c1 = t2.c1) and t1.c2 > 1;"""
+
+    qt_8_7 """select * from t1 where exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1) and t1.c2 > 1;"""
+    qt_8_8 """select * from t1 where not exists (select max(c1), count(c2) from t2 where t1.c1 = t2.c1) and t1.c2 > 1;"""
+
+    // 9. single table + olap table + filter + andor + correlated + limit
+    qt_9_1 """select * from t1 where t1.c1 in (select c1 from t2 limit 1) and t1.c2 > 1;"""
+    qt_9_2 """select * from t1 where t1.c1 not in (select c1 from t2 limit 1) and t1.c2 > 1;"""
+
+    qt_9_3 """select * from t1 where t1.c1 in (select c1 from t2 limit 0) and t1.c2 > 1;"""
+    qt_9_4 """select * from t1 where t1.c1 not in (select c1 from t2 limit 0) and t1.c2 > 1;"""
+
+    qt_9_5 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1) and t1.c2 > 1;"""
+    qt_9_6 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 1) and t1.c2 > 1;"""
+
+    qt_9_7 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 0) and t1.c2 > 1;"""
+    qt_9_8 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0) and t1.c2 > 1;"""
+
+    qt_9_9 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) and t1.c2 > 1;"""
+    qt_9_10 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) and t1.c2 > 1;"""
+
+    qt_9_11 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) and t1.c2 > 1;"""
+    qt_9_12 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) and t1.c2 > 1;"""
+
+    // 10. single table + olap table + filter + andor + group by / having
+    qt_10_1 """select * from t1 where t1.c1 in (select c1 from t2 group by c1 having c1 > 1) and t1.c2 > 1;"""
+    qt_10_2 """select * from t1 where t1.c1 not in (select c1 from t2 group by c1 having c1 > 1) and t1.c2 > 1;"""
+
+    qt_10_3 """select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1) and t1.c2 > 1;"""
+    qt_10_4 """select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 group by c1 having c1 > 1) and t1.c2 > 1;"""
+
+    qt_10_5 """select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1) and t1.c2 > 1;"""
+    qt_10_6 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having c1 > 1) and t1.c2 > 1;"""
+
+    qt_10_7 """select * from t1 where exists (select c1, max(c2) from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1) and t1.c2 > 1;"""
+    qt_10_8 """select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1 group by c1 having max(c2) > 1) and t1.c2 > 1;"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_dml.groovy b/regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_dml.groovy
new file mode 100644
index 00000000000000..faf8ca9102738e
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_dml.groovy
@@ -0,0 +1,126 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_dml") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS target;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS target(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + insert into + basic + correlated + equal
+    qt_1_1 """insert into target select * from t1 where t1.c1 in (select c1 from t2);"""
+    qt_1_2 """insert into target select * from t1 where t1.c1 not in (select c1 from t2);"""
+
+    qt_1_3 """insert into target select * from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2);"""
+    qt_1_4 """insert into target select * from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_1_5 """insert into target select * from t1 where exists (select c1 from t2 where t1.c1 = t2.c1);"""
+    qt_1_6 """insert into target select * from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1);"""
+
+    // 2. single table + olap table + update + basic + correlated + non-equal
+    qt_2_1 """update t1 set t1.c1 = 1 where t1.c1 in (select c1 from t2);"""
+    qt_2_2 """update t1 set t1.c1 = 1 where t1.c1 not in (select c1 from t2);"""
+
+    qt_2_3 """update t1 set t1.c1 = 1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2);"""
+    qt_2_4 """update t1 set t1.c1 = 1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_2_5 """update t1 set t1.c1 = 1 where exists (select c1 from t2 where t1.c1 = t2.c1);"""
+    qt_2_6 """update t1 set t1.c1 = 1 where not exists (select c1 from t2 where t1.c1 = t2.c1);"""
+
+    // 3. single table + olap table + delete + basic + correlated + non-equal
+    qt_3_1 """delete from t1 where t1.c1 in (select c1 from t2);"""
+    qt_3_2 """delete from t1 where t1.c1 not in (select c1 from t2);"""
+
+    qt_3_3 """delete from t1 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2);"""
+    qt_3_4 """delete from t1 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_3_5 """delete from t1 where exists (select c1 from t2 where t1.c1 = t2.c1);"""
+    qt_3_6 """delete from t1 where not exists (select c1 from t2 where t1.c1 = t2.c1);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_misc.groovy b/regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_misc.groovy
new file mode 100644
index 00000000000000..0602210f391873
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/misc/subquery_misc_pullup_misc.groovy
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_misc_pullup_misc") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + filter + count + correlated + equal
+    qt_1_1 """select * from t1 where t1.c1 in (select count(c1) from t2);"""
+    qt_1_2 """select * from t1 where t1.c1 not in (select count(c1) from t2);"""
+
+    qt_1_3 """select * from t1 where t1.c1 in (select count(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_1_4 """select * from t1 where t1.c1 not in (select count(c1) from t2 where t1.c2 = t2.c2);"""
+
+    qt_1_5 """select * from t1 where exists (select count(c1) from t2 where t1.c1 = t2.c1);"""
+    qt_1_6 """select * from t1 where not exists (select count(c1) from t2 where t1.c1 = t2.c1);"""
+
+    qt_1_7 """select * from t1 where t1.c1 <= (select count(c1) from t2);"""
+    qt_1_8 """select * from t1 where t1.c1 > (select count(c1) from t2);"""
+
+    qt_1_9 """select * from t1 where t1.c1 <= (select count(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_1_10 """select * from t1 where t1.c1 > (select count(c1) from t2 where t1.c2 = t2.c2);"""
+
+    // 2. single table + olap table + filter + correlated + having
+    qt_2_1 """select * from t1 where t1.c1 in (select c1 from t2 group by c1 having t2.c1 = t1.c2);"""
+    qt_2_2 """select * from t1 where t1.c1 not in (select c1 from t2 group by c1 having t2.c1 = t1.c2);"""
+
+    qt_2_3 """select * from t1 where t1.c1 in (select c1 from t2 group by c1 having t2.c1 > t1.c2);"""
+    qt_2_4 """select * from t1 where t1.c1 not in (select c1 from t2 group by c1 having t2.c1 < t1.c2);"""
+
+    qt_2_5 """select * from t1 where exists (select c1 from t2 group by c1 having t2.c1 = t1.c2);"""
+    qt_2_6 """select * from t1 where not exists (select c1 from t2 group by c1 having t2.c1 = t1.c2);"""
+
+    qt_2_7 """select * from t1 where exists (select c1 from t2 group by c1 having t2.c1 > t1.c2);"""
+    qt_2_8 """select * from t1 where not exists (select c1 from t2 group by c1 having t2.c1 < t1.c2);"""
+
+    qt_2_9 """select * from t1 where t1.c1 > (select c1 from t2 group by c1 having t2.c1 = t1.c2);"""
+    qt_2_10 """select * from t1 where t1.c1 <= (select c1 from t2 group by c1 having t2.c1 = t1.c2);"""
+
+    // 3. single table + olap table + filter + const
+    qt_3_1 """select * from t1 where 1 in (select count(c1) from t2);"""
+    qt_3_2 """select * from t1 where 1 not in (select max(c1) from t2);"""
+    qt_3_3 """select * from t1 where 1 not in (select c1 from t2);"""
+
+    qt_3_4 """select * from t1 where 1 in (select count(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_3_5 """select * from t1 where 1 not in (select max(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_3_6 """select * from t1 where 1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_3_7 """select * from t1 where 1 <= (select count(c1) from t2);"""
+    qt_3_8 """select * from t1 where 1 > (select max(c1) from t2);"""
+    qt_3_9 """select * from t1 where 1 > (select c1 from t2);"""
+
+    qt_3_10 """select * from t1 where 1 <= (select count(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_3_11 """select * from t1 where 1 > (select max(c1) from t2 where t1.c2 = t2.c2);"""
+    qt_3_12 """select * from t1 where 1 > (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    // 4. single table + view + correlated
+    qt_4_1 """select * from (select c1 from t1 where t1.c2 = 1) t1 where t1.c1 in (select count(c1) from t2);"""
+    qt_4_2 """select * from (select t1.c1 from t1, t2 where t1.c1 = t2.c1) t1 where t1.c1 in (select count(c1) from t2);"""
+    qt_4_3 """select * from (select t1.c1 from t1 where t1.c2 = 1 union all select t2.c1 from t2 where t2.c2 = 2) t1 where t1.c1 in (select count(c1) from t2);"""
+
+    qt_4_4 """select * from (select c1 from t1 where t1.c2 = 1) t1 where t1.c1 not in (select count(c1) from t2);"""
+    qt_4_5 """select * from (select t1.c1 from t1, t2 where t1.c1 = t2.c1) t1 where t1.c1 not in (select count(c1) from t2);"""
+    qt_4_6 """select * from (select t1.c1 from t1 where t1.c2 = 1 union all select t2.c1 from t2 where t2.c2 = 2) t1 where t1.c1 not in (select count(c1) from t2);"""
+
+    qt_4_7 """select * from (select c1 from t1 where t1.c2 = 1) t1 where exists (select count(c1) from t2);"""
+    qt_4_8 """select * from (select t1.c1 from t1, t2 where t1.c1 = t2.c1) t1 where exists (select count(c1) from t2);"""
+    qt_4_9 """select * from (select t1.c1 from t1 where t1.c2 = 1 union all select t2.c1 from t2 where t2.c2 = 2) t1 where not exists (select count(c1) from t2);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and.groovy b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and.groovy
new file mode 100644
index 00000000000000..51dfa5022323d3
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and.groovy
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_multitable_pullup_and") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. multi table + olap table + filter/on condition + and + correlated + equal
+    qt_1_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2) and t1.c1 > 1;"""
+    qt_1_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2) and t1.c1 > 1;"""
+
+    qt_1_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2) and t1.c1 > 1;"""
+    qt_1_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2) and t1.c1 > 1;"""
+
+    qt_1_5 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c1 > 1;"""
+    qt_1_6 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c1 > 1;"""
+
+    qt_1_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c1 > 1;"""
+    qt_1_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c1 > 1;"""
+
+    qt_1_9 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+    qt_1_10 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+
+    qt_1_11 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+    qt_1_12 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+
+    // 2. multi table + olap table + filter/on condition + and + correlated + non-equal
+    qt_2_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) and t1.c1 > 1;"""
+    qt_2_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c1 > 1;"""
+
+    qt_2_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) and t1.c1 > 1;"""
+    qt_2_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c1 > 1;"""
+
+    qt_2_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 < t2.c1) and t1.c1 > 1;"""
+    qt_2_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 < t2.c1) and t1.c1 > 1;"""
+
+    qt_2_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 < t2.c1) and t1.c1 > 1;"""
+    qt_2_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 < t2.c1) and t1.c1 > 1;"""
+
+    // 3. multi table + olap table + filter/on condition + and + correlated + (mixed equal & non-equal)
+    qt_3_1 """select * from t1, t3 where t1.c4 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+    qt_3_2 """select * from t1, t3 where t1.c4 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+
+    qt_3_3 """select * from t1 left join t3 on t1.c4 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+    qt_3_4 """select * from t1 left join t3 on t1.c4 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+
+    qt_3_5 """select * from t1, t3 where t1.c4 = t3.c3 and exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+    qt_3_6 """select * from t1, t3 where t1.c4 = t3.c3 and not exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+
+    qt_3_7 """select * from t1 left join t3 on t1.c4 = t3.c3 and exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+    qt_3_8 """select * from t1 left join t3 on t1.c4 = t3.c3 and not exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) and t1.c1 > 1;"""
+
+    // 4. multi table + olap table + filter/on condition + and + non-correlated
+    qt_4_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+    qt_4_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+
+    qt_4_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+    qt_4_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+
+    qt_4_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+    qt_4_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+
+    qt_4_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+    qt_4_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c2 = 1) and t1.c1 > 1;"""
+
+    // 5. multi table + olap table + filter/on condition + and + scalar
+    qt_5_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select max(c1) from t2) and t1.c1 > 1;"""
+    qt_5_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select max(c1) from t2) and t1.c1 > 1;"""
+
+    qt_5_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select max(c1) from t2) and t1.c1 > 1;"""
+    qt_5_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select max(c1) from t2) and t1.c1 > 1;"""
+
+    qt_5_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select max(c1) from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+    qt_5_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select max(c1) from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+
+    qt_5_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select max(c1) from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+    qt_5_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select max(c1) from t2 where t1.c1 = t2.c1) and t1.c1 > 1;"""
+
+    // 6. multi table + olap table + filter/on condition + and + correlated + limit
+    qt_6_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 limit 1) and t1.c1 > 1;"""
+    qt_6_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 limit 0) and t1.c1 > 1;"""
+
+    qt_6_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 limit 1) and t1.c1 > 1;"""
+    qt_6_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 limit 0) and t1.c1 > 1;"""
+
+    qt_6_5 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1) and t1.c1 > 1;"""
+    qt_6_6 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0) and t1.c1 > 1;"""
+
+    qt_6_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1) and t1.c1 > 1;"""
+    qt_6_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0) and t1.c1 > 1;"""
+
+    qt_6_9 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) and t1.c1 > 1;"""
+    qt_6_10 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) and t1.c1 > 1;"""
+
+    qt_6_11 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) and t1.c1 > 1;"""
+    qt_6_12 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) and t1.c1 > 1;"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and_subquery.groovy b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and_subquery.groovy
new file mode 100644
index 00000000000000..5c1318f8c9f3e0
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_and_subquery.groovy
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_multitable_pullup_and_subquery") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+    sql """DROP TABLE IF EXISTS t4;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t4(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t4 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t4 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t4 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t4 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. multi table + olap table + filter/on condition + and + correlated + equal
+    qt_1_1 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_2 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_3 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3);"""
+
+    qt_1_4 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_5 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_6 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3);"""
+
+    qt_1_7 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_8 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_9 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3);"""
+
+    qt_1_10 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_11 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_12 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_13 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_14 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_15 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_16 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_17 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_18 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_19 """select * from t1, t4 where t1.c4 = t3.c4 and exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_20 """select * from t1, t4 where t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_21 """select * from t1, t4 where t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_1_22 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_23 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_24 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_1_25 """select * from t1 left join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_26 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = t2.c1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_27 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = t2.c1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 2. multi table + olap table + filter/on condition + in + and + exists + correlated + equal
+    qt_2_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_4 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_7 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_8 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_10 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_11 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_12 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_13 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_14 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_15 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_16 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_17 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_18 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_19 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_20 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_21 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_22 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_23 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_24 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_25 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_26 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_27 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_28 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_29 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_30 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_31 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_32 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_33 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_34 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_35 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_36 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 3. multi table + olap table + filter/on condition + and + correlated + non-equal
+    qt_3_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_4 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_7 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_8 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) and t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_10 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t1.c1 < t2.c1) and exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_11 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 < t2.c1) and exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_12 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 < t2.c1) and not exists (select c1 from t3 where t1.c2 < t3.c1);"""
+
+    // 4. multi table + olap table + filter/on condition + and + non-correlated
+    qt_4_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_4 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_7 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_8 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_10 """select * from t1 inner join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_11 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_12 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) and not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    qt_4_13 """select * from t1 left join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_14 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_15 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) and not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    qt_4_16 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_17 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = 1) and exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_18 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = 1) and not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    // 5. multi table + olap table + filter/on condition + correlated + and + non-correlated
+    qt_5_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_4 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_7 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_8 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_10 """select * from t1 inner join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_11 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_12 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_5_13 """select * from t1 left join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_14 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_15 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_5_16 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_17 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t2.c1 = 1) and exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_18 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t2.c1 = 1) and not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_basic.groovy b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_basic.groovy
new file mode 100644
index 00000000000000..5e9df4ffc796b1
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_basic.groovy
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_multitable_pullup_basic") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. multi table + olap table + filter/on condition + basic + correlated + equal
+    qt_1_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2);"""
+    qt_1_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2);"""
+
+    qt_1_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2);"""
+    qt_1_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2);"""
+
+    qt_1_5 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2);"""
+    qt_1_6 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_1_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2);"""
+    qt_1_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2);"""
+
+    qt_1_9 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1);"""
+    qt_1_10 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1);"""
+
+    qt_1_11 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1);"""
+    qt_1_12 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1);"""
+
+    // 2. multi table + olap table + filter/on condition + basic + correlated + non-equal
+    qt_2_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2);"""
+    qt_2_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2);"""
+
+    qt_2_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2);"""
+    qt_2_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2);"""
+
+    qt_2_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 < t2.c1);"""
+    qt_2_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 < t2.c1);"""
+
+    qt_2_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 < t2.c1);"""
+    qt_2_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 < t2.c1);"""
+
+    // 3. multi table + olap table + filter/on condition + basic + correlated + (mixed equal & non-equal)
+    qt_3_1 """select * from t1, t3 where t1.c4 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+    qt_3_2 """select * from t1, t3 where t1.c4 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+
+    qt_3_3 """select * from t1 left join t3 on t1.c4 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+    qt_3_4 """select * from t1 left join t3 on t1.c4 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+
+    qt_3_5 """select * from t1, t3 where t1.c4 = t3.c3 and exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+    qt_3_6 """select * from t1, t3 where t1.c4 = t3.c3 and not exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+
+    qt_3_7 """select * from t1 left join t3 on t1.c4 = t3.c3 and exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+    qt_3_8 """select * from t1 left join t3 on t1.c4 = t3.c3 and not exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3);"""
+
+    // 4. multi table + olap table + filter/on condition + basic + non-correlated
+    qt_4_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = 1);"""
+    qt_4_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = 1);"""
+
+    qt_4_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = 1);"""
+    qt_4_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = 1);"""
+
+    qt_4_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c2 = 1);"""
+    qt_4_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c2 = 1);"""
+
+    qt_4_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c2 = 1);"""
+    qt_4_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c2 = 1);"""
+
+    // 5. multi table + olap table + filter/on condition + basic + scalar
+    qt_5_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select max(c1) from t2);"""
+    qt_5_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select max(c1) from t2);"""
+
+    qt_5_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select max(c1) from t2);"""
+    qt_5_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select max(c1) from t2);"""
+
+    qt_5_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+    qt_5_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+
+    qt_5_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+    qt_5_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select max(c1) from t2 where t1.c1 = t2.c1);"""
+
+    // 6. multi table + olap table + filter/on condition + basic + correlated + limit
+    qt_6_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 limit 1);"""
+    qt_6_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 limit 0);"""
+
+    qt_6_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 limit 1);"""
+    qt_6_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 limit 0);"""
+
+    qt_6_5 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1);"""
+    qt_6_6 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0);"""
+
+    qt_6_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1);"""
+    qt_6_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0);"""
+
+    qt_6_9 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1 limit 0);"""
+    qt_6_10 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1);"""
+
+    qt_6_11 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1 limit 0);"""
+    qt_6_12 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or.groovy b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or.groovy
new file mode 100644
index 00000000000000..67168365f01483
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or.groovy
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_multitable_pullup_or") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. multi table + olap table + filter/on condition + or + correlated + equal
+    qt_1_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2) or t1.c1 > 1;"""
+    qt_1_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2) or t1.c1 > 1;"""
+
+    qt_1_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2) or t1.c1 > 1;"""
+    qt_1_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2) or t1.c1 > 1;"""
+
+    qt_1_5 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or t1.c1 > 1;"""
+    qt_1_6 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c1 > 1;"""
+
+    qt_1_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or t1.c1 > 1;"""
+    qt_1_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c1 > 1;"""
+
+    qt_1_9 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+    qt_1_10 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+
+    qt_1_11 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+    qt_1_12 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+
+    // 2. multi table + olap table + filter/on condition + or + correlated + non-equal
+    qt_2_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) or t1.c1 > 1;"""
+    qt_2_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c1 > 1;"""
+
+    qt_2_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) or t1.c1 > 1;"""
+    qt_2_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c1 > 1;"""
+
+    qt_2_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 < t2.c1) or t1.c1 > 1;"""
+    qt_2_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 < t2.c1) or t1.c1 > 1;"""
+
+    qt_2_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 < t2.c1) or t1.c1 > 1;"""
+    qt_2_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 < t2.c1) or t1.c1 > 1;"""
+
+    // 3. multi table + olap table + filter/on condition + or + correlated + (mixed equal & non-equal)
+    qt_3_1 """select * from t1, t3 where t1.c4 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+    qt_3_2 """select * from t1, t3 where t1.c4 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+
+    qt_3_3 """select * from t1 left join t3 on t1.c4 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+    qt_3_4 """select * from t1 left join t3 on t1.c4 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+
+    qt_3_5 """select * from t1, t3 where t1.c4 = t3.c3 and exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+    qt_3_6 """select * from t1, t3 where t1.c4 = t3.c3 and not exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+
+    qt_3_7 """select * from t1 left join t3 on t1.c4 = t3.c3 and exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+    qt_3_8 """select * from t1 left join t3 on t1.c4 = t3.c3 and not exists (select c1 from t2 where t1.c2 = t2.c2 and t1.c3 < t2.c3) or t1.c1 > 1;"""
+
+    // 4. multi table + olap table + filter/on condition + or + non-correlated
+    qt_4_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+    qt_4_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+
+    qt_4_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+    qt_4_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+
+    qt_4_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+    qt_4_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+
+    qt_4_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+    qt_4_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c2 = 1) or t1.c1 > 1;"""
+
+    // 5. multi table + olap table + filter/on condition + or + scalar
+    qt_5_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select max(c1) from t2) or t1.c1 > 1;"""
+    qt_5_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select max(c1) from t2) or t1.c1 > 1;"""
+
+    qt_5_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select max(c1) from t2) or t1.c1 > 1;"""
+    qt_5_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select max(c1) from t2) or t1.c1 > 1;"""
+
+    qt_5_5 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select max(c1) from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+    qt_5_6 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select max(c1) from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+
+    qt_5_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select max(c1) from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+    qt_5_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select max(c1) from t2 where t1.c1 = t2.c1) or t1.c1 > 1;"""
+
+    // 6. multi table + olap table + filter/on condition + or + correlated + limit
+    qt_6_1 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 limit 1) or t1.c1 > 1;"""
+    qt_6_2 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 limit 0) or t1.c1 > 1;"""
+
+    qt_6_3 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 limit 1) or t1.c1 > 1;"""
+    qt_6_4 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 limit 0) or t1.c1 > 1;"""
+
+    qt_6_5 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1) or t1.c1 > 1;"""
+    qt_6_6 """select * from t1, t3 where t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0) or t1.c1 > 1;"""
+
+    qt_6_7 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2 limit 1) or t1.c1 > 1;"""
+    qt_6_8 """select * from t1 left join t3 on t1.c3 = t3.c3 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2 limit 0) or t1.c1 > 1;"""
+
+    qt_6_9 """select * from t1, t3 where t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) or t1.c1 > 1;"""
+    qt_6_10 """select * from t1, t3 where t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) or t1.c1 > 1;"""
+
+    qt_6_11 """select * from t1 left join t3 on t1.c3 = t3.c3 and exists (select c1 from t2 where t1.c1 = t2.c1 limit 0) or t1.c1 > 1;"""
+    qt_6_12 """select * from t1 left join t3 on t1.c3 = t3.c3 and not exists (select c1 from t2 where t1.c1 = t2.c1 limit 1) or t1.c1 > 1;"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or_subquery.groovy b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or_subquery.groovy
new file mode 100644
index 00000000000000..8a41ecd25109c5
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/multitable/subquery_multitable_pullup_or_subquery.groovy
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_multitable_pullup_or_subquery") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+    sql """DROP TABLE IF EXISTS t4;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t4(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t4 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t4 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t4 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t4 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. multi table + olap table + filter/on condition + or + correlated + equal
+    qt_1_1 """select * from t1, t4 where t1.c4 = t3.c4 or t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_2 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3);"""
+    qt_1_3 """select * from t1, t4 where t1.c4 = t3.c4 or t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3);"""
+
+    qt_1_4 """select * from t1 left join t4 on t1.c4 = t3.c4 or t1.c1 in (select c1 from t2) and t1.c2 in (select c1 from t3);"""
+    qt_1_5 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3);"""
+    qt_1_6 """select * from t1 left join t4 on t1.c4 = t3.c4 or t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3);"""
+
+    qt_1_7 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or t1.c2 in (select c1 from t3);"""
+    qt_1_8 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3);"""
+    qt_1_9 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3);"""
+
+    qt_1_10 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_11 """select * from t1, t4 where t1.c4 = t3.c4 or t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_12 """select * from t1, t4 where t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_13 """select * from t1 left join t4 on t1.c4 = t3.c4 or t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) and t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_14 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_15 """select * from t1 left join t4 on t1.c4 = t3.c4 or t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_16 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_17 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_1_18 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_1_19 """select * from t1, t4 where t1.c4 = t3.c4 and exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_20 """select * from t1, t4 where t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_21 """select * from t1, t4 where t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_1_22 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_23 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_24 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = t2.c1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_1_25 """select * from t1 left join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_26 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = t2.c1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_1_27 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = t2.c1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 2. multi table + olap table + filter/on condition + in + or + exists + correlated + equal
+    qt_2_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) and t1.c2 or in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_4 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_7 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_8 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_10 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_11 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_2_12 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) or t1.c2 not in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_2_13 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_14 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_15 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_16 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_17 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_18 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_19 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_20 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_21 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_22 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_23 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_24 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_25 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_26 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_27 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_28 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_29 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_30 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_31 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_32 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_2_33 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_34 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_35 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_2_36 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    // 3. multi table + olap table + filter/on condition + or + correlated + non-equal
+    qt_3_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_4 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_7 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_8 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 in (select c1 from t3 where t1.c4 < t3.c2);"""
+    qt_3_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) or t1.c3 not in (select c1 from t3 where t1.c4 < t3.c2);"""
+
+    qt_3_10 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t1.c1 < t2.c1) or exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_11 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 < t2.c1) or exists (select c1 from t3 where t1.c2 < t3.c1);"""
+    qt_3_12 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 < t2.c1) or not exists (select c1 from t3 where t1.c2 < t3.c1);"""
+
+    // 4. multi table + olap table + filter/on condition + or + non-correlated
+    qt_4_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_4 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_7 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_8 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t3.c2 = 1);"""
+    qt_4_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 not in (select c1 from t3 where t3.c2 = 1);"""
+
+    qt_4_10 """select * from t1 inner join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_11 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_12 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) or not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    qt_4_13 """select * from t1 left join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_14 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_15 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t1.c1 = 1) or not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    qt_4_16 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_17 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = 1) or exists (select c1 from t3 where t3.c1 = 1);"""
+    qt_4_18 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t1.c1 = 1) or not exists (select c1 from t3 where t3.c1 = 1);"""
+
+    // 5. multi table + olap table + filter/on condition + correlated + or + non-correlated
+    qt_5_1 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_2 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_3 """select * from t1 inner join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_4 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_5 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_6 """select * from t1 left join t4 on t1.c4 = t3.c4 where t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_7 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_8 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+    qt_5_9 """select * from t1 left join t4 on t1.c4 = t3.c4 and t1.c1 not in (select c1 from t2 where t1.c2 = 1) or t1.c2 in (select c1 from t3 where t1.c3 = t3.c2);"""
+
+    qt_5_10 """select * from t1 inner join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_11 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_12 """select * from t1 inner join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_5_13 """select * from t1 left join t4 on t1.c4 = t3.c4 where exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_14 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_15 """select * from t1 left join t4 on t1.c4 = t3.c4 where not exists (select c1 from t2 where t2.c1 = 1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+
+    qt_5_16 """select * from t1 left join t4 on t1.c4 = t3.c4 and exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_17 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t2.c1 = 1) or exists (select c1 from t3 where t1.c2 = t3.c2);"""
+    qt_5_18 """select * from t1 left join t4 on t1.c4 = t3.c4 and not exists (select c1 from t2 where t2.c1 = 1) or not exists (select c1 from t3 where t1.c2 = t3.c2);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_groupby.groovy b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_groupby.groovy
new file mode 100644
index 00000000000000..3323caa364cfde
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_groupby.groovy
@@ -0,0 +1,273 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_orderby") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + group by + correlated + equal
+    qt_1_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c1 from t2);"""
+    qt_1_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c2 from t2);"""
+
+    qt_1_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c1 from t2);"""
+    qt_1_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c2 from t2);"""
+
+    qt_1_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c1 from t2);"""
+    qt_1_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c2 from t2);"""
+
+    qt_1_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c1 from t2);"""
+    qt_1_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c2 from t2);"""
+
+    qt_1_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c1 from t3);"""
+    qt_1_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c2 from t3);"""
+
+    qt_1_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c1 from t3);"""
+    qt_1_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c2 from t3);"""
+
+    qt_1_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c1 from t3);"""
+    qt_1_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c2 from t3);"""
+
+    qt_1_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c1 from t3);"""
+    qt_1_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c2 from t3);"""
+
+    qt_1_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t2);"""
+    qt_1_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t2);"""
+
+    qt_1_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t2);"""
+    qt_1_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t2);"""
+
+    qt_1_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t3);"""
+    qt_1_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t3);"""
+
+    qt_1_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t3);"""
+    qt_1_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t3);"""
+
+    qt_1_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t3);"""
+    qt_1_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t3);"""
+
+    qt_1_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t3);"""
+    qt_1_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t3);"""
+
+    // 2. single table + olap table + group by + correlated + explicit equal
+    qt_2_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    // 3. single table + olap table + group by + correlated + explicit non-equal
+    qt_3_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    // 4. single table + olap table + group by + correlated + scalar group by
+    qt_4_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c1 not in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 not in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by not exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by not exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_having.groovy b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_having.groovy
new file mode 100644
index 00000000000000..63d58b1ccb2a7a
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_having.groovy
@@ -0,0 +1,273 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_having") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + group by + having + correlated + equal
+    qt_1_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c1 from t2);"""
+    qt_1_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c2 from t2);"""
+
+    qt_1_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c1 from t2);"""
+    qt_1_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c2 from t2);"""
+
+    qt_1_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c1 from t2);"""
+    qt_1_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c2 from t2);"""
+
+    qt_1_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c1 from t2);"""
+    qt_1_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c2 from t2);"""
+
+    qt_1_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c1 from t3);"""
+    qt_1_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c2 from t3);"""
+
+    qt_1_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c1 from t3);"""
+    qt_1_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c2 from t3);"""
+
+    qt_1_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c1 from t3);"""
+    qt_1_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c2 from t3);"""
+
+    qt_1_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c1 from t3);"""
+    qt_1_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c2 from t3);"""
+
+    qt_1_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t2);"""
+    qt_1_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t2);"""
+
+    qt_1_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t2);"""
+    qt_1_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t2);"""
+
+    qt_1_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t3);"""
+    qt_1_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t3);"""
+
+    qt_1_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t3);"""
+    qt_1_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t3);"""
+
+    qt_1_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t3);"""
+    qt_1_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t3);"""
+
+    qt_1_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t3);"""
+    qt_1_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t3);"""
+
+    // 2. single table + olap table + group by + having + correlated + explicit equal
+    qt_2_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    // 3. single table + olap table + group by + having + correlated + explicit non-equal
+    qt_3_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    // 4. single table + olap table + group by + having + correlated + scalar group by
+    qt_4_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c1 not in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having t1.c2 not in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) group by t1.c2 having not exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_orderby.groovy b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_orderby.groovy
new file mode 100644
index 00000000000000..97a8227dd81535
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_orderby.groovy
@@ -0,0 +1,273 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_orderby") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + order by + correlated + equal
+    qt_1_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c1 from t2);"""
+    qt_1_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c2 from t2);"""
+
+    qt_1_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c1 from t2);"""
+    qt_1_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c2 from t2);"""
+
+    qt_1_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c1 from t2);"""
+    qt_1_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c2 from t2);"""
+
+    qt_1_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c1 from t2);"""
+    qt_1_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c2 from t2);"""
+
+    qt_1_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c1 from t3);"""
+    qt_1_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c2 from t3);"""
+
+    qt_1_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c1 from t3);"""
+    qt_1_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c2 from t3);"""
+
+    qt_1_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c1 from t3);"""
+    qt_1_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c2 from t3);"""
+
+    qt_1_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c1 from t3);"""
+    qt_1_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c2 from t3);"""
+
+    qt_1_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t2);"""
+    qt_1_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t2);"""
+
+    qt_1_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t2);"""
+    qt_1_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t2);"""
+
+    qt_1_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t3);"""
+    qt_1_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t3);"""
+
+    qt_1_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t3);"""
+    qt_1_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t3);"""
+
+    qt_1_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t3);"""
+    qt_1_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t3);"""
+
+    qt_1_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t3);"""
+    qt_1_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t3);"""
+
+    // 2. single table + olap table + order by + correlated + explicit equal
+    qt_2_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t2 where t1.c4 = t2.c4);"""
+    qt_2_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t2 where t1.c4 = t2.c4);"""
+
+    qt_2_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    qt_2_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t3 where t1.c4 = t3.c4);"""
+    qt_2_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t3 where t1.c4 = t3.c4);"""
+
+    // 3. single table + olap table + order by + correlated + explicit non-equal
+    qt_3_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t2 where t1.c4 < t2.c4);"""
+    qt_3_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t2 where t1.c4 < t2.c4);"""
+
+    qt_3_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    qt_3_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c1 from t3 where t1.c4 < t3.c4);"""
+    qt_3_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select c2 from t3 where t1.c4 < t3.c4);"""
+
+    // 4. single table + olap table + order by + correlated + scalar order by
+    qt_4_1 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_2 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_3 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_4 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_5 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_6 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_7 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_8 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_9 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_10 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_11 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_12 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by t1.c1 not in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_13 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_14 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_15 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_16 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by t1.c2 not in (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_17 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_18 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_19 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select max(c1) from t2 where t1.c4 = t2.c4);"""
+    qt_4_20 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select count(c2) from t2 where t1.c4 = t2.c4);"""
+
+    qt_4_21 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_22 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_23 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_24 """select t1.c1 from t1 where t1.c1 in (select c1 from t2) order by not exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_25 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_26 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+
+    qt_4_27 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select max(c1) from t3 where t1.c4 = t3.c4);"""
+    qt_4_28 """select t1.c2 from t1 where t1.c1 in (select c1 from t2) order by not exists (select count(c2) from t3 where t1.c4 = t3.c4);"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_selectlist.groovy b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_selectlist.groovy
new file mode 100644
index 00000000000000..9ab597472106ea
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_selectlist.groovy
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_selectlist") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + select list + basic + correlated + equal
+    qt_1_1 """select t1.c1 in (select c1 from t2) from t1;"""
+    qt_1_2 """select t1.c1 not in (select c1 from t2) from t1"""
+
+    qt_1_3 """select t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) from t1;"""
+    qt_1_4 """select t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) from t1;"""
+
+    qt_1_5 """select exists (select c1 from t2 where t1.c1 = t2.c1) from t1;"""
+    qt_1_6 """select not exists (select c1 from t2 where t1.c1 = t2.c1) from t1;"""
+
+    // 2. single table + olap table + select list + basic + correlated + non-equal
+    qt_2_1 """select t1.c1 in (select c1 from t2 where t1.c2 < t2.c2) from t1;"""
+    qt_2_2 """select t1.c1 not in (select c1 from t2 where t1.c2 < t2.c2) from t1;"""
+
+    qt_2_3 """select exists (select c1 from t2 where t1.c1 < t2.c1) from t1;"""
+    qt_2_4 """select not exists (select c1 from t2 where t1.c1 < t2.c1) from t1;"""
+
+    // 3. single table + olap table + select list + basic + correlated + filter
+    qt_3_1 """select t1.c1 in (select c1 from t2) from t1 where t1.c2 in (select c1 from t3);"""
+    qt_3_2 """select t1.c1 not in (select c1 from t2) from t1 where t1.c2 not in (select c1 from t3)"""
+
+    qt_3_3 """select t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) from t1 where t1.c2 in (select c1 from t3);"""
+    qt_3_4 """select t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) from t1 where t1.c2 not in (select c1 from t3);"""
+
+    qt_3_5 """select exists (select c1 from t2 where t1.c1 = t2.c1) from t1 where t1.c2 in (select c1 from t3);"""
+    qt_3_6 """select not exists (select c1 from t2 where t1.c1 = t2.c1) from t1 where t1.c2 not in (select c1 from t3);"""
+
+    // 4. single table + olap table + select list + case when
+    qt_4_1 """select case when t1.c1 in (select c1 from t2) then c1 else c2 end from t1;"""
+    qt_4_2 """select case when t1.c1 not in (select c1 from t2) then c1 else c2 end  from t1"""
+
+    qt_4_3 """select case when t1.c1 in (select c1 from t2 where t1.c2 = t2.c2) then c1 else c2 end from t1;"""
+    qt_4_4 """select case when t1.c1 not in (select c1 from t2 where t1.c2 = t2.c2) then c1 else c2 end from t1;"""
+
+    qt_4_5 """select case when exists (select c1 from t2 where t1.c1 = t2.c1) then c1 else c2 end from t1;"""
+    qt_4_6 """select case when  not exists (select c1 from t2 where t1.c1 = t2.c1) then c1 else c2 end from t1;"""
+}
+
diff --git a/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_winfunc.groovy b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_winfunc.groovy
new file mode 100644
index 00000000000000..d8e7e65ac8c55d
--- /dev/null
+++ b/regression-test/suites/nereids_rules_p0/subquery/topop/subquery_topop_pullup_winfunc.groovy
@@ -0,0 +1,185 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("subquery_basic_pullup_winfunc") {
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+
+    sql """DROP TABLE IF EXISTS t1;"""
+    sql """DROP TABLE IF EXISTS t2;"""
+    sql """DROP TABLE IF EXISTS t3;"""
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t1(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t2(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """
+    CREATE TABLE IF NOT EXISTS t3(
+      `c1` int(11) NULL,
+      `c2` int(11) NULL,
+      `c3` int(11) NULL,
+      `c4` int(11) NULL
+    ) ENGINE = OLAP
+    DISTRIBUTED BY HASH(c1) BUCKETS 4
+    PROPERTIES (
+      "replication_allocation" = "tag.location.default: 1"
+    );
+    """
+
+    sql """INSERT INTO t1 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t1 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t1 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t1 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t2 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t2 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t2 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t2 VALUES (3,3,3,3);"""
+
+    sql """INSERT INTO t3 VALUES (null,null,null,null);"""
+    sql """INSERT INTO t3 VALUES (1,1,1,1);"""
+    sql """INSERT INTO t3 VALUES (2,2,2,2);"""
+    sql """INSERT INTO t3 VALUES (3,3,3,3);"""
+
+    /**
+     * | DML      | insert/update/delete |
+     * | select   | SELECT LIST: SubQuery |
+     * | From     | (OlapScan/ExternalScan)/view/partition(hash/range)/MV |
+     * | Join     | JOIN TYPE: inner/outer(left/right/full)/semi(left/right)/anti(left/right) ON condition: SubQuery |
+     * | Filter   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Group by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Having   | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Order by | EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Winfunc  | Partition by / Order by EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery |
+     * | Limit    | Limit / offset |
+     * | Set op   | union(all)/intersect(all)/except(all) |
+     * | SubQuery | Correlated subquery / non-correlated subquery / Scalar |
+     * EXPRS: OR/AND/not/in/exists/equal/non-equal/composed expr/SubQuery/bitmap type()
+     * 1. (not) in/exists SubQuery
+     * 2. (not) in/exists SubQuery OR/AND (not) in/exists SubQuery
+     * 3. (not) in/exists SubQuery OR/AND (equal/non-equal)
+     * 4. Corelated / non-correlated / Scalar
+     * 5. composed expr：SubQuery
+     * 6. null/not null/uk/no-uk
+     */
+    // 1. single table + olap table + partition by + correlated + equal
+    qt_1_1 """select c1, row_number() over(partition by c1 in (select c1 from t2)) as rn from t1;"""
+    qt_1_2 """select c1, row_number() over(partition by c1 not in (select c1 from t2)) as rn from t1;"""
+
+    qt_1_3 """select c1, row_number() over(partition by exists (select c1 from t2)) as rn from t1;"""
+    qt_1_4 """select c1, row_number() over(partition by not exists (select c1 from t2)) as rn from t1;"""
+
+    // 2. single table + olap table + order by + correlated + equal
+    qt_2_1 """select c1, row_number() over(order by c1 in (select c1 from t2)) as rn from t1;"""
+    qt_2_2 """select c1, row_number() over(order by c1 not in (select c1 from t2)) as rn from t1;"""
+
+    qt_2_3 """select c1, row_number() over(order by exists (select c1 from t2)) as rn from t1;"""
+    qt_2_4 """select c1, row_number() over(order by not exists (select c1 from t2)) as rn from t1;"""
+
+    // 3. single table + olap table + partition by + order by + correlated + equal
+    qt_3_1 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by c1 in (select c1 from t2)) as rn from t1;"""
+    qt_3_2 """select c1, row_number() over(partition by c1 not in (select c1 from t2) order by c1 not in (select c1 from t2)) as rn from t1;"""
+
+    qt_3_3 """select c1, row_number() over(partition by exists (select c1 from t2) order by exists (select c1 from t2)) as rn from t1;"""
+    qt_3_4 """select c1, row_number() over(partition by not exists (select c1 from t2) order by not exists (select c1 from t2)) as rn from t1;"""
+
+    qt_3_5 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by c1 not in (select c1 from t2)) as rn from t1;"""
+    qt_3_6 """select c1, row_number() over(partition by c1 not in (select c1 from t2) order by c1 in (select c1 from t2)) as rn from t1;"""
+
+    qt_3_7 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by exists (select c1 from t2)) as rn from t1;"""
+    qt_3_8 """select c1, row_number() over(partition by not exists (select c1 from t2) order by c1 not in (select c1 from t2)) as rn from t1;"""
+
+    qt_3_9 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by c1 in (select c1 from t3)) as rn from t1;"""
+    qt_3_10 """select c1, row_number() over(partition by c1 not in (select c1 from t2) order by c1 not in (select c1 from t3)) as rn from t1;"""
+
+    qt_3_11 """select c1, row_number() over(partition by exists (select c1 from t2) order by exists (select c1 from t3)) as rn from t1;"""
+    qt_3_12 """select c1, row_number() over(partition by not exists (select c1 from t2) order by not exists (select c1 from t3)) as rn from t1;"""
+
+    qt_3_13 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by c1 not in (select c1 from t3)) as rn from t1;"""
+    qt_3_14 """select c1, row_number() over(partition by c1 not in (select c1 from t2) order by c1 in (select c1 from t3)) as rn from t1;"""
+
+    qt_3_15 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by exists (select c1 from t3)) as rn from t1;"""
+    qt_3_16 """select c1, row_number() over(partition by not exists (select c1 from t2) order by c1 not in (select c1 from t3)) as rn from t1;"""
+
+    // 4. single table + olap table + partition by + order by + case when + equal
+    qt_4_1 """select c1, row_number() over(partition by case when c1 in (select c1 from t2) then c1 else c2 end order by c1 in (select c1 from t3)) as rn from t1;"""
+    qt_4_2 """select c1, row_number() over(partition by case when not c1 in (select c1 from t2) then c1 else c2 end order by c1 not in (select c1 from t3)) as rn from t1;"""
+
+    qt_4_3 """select c1, row_number() over(partition by case when exists (select c1 from t2) then c1 else c2 end order by c1 in (select c1 from t3)) as rn from t1;"""
+    qt_4_4 """select c1, row_number() over(partition by case when not exists (select c1 from t2) then c1 else c2 end order by c1 not in (select c1 from t3)) as rn from t1;"""
+
+    // 5. single table + olap table + partition by + order by + nested case when + equal
+    qt_5_1 """select c1, row_number() over(partition by case when c1 in (select c1 from t2) then c1 in (select c1 from t3) else c2 end) as rn from t1;"""
+    qt_5_2 """select c1, row_number() over(partition by case when not c1 in (select c1 from t2) then c1 not in (select c1 from t3) else c2 end) as rn from t1;"""
+
+    qt_5_3 """select c1, row_number() over(partition by case when exists (select c1 from t2) then c1 in (select c1 from t3) else c2 end) as rn from t1;"""
+    qt_5_4 """select c1, row_number() over(partition by case when not exists (select c1 from t2) then c1 not in (select c1 from t3) else c2 end) as rn from t1;"""
+
+    qt_5_5 """select c1, row_number() over(partition by case when c1 in (select c1 from t2) then c1 else c1 in (select c1 from t3) end) as rn from t1;"""
+    qt_5_6 """select c1, row_number() over(partition by case when not c1 in (select c1 from t2) then c1 else c1 not in (select c1 from t3) end) as rn from t1;"""
+
+    qt_5_7 """select c1, row_number() over(partition by case when exists (select c1 from t2) then c1 else c1 in (select c1 from t3) end) as rn from t1;"""
+    qt_5_8 """select c1, row_number() over(partition by case when not exists (select c1 from t2) then c1 else c1 not in (select c1 from t3) end) as rn from t1;"""
+
+    // 6. single table + olap table + partition by + order by + correlated + equal + filter
+    qt_6_1 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by c1 in (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_2 """select c1, row_number() over(partition by c1 not in (select c1 from t2) order by c1 not in (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+
+    qt_6_3 """select c1, row_number() over(partition by exists (select c1 from t2) order by exists (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_4 """select c1, row_number() over(partition by not exists (select c1 from t2) order by not exists (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+
+    qt_6_5 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by c1 not in (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_6 """select c1, row_number() over(partition by c1 not in (select c1 from t2) order by c1 in (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+
+    qt_6_7 """select c1, row_number() over(partition by c1 in (select c1 from t2) order by exists (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_8 """select c1, row_number() over(partition by not exists (select c1 from t2) order by c1 not in (select c1 from t2)) as rn from t1 where c3 in (select c3 from t3);"""
+
+    qt_6_9 """select c1, row_number() over(partition by case when c1 in (select c1 from t2) then c1 in (select c1 from t3) else c2 end) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_10 """select c1, row_number() over(partition by case when not c1 in (select c1 from t2) then c1 not in (select c1 from t3) else c2 end) as rn from t1 where c3 in (select c3 from t3);"""
+
+    qt_6_11 """select c1, row_number() over(partition by case when exists (select c1 from t2) then c1 in (select c1 from t3) else c2 end) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_12 """select c1, row_number() over(partition by case when not exists (select c1 from t2) then c1 not in (select c1 from t3) else c2 end) as rn from t1 where c3 in (select c3 from t3);"""
+
+    qt_6_13 """select c1, row_number() over(partition by case when c1 in (select c1 from t2) then c1 else c1 in (select c1 from t3) end) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_14 """select c1, row_number() over(partition by case when not c1 in (select c1 from t2) then c1 else c1 not in (select c1 from t3) end) as rn from t1 where c3 in (select c3 from t3);"""
+
+    qt_6_15 """select c1, row_number() over(partition by case when exists (select c1 from t2) then c1 else c1 in (select c1 from t3) end) as rn from t1 where c3 in (select c3 from t3);"""
+    qt_6_16 """select c1, row_number() over(partition by case when not exists (select c1 from t2) then c1 else c1 not in (select c1 from t3) end) as rn from t1 where c3 in (select c3 from t3);"""
+}
+

From 578a7dadc11db1e5b9dc64fdeb603a1dbda73ef6 Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Wed, 27 Dec 2023 14:38:55 +0800
Subject: [PATCH 066/299] [improvement](statistics)Use key column id to query
 column statistic table. (#28885)

Use id as where predicate to load column statistic cache. This could improve performance, because id is the first order key in column statistics table.
---
 .../apache/doris/statistics/StatisticsRepository.java | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
index 76e286e55f73aa..0ffc818b1a0c99 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsRepository.java
@@ -97,10 +97,6 @@ public class StatisticsRepository {
             + " WHERE tbl_id = ${tblId}"
             + " AND part_id IS NOT NULL";
 
-    private static final String QUERY_COLUMN_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
-            + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE "
-            + "tbl_id=${tblId} AND idx_id=${idxId} AND col_id='${colId}'";
-
     private static final String QUERY_PARTITION_STATISTICS = "SELECT * FROM " + FeConstants.INTERNAL_DB_NAME
             + "." + StatisticConstants.STATISTIC_TBL_NAME + " WHERE "
             + " ${inPredicate}"
@@ -357,12 +353,11 @@ public static Map<String, Set<String>> fetchColAndPartsForStats(long tblId) {
 
     public static List<ResultRow> loadColStats(long tableId, long idxId, String colName) {
         Map<String, String> params = new HashMap<>();
-        params.put("tblId", String.valueOf(tableId));
-        params.put("idxId", String.valueOf(idxId));
-        params.put("colId", StatisticsUtil.escapeSQL(colName));
+        String id = constructId(tableId, idxId, colName);
+        params.put("id", StatisticsUtil.escapeSQL(id));
 
         return StatisticsUtil.execStatisticQuery(new StringSubstitutor(params)
-                .replace(QUERY_COLUMN_STATISTICS));
+                .replace(FETCH_COLUMN_STATISTIC_TEMPLATE));
     }
 
     public static List<ResultRow> loadPartStats(Collection<StatisticsCacheKey> keys) {

From e0014308bddb739765139728441b7683afbf70d1 Mon Sep 17 00:00:00 2001
From: walter <w41ter.l@gmail.com>
Date: Wed, 27 Dec 2023 15:17:04 +0800
Subject: [PATCH 067/299] [fix](load) Recycle progress before removing bulk
 load job (#29066)

---
 .../java/org/apache/doris/load/loadv2/BulkLoadJob.java   | 5 +++++
 .../java/org/apache/doris/load/loadv2/LoadManager.java   | 9 +++++++++
 .../org/apache/doris/load/loadv2/ProgressManager.java    | 4 ++++
 3 files changed, 18 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
index 7a9f160b3f8d7b..1939e86f8518cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BulkLoadJob.java
@@ -336,6 +336,11 @@ public UserIdentity getUserInfo() {
         return userInfo;
     }
 
+    public void recycleProgress() {
+        // Recycle memory occupied by Progress.
+        Env.getCurrentProgressManager().removeProgress(String.valueOf(id));
+    }
+
     @Override
     protected void auditFinishedLoadJob() {
         try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index 72bc61239cf8d2..3909ff5d7849e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -421,6 +421,9 @@ public void removeOldLoadJob() {
                     if (job instanceof SparkLoadJob) {
                         ((SparkLoadJob) job).clearSparkLauncherLog();
                     }
+                    if (job instanceof BulkLoadJob) {
+                        ((BulkLoadJob) job).recycleProgress();
+                    }
                     if (list.isEmpty()) {
                         map.remove(job.getLabel());
                     }
@@ -745,6 +748,9 @@ private void cleanLabelInternal(long dbId, String label, boolean isReplay) {
                             if (!job.isCompleted()) {
                                 continue;
                             }
+                            if (job instanceof BulkLoadJob) {
+                                ((BulkLoadJob) job).recycleProgress();
+                            }
                             innerIter.remove();
                             idToLoadJob.remove(job.getId());
                             ++counter;
@@ -765,6 +771,9 @@ private void cleanLabelInternal(long dbId, String label, boolean isReplay) {
                         if (!job.isCompleted()) {
                             continue;
                         }
+                        if (job instanceof BulkLoadJob) {
+                            ((BulkLoadJob) job).recycleProgress();
+                        }
                         iter.remove();
                         idToLoadJob.remove(job.getId());
                         ++counter;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/ProgressManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/ProgressManager.java
index 2453af3788ae52..25829d7c240fd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/ProgressManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/ProgressManager.java
@@ -45,6 +45,10 @@ public void registerProgressSimple(String id) {
         registerProgress(id, 0);
     }
 
+    public void removeProgress(String id) {
+        idToProgress.remove(id);
+    }
+
     public void updateProgress(String id, TUniqueId queryId, TUniqueId fragmentId, int finishedScannerNum) {
         Progress progress = idToProgress.get(id);
         if (progress != null) {

From d3e1afa2e4c38fcbd9e496d9e84133873fb9a139 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 14:21:20 +0700
Subject: [PATCH 068/299] [fix](doc) typo fix in metaInfoActionV2 page (#28510)

---
 .../en/docs/admin-manual/http-actions/fe/meta-info-action-V2.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/docs/admin-manual/http-actions/fe/meta-info-action-V2.md b/docs/en/docs/admin-manual/http-actions/fe/meta-info-action-V2.md
index 27c10017fe5676..99fd341caf2e43 100644
--- a/docs/en/docs/admin-manual/http-actions/fe/meta-info-action-V2.md
+++ b/docs/en/docs/admin-manual/http-actions/fe/meta-info-action-V2.md
@@ -66,7 +66,7 @@ None
 {
     "msg":"success",
     "code":0,
-    "data":["databese list" / "table list" / "table schema"],
+    "data":["database list" / "table list" / "table schema"],
     "count":0
 }
 ```

From 8035287010bb7b1f8fe46b401be15657061439d0 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 14:22:43 +0700
Subject: [PATCH 069/299] [fix](doc) spell errors fixes for memory tracker docs
 (#27865)

---
 .../memory-management/memory-limit-exceeded-analysis.md     | 6 +++---
 .../maint-monitor/memory-management/memory-tracker.md       | 2 +-
 .../memory-management/memory-limit-exceeded-analysis.md     | 4 ++--
 .../maint-monitor/memory-management/memory-tracker.md       | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
index 9155c56b09652a..b1611cb593e52f 100644
--- a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
+++ b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
@@ -55,7 +55,7 @@ Notice:
 2. When the process memory exceeds the limit, BE will trigger memory GC.
 
 ```
-W1127 17:23:16.372572 19896 mem_tracker_limiter.cpp:214] System Mem Exceed Limit Check Faild, Try Alloc: 1062688
+W1127 17:23:16.372572 19896 mem_tracker_limiter.cpp:214] System Mem Exceed Limit Check Failed, Try Alloc: 1062688
 Process Memory Summary:
     process memory used 2.68 GB limit 2.47 GB, sys mem available 50.95 GB min reserve 3.20 GB, tc/jemalloc allocator cache 51.97 MB
 Alloc Stacktrace:
@@ -116,13 +116,13 @@ The low water mark defaults to a maximum of 1.6G, calculated based on `MemTotal`
 ## Query or import a single execution memory limit
 When the following error is returned, it means that the memory limit of a single execution has been exceeded.
 ```
-ERROR 1105 (HY000): errCode = 2, detailMessage = Memory limit exceeded:<consuming tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, failed alloc size 1.03 MB, exceeded tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, limit 100.00 MB, peak used 99.29 MB, current used 99.25 MB>, executing msg:<execute:<ExecNode:VHASH_JOIN_NODE (id=4)>>. backend 172.24.47.117 process memory used 1.13 GB, limit ex 98.92 GB cery tracker If , `set exec_mem_limit=8G` to change limit, details mem usage see log/be.INFO.
+ERROR 1105 (HY000): errCode = 2, detailMessage = Memory limit exceeded:<consuming tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, failed alloc size 1.03 MB, exceeded tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, limit 100.00 MB, peak used 99.29 MB, current used 99.25 MB>, executing msg:<execute:<ExecNode:VHASH_JOIN_NODE (id=4)>>. backend 172.24.47.117 process memory used 1.13 GB, limit 98.92 GB. If query tracker exceed, `set exec_mem_limit=8G` to change limit, details mem usage see log/be.INFO
 ```
 
 ### Error message analysis
 The error message is divided into three parts:
 1. `Memory limit exceeded:<consuming tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>`: It is found that the memory limit is exceeded during the memory application process of query `f78208b15e064527-a84c5c0b04c04fcf`.
-2. `failed alloc size 1.03 MB, exceeded tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, limit 100.00 MB, peak used 99.29 MB, current used 99.25 MB`: The memory requested this time is 1.03 MB The current consumption of `f78208b15e064527-a84c5c0b04c04fcf` memory tracker is 99.28MB plus 1.03MB, which exceeds the limit of 100MB. The value of limit comes from `exec_mem_limit` in session veriables, and the default is 4G.
+2. `failed alloc size 1.03 MB, exceeded tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, limit 100.00 MB, peak used 99.29 MB, current used 99.25 MB`: The memory requested this time is 1.03 MB The current consumption of `f78208b15e064527-a84c5c0b04c04fcf` memory tracker is 99.28MB plus 1.03MB, which exceeds the limit of 100MB. The value of limit comes from `exec_mem_limit` in session variables, and the default is 4G.
 3. `executing msg:<execute:<ExecNode:VHASH_JOIN_NODE (id=4)>>. backend 172.24.47.117 process memory used 1.13 GB, limit 98.92 GB. If query tracker exceeds, `set exec_mem_limit=8G` to change limit, details mem usage see be.INFO.`: The location of this memory application is `VHASH_JOIN_NODE (id=4)`, and it prompts that `set exec_mem_limit` can be used to increase the memory limit of a single query.
 
 ### Log Analysis
diff --git a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
index 835d4b19f3f7d6..46eb9335561a43 100644
--- a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
+++ b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
@@ -70,7 +70,7 @@ For the memory statistics results of historical queries, you can view the `peakM
   - StorageEngine:, the memory consumed by the storage engine during loading the data directory, a subset of Orphan.
   - SegCompaction: The memory sum of all SegCompaction tasks, a subset of Orphan.
   - SegmentMeta: memory use by segment meta data such as footer or index page, a subset of Orphan.
-  - TabletManager: The memory consumed by the storage engine get, add, and delte Tablet, a subset of Orphan.
+  - TabletManager: The memory consumed by the storage engine get, add, and delete Tablet, a subset of Orphan.
   - BufferAllocator: Only used for memory multiplexing in the non-vectorized Partitioned Agg process, a subset of Orphan.
 
 - DataPageCache: Used to cache data Pages to speed up Scan.
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
index 126cc4c9209900..c4bf5d0ac908ca 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
@@ -55,7 +55,7 @@ ERROR 1105 (HY000): errCode = 2, detailMessage = Memory limit exceeded:<consumin
 2、当进程内存超限后，BE会触发内存GC。
 
 ```
-W1127 17:23:16.372572 19896 mem_tracker_limiter.cpp:214] System Mem Exceed Limit Check Faild, Try Alloc: 1062688
+W1127 17:23:16.372572 19896 mem_tracker_limiter.cpp:214] System Mem Exceed Limit Check Failed, Try Alloc: 1062688
 Process Memory Summary:
     process memory used 2.68 GB limit 2.47 GB, sys mem available 50.95 GB min reserve 3.20 GB, tc/jemalloc allocator cache 51.97 MB
 Alloc Stacktrace:
@@ -122,7 +122,7 @@ ERROR 1105 (HY000): errCode = 2, detailMessage = Memory limit exceeded:<consumin
 ### 错误信息分析
 错误信息分为三部分：
 1、`Memory limit exceeded:<consuming tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>`：当前正在执行query `f78208b15e064527-a84c5c0b04c04fcf`的内存申请过程中发现内存超限。
-2、`failed alloc size 1.03 MB, exceeded tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, limit 100.00 MB, peak used 99.29 MB, current used 99.25 MB`：本次尝试申请 1.03MB 的内存，但此时query `f78208b15e064527-a84c5c0b04c04fcf` memory tracker 的当前 consumption 为 99.28MB 加上 1.03MB 后超过了 100MB 的limit，limit的值来自 session veriables 中的 `exec_mem_limit`，默认4G。
+2、`failed alloc size 1.03 MB, exceeded tracker:<Query#Id=f78208b15e064527-a84c5c0b04c04fcf>, limit 100.00 MB, peak used 99.29 MB, current used 99.25 MB`：本次尝试申请 1.03MB 的内存，但此时query `f78208b15e064527-a84c5c0b04c04fcf` memory tracker 的当前 consumption 为 99.28MB 加上 1.03MB 后超过了 100MB 的limit，limit的值来自 session variables 中的 `exec_mem_limit`，默认4G。
 3、`executing msg:<execute:<ExecNode:VHASH_JOIN_NODE (id=4)>>. backend 172.24.47.117 process memory used 1.13 GB, limit 98.92 GB. If query tracker exceed, `set exec_mem_limit=8G` to change limit, details mem usage see be.INFO.`：本次内存申请的位置是`VHASH_JOIN_NODE (id=4)`，并提示可通过 `set exec_mem_limit` 来调高单次查询的内存上限。
 
 ### 日志分析
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
index 0bccb265677271..af63a5c983971c 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
@@ -70,7 +70,7 @@ https://shimo.im/docs/DT6JXDRkdTvdyV3G
   - StorageEngine:，存储引擎加载数据目录过程中消耗的内存，Orphan的子集。
   - SegCompaction: 所有 SegCompaction 任务的内存总和，Orphan的子集。
   - SegmentMeta: memory use by segment meta data such as footer or index page，Orphan的子集。
-  - TabletManager: 存储引擎 get、add、delte Tablet 过程中消耗的内存，Orphan的子集。
+  - TabletManager: 存储引擎 get、add、delete Tablet 过程中消耗的内存，Orphan的子集。
   - BufferAllocator: 仅用于非向量化Partitioned Agg过程中的内存复用，Orphan的子集。
 
 - DataPageCache: 用于缓存数据 Page，用于加速 Scan。

From 0607760efa39a0a8b972f7dad614978a464828be Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Wed, 27 Dec 2023 15:30:52 +0800
Subject: [PATCH 070/299] [typo](resource) fix create odbc resource error msg
 (#29019)

---
 .../main/java/org/apache/doris/analysis/CreateResourceStmt.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
index 76fb41e88d540f..9287c430655d66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateResourceStmt.java
@@ -96,7 +96,7 @@ public void analyze(Analyzer analyzer) throws UserException {
         }
         if (resourceType == ResourceType.ODBC_CATALOG && !Config.enable_odbc_mysql_broker_table) {
             throw new AnalysisException("ODBC table is deprecated, use JDBC instead. Or you can set "
-                    + "`enable_odbc_table=true` in fe.conf to enable ODBC again.");
+                    + "`enable_odbc_mysql_broker_table=true` in fe.conf to enable ODBC again.");
         }
     }
 

From cfed36afbfdeb930f42419cc24a3891370435ae4 Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Wed, 27 Dec 2023 15:42:48 +0800
Subject: [PATCH 071/299] [Fix](topn opt) prevent from merge __TEMP__ column in
 segment iterator (#29121)

---
 be/src/vec/olap/vcollect_iterator.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/be/src/vec/olap/vcollect_iterator.cpp b/be/src/vec/olap/vcollect_iterator.cpp
index ca9e2b729671d9..b13f5333d5e2be 100644
--- a/be/src/vec/olap/vcollect_iterator.cpp
+++ b/be/src/vec/olap/vcollect_iterator.cpp
@@ -255,11 +255,14 @@ Status VCollectIterator::_topn_next(Block* block) {
 
     auto clone_block = block->clone_empty();
     MutableBlock mutable_block = vectorized::MutableBlock::build_mutable_block(&clone_block);
-    // clear TMPE columns to avoid column align problem in mutable_block.add_rows bellow
+    // clear TEMP columns to avoid column align problem in mutable_block.add_rows bellow
     auto all_column_names = mutable_block.get_names();
     for (auto& name : all_column_names) {
         if (name.rfind(BeConsts::BLOCK_TEMP_COLUMN_PREFIX, 0) == 0) {
             mutable_block.erase(name);
+            // clear TEMP columns from block to prevent from storage engine merge with this
+            // fake column
+            block->erase(name);
         }
     }
 

From 2d2f14bc755994a45268f8b6fa0b58f270458b9a Mon Sep 17 00:00:00 2001
From: Ashin Gau <AshinGau@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:45:53 +0800
Subject: [PATCH 072/299] [fix](paimon) use SlotDescriptor to parse the
 required fields (#28990)

Before this PR, Paimon has created the schema of `VectorTable` by accessing meta information. However, once the schema of `VectorTable` in java is not same as `Block` in c++, BE will crashed, and there is no good way to troubleshoot errors.
---
 .../vec/exec/format/table/paimon_reader.cpp   |  7 ++-
 .../doris/common/jni/vec/VectorColumn.java    | 12 +++++
 .../apache/doris/paimon/PaimonJniScanner.java | 44 ++++++++-----------
 3 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/be/src/vec/exec/format/table/paimon_reader.cpp b/be/src/vec/exec/format/table/paimon_reader.cpp
index f1ebb96fa1016f..4d6cfc96b52214 100644
--- a/be/src/vec/exec/format/table/paimon_reader.cpp
+++ b/be/src/vec/exec/format/table/paimon_reader.cpp
@@ -42,9 +42,10 @@ PaimonJniReader::PaimonJniReader(const std::vector<SlotDescriptor*>& file_slot_d
                                  const TFileRangeDesc& range)
         : _file_slot_descs(file_slot_descs), _state(state), _profile(profile) {
     std::vector<std::string> column_names;
+    std::vector<std::string> column_types;
     for (auto& desc : _file_slot_descs) {
-        std::string field = desc->col_name();
-        column_names.emplace_back(field);
+        column_names.emplace_back(desc->col_name());
+        column_types.emplace_back(JniConnector::get_jni_type(desc->type()));
     }
     std::map<String, String> params;
     params["db_name"] = range.table_format_params.paimon_params.db_name;
@@ -57,6 +58,8 @@ PaimonJniReader::PaimonJniReader(const std::vector<SlotDescriptor*>& file_slot_d
     params["tbl_id"] = std::to_string(range.table_format_params.paimon_params.tbl_id);
     params["last_update_time"] =
             std::to_string(range.table_format_params.paimon_params.last_update_time);
+    params["required_fields"] = join(column_names, ",");
+    params["columns_types"] = join(column_types, "#");
 
     // Used to create paimon option
     for (auto& kv : range.table_format_params.paimon_params.paimon_options) {
diff --git a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
index ec7d108bd39d9d..4bb4db86d3b225 100644
--- a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
+++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
@@ -1034,6 +1034,9 @@ public int appendBytesAndOffset(byte[] src) {
     public int appendBytesAndOffset(byte[] src, int offset, int length) {
         int startOffset = childColumns[0].appendBytes(src, offset, length);
         reserve(appendIndex + 1);
+        if (startOffset + length < 0) {
+            throw new RuntimeException("String overflow, offset=" + startOffset + ", length=" + length);
+        }
         OffHeap.putInt(null, offsets + 4L * appendIndex, startOffset + length);
         return appendIndex++;
     }
@@ -1068,6 +1071,9 @@ public void appendBinaryAndOffset(byte[][] batch, boolean isNullable) {
                 bytes = new byte[0];
             }
             int startOffset = childColumns[0].appendBytes(bytes, 0, bytes.length);
+            if (startOffset + bytes.length < 0) {
+                throw new RuntimeException("Binary overflow, offset=" + startOffset + ", length=" + bytes.length);
+            }
             OffHeap.putInt(null, offsets + 4L * appendIndex, startOffset + bytes.length);
             appendIndex++;
         }
@@ -1102,6 +1108,9 @@ public int appendArray(List<ColumnValue> values) {
             childColumns[0].appendValue(v);
         }
         reserve(appendIndex + 1);
+        if (startOffset + length < 0) {
+            throw new RuntimeException("Array overflow, offset=" + startOffset + ", length=" + length);
+        }
         OffHeap.putLong(null, offsets + 8L * appendIndex, startOffset + length);
         return appendIndex++;
     }
@@ -1158,6 +1167,9 @@ public int appendMap(List<ColumnValue> keys, List<ColumnValue> values) {
             childColumns[1].appendValue(v);
         }
         reserve(appendIndex + 1);
+        if (startOffset + length < 0) {
+            throw new RuntimeException("Map overflow, offset=" + startOffset + ", length=" + length);
+        }
         OffHeap.putLong(null, offsets + 8L * appendIndex, startOffset + length);
         return appendIndex++;
     }
diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 849f6fb67bb504..be1505c4ef1bbe 100644
--- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -30,7 +30,6 @@
 import org.apache.paimon.table.Table;
 import org.apache.paimon.table.source.ReadBuilder;
 import org.apache.paimon.table.source.Split;
-import org.apache.paimon.types.DataType;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -43,6 +42,7 @@
 public class PaimonJniScanner extends JniScanner {
     private static final Logger LOG = LoggerFactory.getLogger(PaimonJniScanner.class);
     private static final String PAIMON_OPTION_PREFIX = "paimon_option_prefix.";
+    private final Map<String, String> params;
     private final Map<String, String> paimonOptionParams;
     private final String dbName;
     private final String tblName;
@@ -61,6 +61,13 @@ public class PaimonJniScanner extends JniScanner {
 
     public PaimonJniScanner(int batchSize, Map<String, String> params) {
         LOG.debug("params:{}", params);
+        this.params = params;
+        String[] requiredFields = params.get("required_fields").split(",");
+        String[] requiredTypes = params.get("columns_types").split("#");
+        ColumnType[] columnTypes = new ColumnType[requiredTypes.length];
+        for (int i = 0; i < requiredTypes.length; i++) {
+            columnTypes[i] = ColumnType.parseType(requiredFields[i], requiredTypes[i]);
+        }
         paimonSplit = params.get("paimon_split");
         paimonPredicate = params.get("paimon_predicate");
         dbName = params.get("db_name");
@@ -69,21 +76,17 @@ public PaimonJniScanner(int batchSize, Map<String, String> params) {
         dbId = Long.parseLong(params.get("db_id"));
         tblId = Long.parseLong(params.get("tbl_id"));
         lastUpdateTime = Long.parseLong(params.get("last_update_time"));
-        super.batchSize = batchSize;
-        super.fields = params.get("paimon_column_names").split(",");
-        super.predicates = new ScanPredicate[0];
+        initTableInfo(columnTypes, requiredFields, new ScanPredicate[0], batchSize);
         paimonOptionParams = params.entrySet().stream()
                 .filter(kv -> kv.getKey().startsWith(PAIMON_OPTION_PREFIX))
                 .collect(Collectors
                         .toMap(kv1 -> kv1.getKey().substring(PAIMON_OPTION_PREFIX.length()), kv1 -> kv1.getValue()));
-
     }
 
     @Override
     public void open() throws IOException {
         initTable();
         initReader();
-        parseRequiredTypes();
     }
 
     private void initReader() throws IOException {
@@ -99,30 +102,16 @@ private int[] getProjected() {
 
     private List<Predicate> getPredicates() {
         List<Predicate> predicates = PaimonScannerUtils.decodeStringToObject(paimonPredicate);
-        LOG.info("predicates:{}", predicates);
+        LOG.debug("predicates:{}", predicates);
         return predicates;
     }
 
     private Split getSplit() {
         Split split = PaimonScannerUtils.decodeStringToObject(paimonSplit);
-        LOG.info("split:{}", split);
+        LOG.debug("split:{}", split);
         return split;
     }
 
-    private void parseRequiredTypes() {
-        ColumnType[] columnTypes = new ColumnType[fields.length];
-        for (int i = 0; i < fields.length; i++) {
-            int index = paimonAllFieldNames.indexOf(fields[i]);
-            if (index == -1) {
-                throw new RuntimeException(String.format("Cannot find field %s in schema %s",
-                        fields[i], paimonAllFieldNames));
-            }
-            DataType dataType = table.rowType().getTypeAt(index);
-            columnTypes[i] = PaimonTypeUtils.fromPaimonType(fields[i], dataType);
-        }
-        super.types = columnTypes;
-    }
-
     @Override
     public void close() throws IOException {
         if (reader != null) {
@@ -154,9 +143,12 @@ protected int getNext() throws IOException {
                 recordIterator.releaseBatch();
                 recordIterator = reader.readBatch();
             }
-        } catch (IOException e) {
-            LOG.warn("failed to getNext columnValue ", e);
-            throw new RuntimeException(e);
+        } catch (Exception e) {
+            close();
+            LOG.warn("Failed to get the next batch of paimon. "
+                            + "split: {}, requiredFieldNames: {}, paimonAllFieldNames: {}",
+                    getSplit(), params.get("required_fields"), paimonAllFieldNames, e);
+            throw new IOException(e);
         }
         return rows;
     }
@@ -178,7 +170,7 @@ private void initTable() {
         }
         this.table = tableExt.getTable();
         paimonAllFieldNames = PaimonScannerUtils.fieldNames(this.table.rowType());
-        LOG.info("paimonAllFieldNames:{}", paimonAllFieldNames);
+        LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames);
     }
 
 }

From cd1e109cc3b29cf8214712543d3606aa6476a4f9 Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Wed, 27 Dec 2023 15:57:22 +0800
Subject: [PATCH 073/299] [debug string](pipeline) Add necessary debug info
 (#29119)

---
 be/src/pipeline/pipeline_fragment_context.cpp              | 7 +++++++
 be/src/pipeline/pipeline_fragment_context.h                | 2 +-
 be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index 7957983366373d..034fbaf29513cd 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -963,4 +963,11 @@ bool PipelineFragmentContext::_has_inverted_index_or_partial_update(TOlapTableSi
     return false;
 }
 
+std::string PipelineFragmentContext::debug_string() {
+    fmt::memory_buffer debug_string_buffer;
+    fmt::format_to(debug_string_buffer, "PipelineFragmentContext Info: QueryId = {}\n",
+                   print_id(_query_ctx->query_id()));
+    return fmt::to_string(debug_string_buffer);
+}
+
 } // namespace doris::pipeline
diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h
index 99aac70b1e5811..5009bde9b20e48 100644
--- a/be/src/pipeline/pipeline_fragment_context.h
+++ b/be/src/pipeline/pipeline_fragment_context.h
@@ -148,7 +148,7 @@ class PipelineFragmentContext : public TaskExecutionContext {
     }
     void refresh_next_report_time();
 
-    virtual std::string debug_string() { return ""; }
+    virtual std::string debug_string();
 
     uint64_t create_time() const { return _create_time; }
 
diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
index 86440051a907d2..b628e998f89189 100644
--- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
@@ -1310,6 +1310,7 @@ bool PipelineXFragmentContext::_has_inverted_index_or_partial_update(TOlapTableS
 
 std::string PipelineXFragmentContext::debug_string() {
     fmt::memory_buffer debug_string_buffer;
+    fmt::format_to(debug_string_buffer, "PipelineXFragmentContext Info:\n");
     for (size_t j = 0; j < _tasks.size(); j++) {
         fmt::format_to(debug_string_buffer, "Tasks in instance {}:\n", j);
         for (size_t i = 0; i < _tasks[j].size(); i++) {

From c75e63a2a5c7a1caad7e55befeb7987db8d1ef13 Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Wed, 27 Dec 2023 16:00:52 +0800
Subject: [PATCH 074/299] [Improvement](scan) Use scanner to do projection of
 scan node (#29124)

---
 be/src/exec/exec_node.h                       |  2 +-
 be/src/pipeline/exec/scan_operator.cpp        |  8 +--
 be/src/pipeline/exec/scan_operator.h          |  4 ++
 be/src/pipeline/pipeline_x/operator.cpp       | 10 +--
 be/src/pipeline/pipeline_x/operator.h         | 10 ++-
 .../pipeline/pipeline_x/pipeline_x_task.cpp   |  2 +-
 be/src/vec/exec/scan/pip_scanner_context.h    | 11 +--
 be/src/vec/exec/scan/scanner_context.cpp      | 16 ++++-
 be/src/vec/exec/scan/scanner_context.h        |  3 +
 be/src/vec/exec/scan/scanner_scheduler.cpp    |  2 +-
 be/src/vec/exec/scan/vscan_node.cpp           |  7 +-
 be/src/vec/exec/scan/vscan_node.h             | 14 ++++
 be/src/vec/exec/scan/vscanner.cpp             | 68 ++++++++++++++++++-
 be/src/vec/exec/scan/vscanner.h               |  6 ++
 14 files changed, 138 insertions(+), 25 deletions(-)

diff --git a/be/src/exec/exec_node.h b/be/src/exec/exec_node.h
index eeed37907f9ebe..69f7771767fb1b 100644
--- a/be/src/exec/exec_node.h
+++ b/be/src/exec/exec_node.h
@@ -103,7 +103,7 @@ class ExecNode {
     // TODO: AggregationNode and HashJoinNode cannot be "re-opened" yet.
     [[nodiscard]] virtual Status get_next(RuntimeState* state, vectorized::Block* block, bool* eos);
     // new interface to compatible new optimizers in FE
-    [[nodiscard]] Status get_next_after_projects(
+    [[nodiscard]] virtual Status get_next_after_projects(
             RuntimeState* state, vectorized::Block* block, bool* eos,
             const std::function<Status(RuntimeState*, vectorized::Block*, bool*)>& fn,
             bool clear_data = true);
diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp
index af9529798a3df9..0c0ccd42410148 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1239,10 +1239,10 @@ template <typename Derived>
 Status ScanLocalState<Derived>::_start_scanners(
         const std::list<vectorized::VScannerSPtr>& scanners) {
     auto& p = _parent->cast<typename Derived::Parent>();
-    _scanner_ctx = PipScannerContext::create_shared(state(), this, p._output_tuple_desc, scanners,
-                                                    p.limit(), state()->scan_queue_mem_limit(),
-                                                    p._col_distribute_ids, 1, _scan_dependency,
-                                                    _finish_dependency);
+    _scanner_ctx = PipScannerContext::create_shared(
+            state(), this, p._output_tuple_desc, p.output_row_descriptor(), scanners, p.limit(),
+            state()->scan_queue_mem_limit(), p._col_distribute_ids, 1, _scan_dependency,
+            _finish_dependency);
     return Status::OK();
 }
 
diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h
index afbcaea0a63a7b..833ca55f0957fc 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -427,6 +427,10 @@ class ScanOperatorX : public OperatorX<LocalStateType> {
     Status open(RuntimeState* state) override;
     Status get_block(RuntimeState* state, vectorized::Block* block,
                      SourceState& source_state) override;
+    Status get_block_after_projects(RuntimeState* state, vectorized::Block* block,
+                                    SourceState& source_state) override {
+        return get_block(state, block, source_state);
+    }
     [[nodiscard]] bool is_source() const override { return true; }
 
     const std::vector<TRuntimeFilterDesc>& runtime_filter_descs() override {
diff --git a/be/src/pipeline/pipeline_x/operator.cpp b/be/src/pipeline/pipeline_x/operator.cpp
index b90c9551426f64..e763a6eee1d467 100644
--- a/be/src/pipeline/pipeline_x/operator.cpp
+++ b/be/src/pipeline/pipeline_x/operator.cpp
@@ -193,8 +193,8 @@ Status OperatorXBase::do_projections(RuntimeState* state, vectorized::Block* ori
     return Status::OK();
 }
 
-Status OperatorXBase::get_next_after_projects(RuntimeState* state, vectorized::Block* block,
-                                              SourceState& source_state) {
+Status OperatorXBase::get_block_after_projects(RuntimeState* state, vectorized::Block* block,
+                                               SourceState& source_state) {
     auto local_state = state->get_local_state(operator_id());
     if (_output_row_descriptor) {
         local_state->clear_origin_block();
@@ -461,8 +461,8 @@ Status PipelineXSinkLocalState<DependencyType>::close(RuntimeState* state, Statu
 template <typename LocalStateType>
 Status StreamingOperatorX<LocalStateType>::get_block(RuntimeState* state, vectorized::Block* block,
                                                      SourceState& source_state) {
-    RETURN_IF_ERROR(OperatorX<LocalStateType>::_child_x->get_next_after_projects(state, block,
-                                                                                 source_state));
+    RETURN_IF_ERROR(OperatorX<LocalStateType>::_child_x->get_block_after_projects(state, block,
+                                                                                  source_state));
     return pull(state, block, source_state);
 }
 
@@ -473,7 +473,7 @@ Status StatefulOperatorX<LocalStateType>::get_block(RuntimeState* state, vectori
                                 ->template cast<LocalStateType>();
     if (need_more_input_data(state)) {
         local_state._child_block->clear_column_data();
-        RETURN_IF_ERROR(OperatorX<LocalStateType>::_child_x->get_next_after_projects(
+        RETURN_IF_ERROR(OperatorX<LocalStateType>::_child_x->get_block_after_projects(
                 state, local_state._child_block.get(), local_state._child_source_state));
         source_state = local_state._child_source_state;
         if (local_state._child_block->rows() == 0 &&
diff --git a/be/src/pipeline/pipeline_x/operator.h b/be/src/pipeline/pipeline_x/operator.h
index c4e0a7e94a0e3e..ec2500acf3323f 100644
--- a/be/src/pipeline/pipeline_x/operator.h
+++ b/be/src/pipeline/pipeline_x/operator.h
@@ -271,10 +271,15 @@ class OperatorXBase : public OperatorBase {
         return _output_row_descriptor ? *_output_row_descriptor : _row_descriptor;
     }
 
+    [[nodiscard]] const RowDescriptor* output_row_descriptor() {
+        return _output_row_descriptor.get();
+    }
+
     [[nodiscard]] bool is_source() const override { return false; }
 
-    Status get_next_after_projects(RuntimeState* state, vectorized::Block* block,
-                                   SourceState& source_state);
+    [[nodiscard]] virtual Status get_block_after_projects(RuntimeState* state,
+                                                          vectorized::Block* block,
+                                                          SourceState& source_state);
 
     /// Only use in vectorized exec engine try to do projections to trans _row_desc -> _output_row_desc
     Status do_projections(RuntimeState* state, vectorized::Block* origin_block,
@@ -286,6 +291,7 @@ class OperatorXBase : public OperatorBase {
     template <typename Dependency>
     friend class PipelineXLocalState;
     friend class PipelineXLocalStateBase;
+    friend class VScanner;
     const int _operator_id;
     const int _node_id; // unique w/in single plan tree
     TPlanNodeType::type _type;
diff --git a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
index be62fcac213f84..1a0c0749e8b342 100644
--- a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
+++ b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
@@ -272,7 +272,7 @@ Status PipelineXTask::execute(bool* eos) {
         if (!_dry_run) {
             SCOPED_TIMER(_get_block_timer);
             _get_block_counter->update(1);
-            RETURN_IF_ERROR(_root->get_next_after_projects(_state, block, _data_state));
+            RETURN_IF_ERROR(_root->get_block_after_projects(_state, block, _data_state));
         } else {
             _data_state = SourceState::FINISHED;
         }
diff --git a/be/src/vec/exec/scan/pip_scanner_context.h b/be/src/vec/exec/scan/pip_scanner_context.h
index 42726cb17fe0f0..fbf59fffab20a5 100644
--- a/be/src/vec/exec/scan/pip_scanner_context.h
+++ b/be/src/vec/exec/scan/pip_scanner_context.h
@@ -31,23 +31,26 @@ class PipScannerContext : public vectorized::ScannerContext {
 public:
     PipScannerContext(RuntimeState* state, vectorized::VScanNode* parent,
                       const TupleDescriptor* output_tuple_desc,
+                      const RowDescriptor* output_row_descriptor,
                       const std::list<vectorized::VScannerSPtr>& scanners, int64_t limit_,
                       int64_t max_bytes_in_blocks_queue, const std::vector<int>& col_distribute_ids,
                       const int num_parallel_instances)
-            : vectorized::ScannerContext(state, parent, output_tuple_desc, scanners, limit_,
-                                         max_bytes_in_blocks_queue, num_parallel_instances),
+            : vectorized::ScannerContext(state, parent, output_tuple_desc, output_row_descriptor,
+                                         scanners, limit_, max_bytes_in_blocks_queue,
+                                         num_parallel_instances),
               _col_distribute_ids(col_distribute_ids),
               _need_colocate_distribute(!_col_distribute_ids.empty()) {}
 
     PipScannerContext(RuntimeState* state, ScanLocalStateBase* local_state,
                       const TupleDescriptor* output_tuple_desc,
+                      const RowDescriptor* output_row_descriptor,
                       const std::list<vectorized::VScannerSPtr>& scanners, int64_t limit_,
                       int64_t max_bytes_in_blocks_queue, const std::vector<int>& col_distribute_ids,
                       const int num_parallel_instances,
                       std::shared_ptr<pipeline::ScanDependency> dependency,
                       std::shared_ptr<pipeline::Dependency> finish_dependency)
-            : vectorized::ScannerContext(state, output_tuple_desc, scanners, limit_,
-                                         max_bytes_in_blocks_queue, num_parallel_instances,
+            : vectorized::ScannerContext(state, output_tuple_desc, output_row_descriptor, scanners,
+                                         limit_, max_bytes_in_blocks_queue, num_parallel_instances,
                                          local_state, dependency, finish_dependency),
               _need_colocate_distribute(false) {}
 
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index 4d3820d7634074..954c294574f33d 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -46,6 +46,7 @@ namespace doris::vectorized {
 using namespace std::chrono_literals;
 
 ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* output_tuple_desc,
+                               const RowDescriptor* output_row_descriptor,
                                const std::list<VScannerSPtr>& scanners, int64_t limit_,
                                int64_t max_bytes_in_blocks_queue, const int num_parallel_instances,
                                pipeline::ScanLocalStateBase* local_state,
@@ -54,7 +55,10 @@ ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* outpu
         : _state(state),
           _parent(nullptr),
           _local_state(local_state),
-          _output_tuple_desc(output_tuple_desc),
+          _output_tuple_desc(output_row_descriptor
+                                     ? output_row_descriptor->tuple_descriptors().front()
+                                     : output_tuple_desc),
+          _output_row_descriptor(output_row_descriptor),
           _process_status(Status::OK()),
           _batch_size(state->batch_size()),
           limit(limit_),
@@ -66,6 +70,8 @@ ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* outpu
           _num_parallel_instances(num_parallel_instances),
           _dependency(dependency),
           _finish_dependency(finish_dependency) {
+    DCHECK(_output_row_descriptor == nullptr ||
+           _output_row_descriptor->tuple_descriptors().size() == 1);
     // Use the task exec context as a lock between scanner threads and fragment exection threads
     _task_exec_ctx = _state->get_task_execution_context();
     _query_id = _state->get_query_ctx()->query_id();
@@ -92,13 +98,17 @@ ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* outpu
 
 ScannerContext::ScannerContext(doris::RuntimeState* state, doris::vectorized::VScanNode* parent,
                                const doris::TupleDescriptor* output_tuple_desc,
+                               const RowDescriptor* output_row_descriptor,
                                const std::list<VScannerSPtr>& scanners, int64_t limit_,
                                int64_t max_bytes_in_blocks_queue, const int num_parallel_instances,
                                pipeline::ScanLocalStateBase* local_state)
         : _state(state),
           _parent(parent),
           _local_state(local_state),
-          _output_tuple_desc(output_tuple_desc),
+          _output_tuple_desc(output_row_descriptor
+                                     ? output_row_descriptor->tuple_descriptors().front()
+                                     : output_tuple_desc),
+          _output_row_descriptor(output_row_descriptor),
           _process_status(Status::OK()),
           _batch_size(state->batch_size()),
           limit(limit_),
@@ -108,6 +118,8 @@ ScannerContext::ScannerContext(doris::RuntimeState* state, doris::vectorized::VS
           _scanners(scanners),
           _scanners_ref(scanners.begin(), scanners.end()),
           _num_parallel_instances(num_parallel_instances) {
+    DCHECK(_output_row_descriptor == nullptr ||
+           _output_row_descriptor->tuple_descriptors().size() == 1);
     // Use the task exec context as a lock between scanner threads and fragment exection threads
     _task_exec_ctx = _state->get_task_execution_context();
     _query_id = _state->get_query_ctx()->query_id();
diff --git a/be/src/vec/exec/scan/scanner_context.h b/be/src/vec/exec/scan/scanner_context.h
index ba9c1fdee10a5b..6a3e8553f8fc3a 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -70,6 +70,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext> {
 
 public:
     ScannerContext(RuntimeState* state, VScanNode* parent, const TupleDescriptor* output_tuple_desc,
+                   const RowDescriptor* output_row_descriptor,
                    const std::list<VScannerSPtr>& scanners, int64_t limit_,
                    int64_t max_bytes_in_blocks_queue, const int num_parallel_instances = 1,
                    pipeline::ScanLocalStateBase* local_state = nullptr);
@@ -187,6 +188,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext> {
 
 protected:
     ScannerContext(RuntimeState* state_, const TupleDescriptor* output_tuple_desc,
+                   const RowDescriptor* output_row_descriptor,
                    const std::list<VScannerSPtr>& scanners_, int64_t limit_,
                    int64_t max_bytes_in_blocks_queue_, const int num_parallel_instances,
                    pipeline::ScanLocalStateBase* local_state,
@@ -203,6 +205,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext> {
 
     // the comment of same fields in VScanNode
     const TupleDescriptor* _output_tuple_desc = nullptr;
+    const RowDescriptor* _output_row_descriptor = nullptr;
 
     // _transfer_lock is used to protect the critical section
     // where the ScanNode and ScannerScheduler interact.
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 84f56813c34a9e..6ec83e8bd6aec8 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -351,7 +351,7 @@ void ScannerScheduler::_scanner_scan(ScannerScheduler* scheduler,
 
         BlockUPtr block = ctx->get_free_block();
 
-        status = scanner->get_block(state, block.get(), &eos);
+        status = scanner->get_block_after_projects(state, block.get(), &eos);
         // The VFileScanner for external table may try to open not exist files,
         // Because FE file cache for external table may out of date.
         // So, NOT_FOUND for VFileScanner is not a fail case.
diff --git a/be/src/vec/exec/scan/vscan_node.cpp b/be/src/vec/exec/scan/vscan_node.cpp
index 5176d7900b3c7e..c45468e0514d30 100644
--- a/be/src/vec/exec/scan/vscan_node.cpp
+++ b/be/src/vec/exec/scan/vscan_node.cpp
@@ -323,10 +323,11 @@ Status VScanNode::_start_scanners(const std::list<VScannerSPtr>& scanners,
     if (_is_pipeline_scan) {
         int max_queue_size = _shared_scan_opt ? std::max(query_parallel_instance_num, 1) : 1;
         _scanner_ctx = pipeline::PipScannerContext::create_shared(
-                _state, this, _output_tuple_desc, scanners, limit(), _state->scan_queue_mem_limit(),
-                _col_distribute_ids, max_queue_size);
+                _state, this, _output_tuple_desc, _output_row_descriptor.get(), scanners, limit(),
+                _state->scan_queue_mem_limit(), _col_distribute_ids, max_queue_size);
     } else {
-        _scanner_ctx = ScannerContext::create_shared(_state, this, _output_tuple_desc, scanners,
+        _scanner_ctx = ScannerContext::create_shared(_state, this, _output_tuple_desc,
+                                                     _output_row_descriptor.get(), scanners,
                                                      limit(), _state->scan_queue_mem_limit());
     }
     return Status::OK();
diff --git a/be/src/vec/exec/scan/vscan_node.h b/be/src/vec/exec/scan/vscan_node.h
index 5917d0ff46b5eb..7cd8b15991ef2d 100644
--- a/be/src/vec/exec/scan/vscan_node.h
+++ b/be/src/vec/exec/scan/vscan_node.h
@@ -177,6 +177,20 @@ class VScanNode : public ExecNode, public RuntimeFilterConsumer {
 
     RuntimeProfile* scanner_profile() { return _scanner_profile.get(); }
 
+    Status get_next_after_projects(
+            RuntimeState* state, vectorized::Block* block, bool* eos,
+            const std::function<Status(RuntimeState*, vectorized::Block*, bool*)>& fn,
+            bool clear_data = true) override {
+        Defer defer([block, this]() {
+            if (block && !block->empty()) {
+                COUNTER_UPDATE(_output_bytes_counter, block->allocated_bytes());
+                COUNTER_UPDATE(_block_count_counter, 1);
+            }
+        });
+        _peak_memory_usage_counter->set(_mem_tracker->peak_consumption());
+        return get_next(state, block, eos);
+    }
+
 protected:
     // Different data sources register different profiles by implementing this method
     virtual Status _init_profile();
diff --git a/be/src/vec/exec/scan/vscanner.cpp b/be/src/vec/exec/scan/vscanner.cpp
index a7c0c3c4062ca8..0a7a8c9c019c09 100644
--- a/be/src/vec/exec/scan/vscanner.cpp
+++ b/be/src/vec/exec/scan/vscanner.cpp
@@ -35,7 +35,8 @@ VScanner::VScanner(RuntimeState* state, VScanNode* parent, int64_t limit, Runtim
           _local_state(nullptr),
           _limit(limit),
           _profile(profile),
-          _output_tuple_desc(parent->output_tuple_desc()) {
+          _output_tuple_desc(parent->output_tuple_desc()),
+          _output_row_descriptor(_parent->_output_row_descriptor.get()) {
     _total_rf_num = _parent->runtime_filter_num();
 }
 
@@ -46,7 +47,8 @@ VScanner::VScanner(RuntimeState* state, pipeline::ScanLocalStateBase* local_stat
           _local_state(local_state),
           _limit(limit),
           _profile(profile),
-          _output_tuple_desc(_local_state->output_tuple_desc()) {
+          _output_tuple_desc(_local_state->output_tuple_desc()),
+          _output_row_descriptor(_local_state->_parent->output_row_descriptor()) {
     _total_rf_num = _local_state->runtime_filter_num();
 }
 
@@ -58,9 +60,30 @@ Status VScanner::prepare(RuntimeState* state, const VExprContextSPtrs& conjuncts
         }
     }
 
+    const auto& projections = _parent ? _parent->_projections : _local_state->_projections;
+    if (!projections.empty()) {
+        _projections.resize(projections.size());
+        for (size_t i = 0; i != projections.size(); ++i) {
+            RETURN_IF_ERROR(projections[i]->clone(state, _projections[i]));
+        }
+    }
+
     return Status::OK();
 }
 
+Status VScanner::get_block_after_projects(RuntimeState* state, vectorized::Block* block,
+                                          bool* eos) {
+    auto& row_descriptor =
+            _parent ? _parent->_row_descriptor : _local_state->_parent->row_descriptor();
+    if (_output_row_descriptor) {
+        _origin_block.clear_column_data(row_descriptor.num_materialized_slots());
+        auto status = get_block(state, &_origin_block, eos);
+        if (UNLIKELY(!status.ok())) return status;
+        return _do_projections(&_origin_block, block);
+    }
+    return get_block(state, block, eos);
+}
+
 Status VScanner::get_block(RuntimeState* state, Block* block, bool* eof) {
     // only empty block should be here
     DCHECK(block->rows() == 0);
@@ -138,6 +161,47 @@ Status VScanner::_filter_output_block(Block* block) {
     return st;
 }
 
+Status VScanner::_do_projections(vectorized::Block* origin_block, vectorized::Block* output_block) {
+    auto projection_timer = _parent ? _parent->_projection_timer : _local_state->_projection_timer;
+    auto exec_timer = _parent ? _parent->_exec_timer : _local_state->_exec_timer;
+    SCOPED_TIMER(exec_timer);
+    SCOPED_TIMER(projection_timer);
+
+    MutableBlock mutable_block =
+            VectorizedUtils::build_mutable_mem_reuse_block(output_block, *_output_row_descriptor);
+    auto rows = origin_block->rows();
+
+    if (rows != 0) {
+        auto& mutable_columns = mutable_block.mutable_columns();
+
+        if (mutable_columns.size() != _projections.size()) {
+            return Status::InternalError(
+                    "Logical error in scanner, output of projections {} mismatches with "
+                    "scanner output {}",
+                    _projections.size(), mutable_columns.size());
+        }
+
+        for (int i = 0; i < mutable_columns.size(); ++i) {
+            auto result_column_id = -1;
+            RETURN_IF_ERROR(_projections[i]->execute(origin_block, &result_column_id));
+            auto column_ptr = origin_block->get_by_position(result_column_id)
+                                      .column->convert_to_full_column_if_const();
+            //TODO: this is a quick fix, we need a new function like "change_to_nullable" to do it
+            if (mutable_columns[i]->is_nullable() xor column_ptr->is_nullable()) {
+                DCHECK(mutable_columns[i]->is_nullable() && !column_ptr->is_nullable());
+                reinterpret_cast<ColumnNullable*>(mutable_columns[i].get())
+                        ->insert_range_from_not_nullable(*column_ptr, 0, rows);
+            } else {
+                mutable_columns[i]->insert_range_from(*column_ptr, 0, rows);
+            }
+        }
+        DCHECK(mutable_block.rows() == rows);
+        output_block->set_columns(std::move(mutable_columns));
+    }
+
+    return Status::OK();
+}
+
 Status VScanner::try_append_late_arrival_runtime_filter() {
     if (_applied_rf_num == _total_rf_num) {
         return Status::OK();
diff --git a/be/src/vec/exec/scan/vscanner.h b/be/src/vec/exec/scan/vscanner.h
index 29daf9a68c557f..b00c3f348f41d3 100644
--- a/be/src/vec/exec/scan/vscanner.h
+++ b/be/src/vec/exec/scan/vscanner.h
@@ -67,6 +67,7 @@ class VScanner {
     virtual Status open(RuntimeState* state) { return Status::OK(); }
 
     Status get_block(RuntimeState* state, Block* block, bool* eos);
+    Status get_block_after_projects(RuntimeState* state, vectorized::Block* block, bool* eos);
 
     virtual Status close(RuntimeState* state);
 
@@ -89,6 +90,8 @@ class VScanner {
     // Filter the output block finally.
     Status _filter_output_block(Block* block);
 
+    Status _do_projections(vectorized::Block* origin_block, vectorized::Block* output_block);
+
     // Not virtual, all child will call this method explictly
     Status prepare(RuntimeState* state, const VExprContextSPtrs& conjuncts);
 
@@ -172,6 +175,7 @@ class VScanner {
     RuntimeProfile* _profile = nullptr;
 
     const TupleDescriptor* _output_tuple_desc = nullptr;
+    const RowDescriptor* _output_row_descriptor = nullptr;
 
     // If _input_tuple_desc is set, the scanner will read data into
     // this _input_block first, then convert to the output block.
@@ -189,6 +193,8 @@ class VScanner {
     // Cloned from _conjuncts of scan node.
     // It includes predicate in SQL and runtime filters.
     VExprContextSPtrs _conjuncts;
+    VExprContextSPtrs _projections;
+    vectorized::Block _origin_block;
 
     VExprContextSPtrs _common_expr_ctxs_push_down;
     // Late arriving runtime filters will update _conjuncts.

From 26842049f3a43bb508705f5c9f066506c0943c63 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:42:00 +0700
Subject: [PATCH 075/299] [fix](doc) typo fix in fqdn page (#28509)

---
 docs/en/docs/admin-manual/cluster-management/fqdn.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/docs/admin-manual/cluster-management/fqdn.md b/docs/en/docs/admin-manual/cluster-management/fqdn.md
index 0693392c006090..c4c8673f72ee1f 100644
--- a/docs/en/docs/admin-manual/cluster-management/fqdn.md
+++ b/docs/en/docs/admin-manual/cluster-management/fqdn.md
@@ -30,9 +30,9 @@ under the License.
 
 This article introduces how to enable the use of Apache Doris based on FQDN (Fully Qualified Domain Name). FQDN is the complete domain name of a specific computer or host on the Internet.
 
-After Doris supports FQDN, communication between nodes is entirely based on FQDN. When adding various types of nodes, the FQDN should be directly specified. For example, the command to add a BE node is' ALT SYSTEM ADD BACK END "be_host: eartbeat_service_port",
+After Doris supports FQDN, communication between nodes is entirely based on FQDN. When adding various types of nodes, the FQDN should be directly specified. For example, the command to add a BE node is' ALTER SYSTEM ADD BACKEND "be_host:heartbeat_service_port",
 
-'be_host' was previously the IP address of the BE node. After starting the FQDN, be_ The host should specify the FQDN of the BE node.
+'be_host' was previously the IP address of the BE node. After starting the FQDN, be_host should specify the FQDN of the BE node.
 
 ## Preconditions
 

From 64ca508c0b8eb3533337c3a2a880f9b67279e2d7 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:42:28 +0700
Subject: [PATCH 076/299] [fix](doc) typo fix in schema change docs (#28453)

---
 docs/en/docs/advanced/alter-table/schema-change.md    | 2 +-
 docs/zh-CN/docs/advanced/alter-table/schema-change.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/docs/advanced/alter-table/schema-change.md b/docs/en/docs/advanced/alter-table/schema-change.md
index c2d5f61b89b54f..31a335114aef2a 100644
--- a/docs/en/docs/advanced/alter-table/schema-change.md
+++ b/docs/en/docs/advanced/alter-table/schema-change.md
@@ -211,7 +211,7 @@ When done, the Schema becomes:
 +-----------+-------+-------------+------+------+---------+-------+
 ```
 
-Because the Schema Chanage job is an asynchronous operation, only one Schema chanage job can be performed on the same table at the same time. To check the operation status of the job, you can use the following command
+Because the Schema Change job is an asynchronous operation, only one Schema change job can be performed on the same table at the same time. To check the operation status of the job, you can use the following command
 
 ```sql
 SHOW ALTER TABLE COLUMN\G;
diff --git a/docs/zh-CN/docs/advanced/alter-table/schema-change.md b/docs/zh-CN/docs/advanced/alter-table/schema-change.md
index fa73ff9e52879b..3a3e993f86d4a1 100644
--- a/docs/zh-CN/docs/advanced/alter-table/schema-change.md
+++ b/docs/zh-CN/docs/advanced/alter-table/schema-change.md
@@ -219,7 +219,7 @@ alter table example_tbl modify column k3 varchar(50) key null comment 'to 50'
 +-----------+-------+-------------+------+------+---------+-------+
 ```
 
-因为Schema Chanage 作业是异步操作，同一个表同时只能进行一个Schema chanage 作业，查看作业运行情况，可以通过下面这个命令
+因为Schema Change 作业是异步操作，同一个表同时只能进行一个Schema change 作业，查看作业运行情况，可以通过下面这个命令
 
 ```sql
 SHOW ALTER TABLE COLUMN\G;

From 1c0bbcf41653ebf2af8167598c4654c07d50acfc Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:42:43 +0700
Subject: [PATCH 077/299] [fix](doc) typo fix in sql-interception docs (#28445)

---
 docs/en/docs/admin-manual/sql-interception.md    | 2 +-
 docs/zh-CN/docs/admin-manual/sql-interception.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/docs/admin-manual/sql-interception.md b/docs/en/docs/admin-manual/sql-interception.md
index 919322bd9eb503..3141ada4252875 100644
--- a/docs/en/docs/admin-manual/sql-interception.md
+++ b/docs/en/docs/admin-manual/sql-interception.md
@@ -86,7 +86,7 @@ ALTER SQL_BLOCK_RULE test_rule PROPERTIES("sql"="select \\* from test_table","en
 ALTER SQL_BLOCK_RULE test_rule2 PROPERTIES("partition_num" = "10","tablet_num"="300","enable"="true")
 ```
 
-- drop SQL block rule，Support multiple rules, separated by `,`,Please see the specific grammar[DROP SQL BLOCK RULR](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-SQL-BLOCK-RULE.md)
+- drop SQL block rule，Support multiple rules, separated by `,`,Please see the specific grammar[DROP SQL BLOCK RULE](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-SQL-BLOCK-RULE.md)
 ```sql
 DROP SQL_BLOCK_RULE test_rule1,test_rule2
 ```
diff --git a/docs/zh-CN/docs/admin-manual/sql-interception.md b/docs/zh-CN/docs/admin-manual/sql-interception.md
index 27ea3a678ba7ef..66b726baf154dc 100644
--- a/docs/zh-CN/docs/admin-manual/sql-interception.md
+++ b/docs/zh-CN/docs/admin-manual/sql-interception.md
@@ -86,7 +86,7 @@ ALTER SQL_BLOCK_RULE test_rule PROPERTIES("sql"="select \\* from test_table","en
 ALTER SQL_BLOCK_RULE test_rule2 PROPERTIES("partition_num" = "10","tablet_num"="300","enable"="true")
 ```
 
-- 删除SQL阻止规则，支持多规则，以`,`隔开，具体语法请参阅 [DROP SQL BLOCK RULR](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-SQL-BLOCK-RULE.md)
+- 删除SQL阻止规则，支持多规则，以`,`隔开，具体语法请参阅 [DROP SQL BLOCK RULE](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-SQL-BLOCK-RULE.md)
 ```
 DROP SQL_BLOCK_RULE test_rule1,test_rule2
 ```

From 42690415635fd3698b855474caadbde5c0aacdab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= <jianxie0@gmail.com>
Date: Wed, 27 Dec 2023 16:45:03 +0800
Subject: [PATCH 078/299] [enhancement](Nereids): catch exception when calling
 getConstraintMap for external map (#29112)

---
 .../org/apache/doris/catalog/TableIf.java     | 11 ++++-
 .../rules/rewrite/EliminateJoinByFK.java      |  4 --
 .../rules/rewrite/PullUpJoinFromUnionAll.java | 40 +++++--------------
 3 files changed, 19 insertions(+), 36 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index a05c23dc8ffeb4..50a32e3d00ec11 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -161,8 +161,11 @@ default int getBaseColumnIdxByName(String colName) {
 
     void write(DataOutput out) throws IOException;
 
+    // Don't use it outside due to its thread-unsafe, use get specific constraints instead.
     default Map<String, Constraint> getConstraintsMap() {
-        throw new RuntimeException(String.format("Not implemented constraint for table %s", this));
+        throw new RuntimeException(String.format("Not implemented constraint for table %s. "
+                + "And the function can't be called outside, consider get specific function "
+                + "like getForeignKeyConstraints/getPrimaryKeyConstraints/getUniqueConstraints.", this));
     }
 
     default Set<ForeignKeyConstraint> getForeignKeyConstraints() {
@@ -172,6 +175,8 @@ default Set<ForeignKeyConstraint> getForeignKeyConstraints() {
                     .filter(ForeignKeyConstraint.class::isInstance)
                     .map(ForeignKeyConstraint.class::cast)
                     .collect(ImmutableSet.toImmutableSet());
+        } catch (Exception ignored) {
+            return ImmutableSet.of();
         } finally {
             readUnlock();
         }
@@ -184,6 +189,8 @@ default Set<PrimaryKeyConstraint> getPrimaryKeyConstraints() {
                     .filter(PrimaryKeyConstraint.class::isInstance)
                     .map(PrimaryKeyConstraint.class::cast)
                     .collect(ImmutableSet.toImmutableSet());
+        } catch (Exception ignored) {
+            return ImmutableSet.of();
         } finally {
             readUnlock();
         }
@@ -196,6 +203,8 @@ default Set<UniqueConstraint> getUniqueConstraints() {
                     .filter(UniqueConstraint.class::isInstance)
                     .map(UniqueConstraint.class::cast)
                     .collect(ImmutableSet.toImmutableSet());
+        } catch (Exception ignored) {
+            return ImmutableSet.of();
         } finally {
             readUnlock();
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java
index 49c99605bbce06..17fcf7842ef274 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java
@@ -18,7 +18,6 @@
 package org.apache.doris.nereids.rules.rewrite;
 
 import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Table;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.nereids.jobs.JobContext;
 import org.apache.doris.nereids.trees.expressions.Alias;
@@ -90,9 +89,6 @@ public Plan visitLogicalRelation(LogicalRelation relation, ForeignKeyContext con
             if (!(relation instanceof LogicalCatalogRelation)) {
                 return relation;
             }
-            if (!(((LogicalCatalogRelation) relation).getTable() instanceof Table)) {
-                return relation;
-            }
             context.putAllForeignKeys(((LogicalCatalogRelation) relation).getTable());
             relation.getOutput().stream()
                     .filter(SlotReference.class::isInstance)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java
index 08979d33933ade..16d61731d37f4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PullUpJoinFromUnionAll.java
@@ -17,7 +17,6 @@
 
 package org.apache.doris.nereids.rules.rewrite;
 
-import org.apache.doris.catalog.constraint.Constraint;
 import org.apache.doris.catalog.constraint.ForeignKeyConstraint;
 import org.apache.doris.catalog.constraint.PrimaryKeyConstraint;
 import org.apache.doris.catalog.constraint.UniqueConstraint;
@@ -378,48 +377,27 @@ private LogicalCatalogRelation findTableFromSlot(SlotReference targetSlot,
     }
 
     private ForeignKeyConstraint getFkInfoFromConstraint(LogicalCatalogRelation table) {
-        table.getTable().readLock();
-        try {
-            for (Map.Entry<String, Constraint> constraintMap : table.getTable().getConstraintsMap().entrySet()) {
-                Constraint constraint = constraintMap.getValue();
-                if (constraint instanceof ForeignKeyConstraint) {
-                    return (ForeignKeyConstraint) constraint;
-                }
-            }
+        Set<ForeignKeyConstraint> foreignKeyConstraints = table.getTable().getForeignKeyConstraints();
+        if (foreignKeyConstraints.isEmpty()) {
             return null;
-        } finally {
-            table.getTable().readUnlock();
         }
+        return foreignKeyConstraints.stream().iterator().next();
     }
 
     private Set<String> getPkInfoFromConstraint(LogicalCatalogRelation table) {
-        table.getTable().readLock();
-        try {
-            for (Map.Entry<String, Constraint> constraintMap : table.getTable().getConstraintsMap().entrySet()) {
-                Constraint constraint = constraintMap.getValue();
-                if (constraint instanceof PrimaryKeyConstraint) {
-                    return ((PrimaryKeyConstraint) constraint).getPrimaryKeyNames();
-                }
-            }
+        Set<PrimaryKeyConstraint> primaryKeyConstraints = table.getTable().getPrimaryKeyConstraints();
+        if (primaryKeyConstraints.isEmpty()) {
             return null;
-        } finally {
-            table.getTable().readUnlock();
         }
+        return primaryKeyConstraints.stream().iterator().next().getPrimaryKeyNames();
     }
 
     private Set<String> getUkInfoFromConstraint(LogicalCatalogRelation table) {
-        table.getTable().readLock();
-        try {
-            for (Map.Entry<String, Constraint> constraintMap : table.getTable().getConstraintsMap().entrySet()) {
-                Constraint constraint = constraintMap.getValue();
-                if (constraint instanceof UniqueConstraint) {
-                    return ((UniqueConstraint) constraint).getUniqueColumnNames();
-                }
-            }
+        Set<UniqueConstraint> uniqueConstraints = table.getTable().getUniqueConstraints();
+        if (uniqueConstraints.isEmpty()) {
             return null;
-        } finally {
-            table.getTable().readUnlock();
         }
+        return uniqueConstraints.stream().iterator().next().getUniqueColumnNames();
     }
 
     private boolean checkJoinRoot(LogicalJoin joinRoot) {

From cec7eba1cceeb8289470eb65323d83640e4fc0ee Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:51:58 +0700
Subject: [PATCH 079/299] [fix](doc) spell errors fixes for table-repair and
 table local debug (#28153)

---
 .../admin-manual/maint-monitor/tablet-local-debug.md   |  2 +-
 .../maint-monitor/tablet-repair-and-balance.md         | 10 +++++-----
 .../admin-manual/maint-monitor/tablet-local-debug.md   |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/en/docs/admin-manual/maint-monitor/tablet-local-debug.md b/docs/en/docs/admin-manual/maint-monitor/tablet-local-debug.md
index 298ae38ee9baa1..173ac02cd9af6c 100644
--- a/docs/en/docs/admin-manual/maint-monitor/tablet-local-debug.md
+++ b/docs/en/docs/admin-manual/maint-monitor/tablet-local-debug.md
@@ -52,7 +52,7 @@ PartitionName: tbl1
     DetailCmd: SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020';
 ```
 
-Execute `DetailCmd` in the previous step to obtain information such as BackendId/SchemHash.
+Execute `DetailCmd` in the previous step to obtain information such as BackendId/SchemaHash.
 
 ```
 mysql>  SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020'\G
diff --git a/docs/en/docs/admin-manual/maint-monitor/tablet-repair-and-balance.md b/docs/en/docs/admin-manual/maint-monitor/tablet-repair-and-balance.md
index cc12f19f0159a8..0ad922b45479aa 100644
--- a/docs/en/docs/admin-manual/maint-monitor/tablet-repair-and-balance.md
+++ b/docs/en/docs/admin-manual/maint-monitor/tablet-repair-and-balance.md
@@ -28,11 +28,11 @@ under the License.
 
 Beginning with version 0.9.0, Doris introduced an optimized replica management strategy and supported a richer replica status viewing tool. This document focuses on Doris data replica balancing, repair scheduling strategies, and replica management operations and maintenance methods. Help users to more easily master and manage the replica status in the cluster.
 
-> Repairing and balancing copies of tables with Collocation attributes can be referred to [HERE](../../query-acceleration/join-optimization/colocation-join.md)
+> Repairing and balancing copies of tables with Colocation attributes can be referred to [HERE](../../query-acceleration/join-optimization/colocation-join.md)
 
 ## Noun Interpretation
 
-1. Tablet: The logical fragmentation of a Doris table, where a table has multiple fragmentations.
+1. Tablet: The logical fragmentation of a Doris table, where a table has multiple fragments.
 2. Replica: A sliced copy, defaulting to three copies of a slice.
 3. Healthy Replica: A healthy copy that survives at Backend and has a complete version.
 4. Tablet Checker (TC): A resident background thread that scans all Tablets regularly, checks the status of these Tablets, and decides whether to send them to Tablet Scheduler based on the results.
@@ -133,7 +133,7 @@ For different states, we adopt different repair methods:
 
 3. REPLICA\_MISSING\_IN\_CLUSTER
 
-	This state processing method is the same as REPLICAMISSING.
+	This state processing method is the same as REPLICA\_MISSING.
 
 4. REDUNDANT
 
@@ -405,7 +405,7 @@ Tablet state view mainly looks at the state of the tablet, as well as the state
 
 	* TabletId: The ID of the Tablet waiting to be scheduled. A scheduling task is for only one Tablet
 	* Type: Task type, which can be REPAIR (repair) or BALANCE (balance)
-	* Status: The current status of the Tablet, such as REPLICAMISSING (copy missing)
+	* Status: The current status of the Tablet, such as REPLICA\_MISSING (copy missing)
 	* State: The status of the scheduling task may be PENDING/RUNNING/FINISHED/CANCELLED/TIMEOUT/UNEXPECTED
 	* OrigPrio: Initial Priority
 	* DynmPrio: Current dynamically adjusted priority
@@ -605,7 +605,7 @@ The following adjustable parameters are all configurable parameters in fe.conf.
 
 * use\_new\_tablet\_scheduler
 
-	* Description: Whether to enable the new replica scheduling mode. The new replica scheduling method is the replica scheduling method introduced in this document. If turned on, `disable_colocate_join` must be `true`. Because the new scheduling strategy does not support data fragmentation scheduling of co-locotion tables for the time being.
+	* Description: Whether to enable the new replica scheduling mode. The new replica scheduling method is the replica scheduling method introduced in this document. If turned on, `disable_colocate_join` must be `true`. Because the new scheduling strategy does not support data fragmentation scheduling of co-location tables for the time being.
 	* Default value:true
 	* Importance: High
 
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/tablet-local-debug.md b/docs/zh-CN/docs/admin-manual/maint-monitor/tablet-local-debug.md
index 710ad31540c5f8..cb6e1cd947c792 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/tablet-local-debug.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/tablet-local-debug.md
@@ -52,7 +52,7 @@ PartitionName: tbl1
     DetailCmd: SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020';
 ```
 
-执行上一步中的 `DetailCmd` 获取 BackendId/SchemHash 等信息。
+执行上一步中的 `DetailCmd` 获取 BackendId/SchemaHash 等信息。
 
 ```
 mysql>  SHOW PROC '/dbs/10004/10016/partitions/10015/10017/10020'\G

From ef8ae75e37db2f97f4440a823e6da94bcf1f9770 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:55:37 +0700
Subject: [PATCH 080/299] [fix](doc) updated node version as per given in the
 url. (#28427)

---
 docs/en/docs/install/source-install/compilation-arm.md | 10 +++++-----
 .../docs/install/source-install/compilation-arm.md     |  8 ++++----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/docs/en/docs/install/source-install/compilation-arm.md b/docs/en/docs/install/source-install/compilation-arm.md
index bbd48ff3860492..adc9027822fcfe 100644
--- a/docs/en/docs/install/source-install/compilation-arm.md
+++ b/docs/en/docs/install/source-install/compilation-arm.md
@@ -64,7 +64,7 @@ For detailed instructions, please refer to [Compile with ldb-toolchain](./compil
 Note that you need to download the corresponding aarch64 versions of jdk and nodejs:
 
 1. [Java8-aarch64](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u291-linux-aarch64.tar.gz)
-2. [Node v12.13.0-aarch64](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v16.3.0-linux-arm64.tar.xz)
+2. [Node v16.3.0-aarch64](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v16.3.0-linux-arm64.tar.xz)
 
 ## CentOS & Ubuntu
 
@@ -497,7 +497,7 @@ USE_AVX2=OFF sh build.sh
 
    - Problem Description
 
-     - A "file not found" error occurrs during compilation:
+     - A "file not found" error occurs during compilation:
 
        > Couldn't find pkg.m4 from pkg-config. Install the appropriate package for your distribution or set ACLOCAL_PATH to the directory containing pkg.m4.
 
@@ -581,11 +581,11 @@ USE_AVX2=OFF sh build.sh
   
    - Cause
   
-     Not compiled with ldb-toolschain
+     Not compiled with ldb-toolchain
   
    - Solution
   
-     - Check if the ldb-toolschain environment variable is configured
+     - Check if the ldb-toolchain environment variable is configured
      - Check if gcc version is `gcc-11`
-     - Delete the ldb directory after the `ldb-toolschain.sh` script is executed, re-execute and configure the environment variables, and verify the gcc version
+     - Delete the ldb directory after the `ldb_toolchain_gen.aarch64.sh` script is executed, re-execute and configure the environment variables, and verify the gcc version
 
diff --git a/docs/zh-CN/docs/install/source-install/compilation-arm.md b/docs/zh-CN/docs/install/source-install/compilation-arm.md
index 4cf7e5564e22b8..c08821deb6e219 100644
--- a/docs/zh-CN/docs/install/source-install/compilation-arm.md
+++ b/docs/zh-CN/docs/install/source-install/compilation-arm.md
@@ -64,7 +64,7 @@ under the License.
 注意其中 jdk 和 nodejs 都需要下载对应的 aarch64 版本：
 
 1. [Java8-aarch64](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/jdk-8u291-linux-aarch64.tar.gz)
-2. [Node v12.13.0-aarch64](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v16.3.0-linux-arm64.tar.xz)
+2. [Node v16.3.0-aarch64](https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/node-v16.3.0-linux-arm64.tar.xz)
 
 ## CentOS & Ubuntu
 
@@ -585,11 +585,11 @@ USE_AVX2=OFF sh build.sh
 
    - 问题原因
 
-     未使用 Ldb-Toolschain 进行编译
+     未使用 ldb-toolchain 进行编译
 
    - 解决方案
 
-     - 检查 Ldb-Toolschain 环境变量是否配置
+     - 检查 ldb-toolchain 环境变量是否配置
      - 查看 gcc 版本是否是 `gcc-11`
-     - 删除 `ldb-toolschain.sh` 脚本执行后的 ldb 目录，重新执行并配置环境变量，验证 gcc 版本
+     - 删除 `ldb_toolchain_gen.aarch64.sh` 脚本执行后的 ldb 目录，重新执行并配置环境变量，验证 gcc 版本
 

From 522e48c764ff2aa312cb7c9021cae5a75fb8cbb3 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:55:59 +0700
Subject: [PATCH 081/299] [fix](doc) spell errors fixes for be-olap-error-code
 docs (#27866)

---
 docs/en/docs/admin-manual/maint-monitor/be-olap-error-code.md | 2 +-
 .../docs/admin-manual/maint-monitor/be-olap-error-code.md     | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/docs/admin-manual/maint-monitor/be-olap-error-code.md b/docs/en/docs/admin-manual/maint-monitor/be-olap-error-code.md
index 08d66ad09b3c10..1a3a1bfa786967 100644
--- a/docs/en/docs/admin-manual/maint-monitor/be-olap-error-code.md
+++ b/docs/en/docs/admin-manual/maint-monitor/be-olap-error-code.md
@@ -96,7 +96,7 @@ under the License.
 | OLAP_ERR_CE_TRY_CE_LOCK_ERROR | -306 | Attempt to obtain execution command lock error |
 | Tablet error exception code | | |
 | OLAP_ERR_TABLE_VERSION_DUPLICATE_ERROR | -400 | Tablet copy version error |
-| OLAP_ERR_TABLE_VERSION_INDEX_MISMATCH_ERROR | -401 | teblet version index mismatch exception |
+| OLAP_ERR_TABLE_VERSION_INDEX_MISMATCH_ERROR | -401 | tablet version index mismatch exception |
 | OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR | -402 | The initial version of the tablet is not checked here, because if the BE is restarted during a schema-change of a tablet, we may encounter an empty tablet exception |
 | OLAP_ERR_TABLE_INDEX_FIND_ERROR | -403 | Unable to get the position of the first block or failure to find the last block of the block will cause this exception |
 | OLAP_ERR_TABLE_CREATE_FROM_HEADER_ERROR | -404 | This exception is triggered when the tablet cannot be loaded |
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/be-olap-error-code.md b/docs/zh-CN/docs/admin-manual/maint-monitor/be-olap-error-code.md
index 2609295da28004..e69871005152d4 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/be-olap-error-code.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/be-olap-error-code.md
@@ -104,8 +104,8 @@ under the License.
 | OLAP_ERR_CE_TABLET_ID_EXIST                      | -305   | tablet Id不存在错误                                          |
 | OLAP_ERR_CE_TRY_CE_LOCK_ERROR                    | -306   | 尝试获取执行命令锁错误                                       |
 | Tablet错误异常代码                               |        |                                                              |
-| OLAP_ERR_TABLE_VERSION_DUPLICATE_ERROR           | -400   | tablet副本版本错误                                           |
-| OLAP_ERR_TABLE_VERSION_INDEX_MISMATCH_ERROR      | -401   | teblet版本索引不匹配异常                                     |
+| OLAP_ERR_TABLE_VERSION_DUPLICATE_ERROR           | -400   | tablet 副本版本错误                                          |
+| OLAP_ERR_TABLE_VERSION_INDEX_MISMATCH_ERROR      | -401   | tablet 版本索引不匹配异常                                     |
 | OLAP_ERR_TABLE_INDEX_VALIDATE_ERROR              | -402   | 这里不检查tablet的初始版本，因为如果在一个tablet进行schema-change时重新启动 BE，我们可能会遇到空tablet异常 |
 | OLAP_ERR_TABLE_INDEX_FIND_ERROR                  | -403   | 无法获得第一个Block块位置 或者找到最后一行Block块失败会引发此异常 |
 | OLAP_ERR_TABLE_CREATE_FROM_HEADER_ERROR          | -404   | 无法加载Tablet的时候会触发此异常                             |

From c1303e9eeed506dc7587da965b439456b4a4020c Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:56:44 +0700
Subject: [PATCH 082/299] [fix](doc) example case fixed in best practices
 (#28156)

---
 docs/en/docs/data-table/best-practice.md    | 4 ++--
 docs/zh-CN/docs/data-table/best-practice.md | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/en/docs/data-table/best-practice.md b/docs/en/docs/data-table/best-practice.md
index c57488ffe9f004..2dda2d91a26b3d 100644
--- a/docs/en/docs/data-table/best-practice.md
+++ b/docs/en/docs/data-table/best-practice.md
@@ -164,10 +164,10 @@ Database Session
 session -u data (visitorid, sessionid, visittime, city, province, ip, browser, url)
 ```
 
-In addition to visitorid analysis, there are Brower and province analysis cases, Rollup can be established separately.
+In addition to visitorid analysis, there are browser and province analysis cases, Rollup can be established separately.
 
 ```
-ALTER TABLE session_data ADD ROLLUP rollup_brower(brower,province,ip,url) DUPLICATE KEY(brower,province);
+ALTER TABLE session_data ADD ROLLUP rollup_browser(browser,province,ip,url) DUPLICATE KEY(browser,province);
 ```
 
 ## Schema Change
diff --git a/docs/zh-CN/docs/data-table/best-practice.md b/docs/zh-CN/docs/data-table/best-practice.md
index 1c05b905fb3107..fc71426af1634c 100644
--- a/docs/zh-CN/docs/data-table/best-practice.md
+++ b/docs/zh-CN/docs/data-table/best-practice.md
@@ -160,13 +160,13 @@ ALTER TABLE site_visit ADD ROLLUP rollup_city(city, pv);
 如对于 session_data 表：
 
 ```text
-session_data(visitorid, sessionid, visittime, city, province, ip, brower, url)
+session_data(visitorid, sessionid, visittime, city, province, ip, browser, url)
 ```
 
-如果除了通过 visitorid 分析访问情况外，还有通过 brower, province 分析的情形，可以单独建立 Rollup。
+如果除了通过 visitorid 分析访问情况外，还有通过 browser, province 分析的情形，可以单独建立 Rollup。
 
 ```sql
-ALTER TABLE session_data ADD ROLLUP rollup_brower(brower,province,ip,url) DUPLICATE KEY(brower,province);
+ALTER TABLE session_data ADD ROLLUP rollup_browser(browser,province,ip,url) DUPLICATE KEY(browser,province);
 ```
 
 ## Schema Change

From d70743a2fca7ba609609844eb760fd051270d36c Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Wed, 27 Dec 2023 15:57:07 +0700
Subject: [PATCH 083/299] [fix](doc) spell errors fixes ha-action.md (#28155)

---
 docs/en/docs/admin-manual/http-actions/fe/ha-action.md    | 2 +-
 docs/zh-CN/docs/admin-manual/http-actions/fe/ha-action.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/docs/admin-manual/http-actions/fe/ha-action.md b/docs/en/docs/admin-manual/http-actions/fe/ha-action.md
index af67d6a2e4df7f..238457590745a4 100644
--- a/docs/en/docs/admin-manual/http-actions/fe/ha-action.md
+++ b/docs/en/docs/admin-manual/http-actions/fe/ha-action.md
@@ -68,7 +68,7 @@ None
 			"Value": "name: 192.168.1.1_9213_1597652404352, role: FOLLOWER, 192.168.1.1:9213",
 			"Name": "192.168.1.1_9213_1597652404352"
 		}],
-		"removedFronteds": [],
+		"removedFrontends": [],
 		"CanRead": [{
 			"Value": true,
 			"Name": "Status"
diff --git a/docs/zh-CN/docs/admin-manual/http-actions/fe/ha-action.md b/docs/zh-CN/docs/admin-manual/http-actions/fe/ha-action.md
index 921890a6596e81..1d7e2f93960cbb 100644
--- a/docs/zh-CN/docs/admin-manual/http-actions/fe/ha-action.md
+++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/ha-action.md
@@ -68,7 +68,7 @@ HA Action 用于获取 FE 集群的高可用组信息。
 			"Value": "name: 192.168.1.1_9213_1597652404352, role: FOLLOWER, 192.168.1.1:9213",
 			"Name": "192.168.1.1_9213_1597652404352"
 		}],
-		"removedFronteds": [],
+		"removedFrontends": [],
 		"CanRead": [{
 			"Value": true,
 			"Name": "Status"

From 286af3bce976d16bc563717c533b3f1fd28c8a94 Mon Sep 17 00:00:00 2001
From: Mingyu Chen <morningman@163.com>
Date: Wed, 27 Dec 2023 18:51:01 +0800
Subject: [PATCH 084/299] [doc](file-cache) fix wrong config (#29156)

* [doc](file-cache) fix wrong config

* 2
---
 docs/en/docs/lakehouse/filecache.md    | 55 +++++++++++++-------------
 docs/zh-CN/docs/lakehouse/filecache.md | 53 +++++++++++++------------
 2 files changed, 55 insertions(+), 53 deletions(-)

diff --git a/docs/en/docs/lakehouse/filecache.md b/docs/en/docs/lakehouse/filecache.md
index fa9272e708887c..c2a0304f9132ec 100644
--- a/docs/en/docs/lakehouse/filecache.md
+++ b/docs/en/docs/lakehouse/filecache.md
@@ -53,42 +53,43 @@ SET GLOBAL enable_file_cache = true;
 > The File Cache is only applicable to external queries for files (such as Hive, Hudi). It has no effect on internal table queries, or non-file external queries (such as JDBC, Elasticsearch), etc.
 
 ### Configurations for BE
+
 Add settings to the BE node's configuration file `conf/be.conf`, and restart the BE node for the configuration to take effect.
 
-|  Parameter   | Description  |
-|  ---  | ---  |
-| `enable_file_cache`  | Whether to enable File Cache, default false |
-| `file_cache_path` | Parameters about cache path, json format, for exmaple: `[{"path": "/path/to/file_cache1", "total_size":53687091200,"query_limit": "10737418240"},{"path": "/path/to/file_cache2", "total_size":53687091200,"query_limit": "10737418240"},{"path": "/path/to/file_cache3", "total_size":53687091200,"query_limit": "10737418240"}]`. `path` is the path to save cached data; `total_size` is the max size of cached data; `query_limit` is the max size of cached data for a single query. |
-| `file_cache_min_file_segment_size` | Min size of a single cached block, default 1MB, should greater than 4096 |
-| `file_cache_max_file_segment_size` | Max size of a single cached block, default 4MB, should greater than 4096 |
-| `enable_file_cache_query_limit` | Whether to limit the cache size used by a single query, default false |
-| `clear_file_cache` | Whether to delete the previous cache data when the BE restarts, default false |
+|  Parameter | Required  | Description  |
+|  ---  | ---  | --- |
+| `enable_file_cache` | Yes | Whether to enable File Cache, default false |
+| `file_cache_path` | Yes | Parameters about cache path, json format, for exmaple: `[{"path": "/path/to/file_cache1", "total_size":53687091200,"query_limit": 10737418240},{"path": "/path/to/file_cache2", "total_size":53687091200,"query_limit": 10737418240},{"path": "/path/to/file_cache3", "total_size":53687091200,"query_limit": 10737418240}]`. `path` is the path to save cached data; `total_size` is the max size of cached data; `query_limit` is the max size of cached data for a single query. |
+| `file_cache_min_file_segment_size` | No | Min size of a single cached block, default 1MB, should greater than 4096 |
+| `file_cache_max_file_segment_size` | No | Max size of a single cached block, default 4MB, should greater than 4096 |
+| `enable_file_cache_query_limit` | No | Whether to limit the cache size used by a single query, default false |
+| `clear_file_cache` | No | Whether to delete the previous cache data when the BE restarts, default false |
 
 ## Check whether a query hits cache
 
 Execute `set enable_profile = true` to enable the session variable, and you can view the query profile in the Queris tab of FE's web page. The metrics related to File Cache are as follows:
 
 ```
--  FileCache:
-  -  IOHitCacheNum:  552
-  -  IOTotalNum:  835
-  -  ReadFromFileCacheBytes:  19.98  MB
-  -  ReadFromWriteCacheBytes:  0.00  
-  -  ReadTotalBytes:  29.52  MB
-  -  WriteInFileCacheBytes:  915.77  MB
-  -  WriteInFileCacheNum:  283 
+-  FileCache:  0ns
+    -  BytesScannedFromCache:  2.02  GB
+    -  BytesScannedFromRemote:  0.00  
+    -  BytesWriteIntoCache:  0.00  
+    -  LocalIOUseTimer:  2s723ms
+    -  NumLocalIOTotal:  444
+    -  NumRemoteIOTotal:  0
+    -  NumSkipCacheIOTotal:  0
+    -  RemoteIOUseTimer:  0ns
+    -  WriteCacheIOUseTimer:  0ns
 ```
 
-- `IOTotalNum`:  Number of remote access
-- `IOHitCacheNum`: Number of cache hits
-- `ReadFromFileCacheBytes`: Amount of data read from cache file
-- `ReadTotalBytes`: Total amount of data read
-- `SkipCacheBytes`: Failed to create the cache file, or the cache file was deleted. The amount of data that needs to be read from the remote again
-- `WriteInFileCacheBytes`: Amount of data saved to cache file
-- `WriteInFileCacheNum`: The number of blocks saved, so 'WriteInFileCacheBytes'/' WriteInFileCacheBytes' is the average size of blocks
-
-`IOHitCacheNum` / `IOTotalNum` Equal to 1, indicating that read data only from file cache
+- `BytesScannedFromCache`: The amount of data read from the local cache.
+- `BytesScannedFromRemote`: The amount of data read from the remote end.
+- `BytesWriteIntoCache`: The amount of data written to the cache.
+- `LocalIOUseTimer`: Local cached IO time.
+- `RemoteIOUseTimer`: IO time for remote reading.
+- `NumLocalIOTotal`: The number of locally cached IOs.
+- `NumRemoteIOTotal`: Number of remote IOs.
+- `WriteCacheIOUseTimer`: IO time to write cache.
 
-`ReadFromFileCacheBytes` / `ReadTotalBytes` Equal to 1, indicating that read data only from file cache
+If `BytesScannedFromRemote` is 0, it means all caches are hit.
 
-`ReadFromFileCacheBytes` The smaller the better, the smaller the amount of data read from remote
diff --git a/docs/zh-CN/docs/lakehouse/filecache.md b/docs/zh-CN/docs/lakehouse/filecache.md
index 79035141e2de55..d153c811f9df90 100644
--- a/docs/zh-CN/docs/lakehouse/filecache.md
+++ b/docs/zh-CN/docs/lakehouse/filecache.md
@@ -57,39 +57,40 @@ SET GLOBAL enable_file_cache = true;
 
 添加参数到 BE 节点的配置文件 conf/be.conf 中，并重启 BE 节点让配置生效。
 
-|  参数   | 说明  |
-|  ---  | ---  |
-| `enable_file_cache`  | 是否启用 File Cache，默认 false |
-| `file_cache_path` | 缓存目录的相关配置，json格式，例子: `[{"path": "/path/to/file_cache1", "total_size":53687091200,"query_limit": "10737418240"},{"path": "/path/to/file_cache2", "total_size":53687091200,"query_limit": "10737418240"},{"path": "/path/to/file_cache3", "total_size":53687091200,"query_limit": "10737418240"}]`。`path` 是缓存的保存路径，`total_size` 是缓存的大小上限，`query_limit` 是单个查询能够使用的最大缓存大小。 |
-| `file_cache_min_file_segment_size` | 单个 Block 的大小下限，默认 1MB，需要大于 4096 |
-| `file_cache_max_file_segment_size` | 单个 Block 的大小上限，默认 4MB，需要大于 4096 |
-| `enable_file_cache_query_limit` | 是否限制单个 query 使用的缓存大小，默认 false |
-| `clear_file_cache` | BE 重启时是否删除之前的缓存数据，默认 false |
+|  参数   | 必选项 | 说明  |
+|  ---  | ---  | --- |
+| `enable_file_cache`  | 是 | 是否启用 File Cache，默认 false |
+| `file_cache_path` | 是 | 缓存目录的相关配置，json格式，例子: `[{"path": "/path/to/file_cache1", "total_size":53687091200,"query_limit": 10737418240},{"path": "/path/to/file_cache2", "total_size":53687091200,"query_limit": 10737418240},{"path": "/path/to/file_cache3", "total_size":53687091200,"query_limit": 10737418240}]`。`path` 是缓存的保存路径，`total_size` 是缓存的大小上限，`query_limit` 是单个查询能够使用的最大缓存大小。 |
+| `file_cache_min_file_segment_size` | 否 | 单个 Block 的大小下限，默认 1MB，需要大于 4096 |
+| `file_cache_max_file_segment_size` | 否 | 单个 Block 的大小上限，默认 4MB，需要大于 4096 |
+| `enable_file_cache_query_limit` | 否 | 是否限制单个 query 使用的缓存大小，默认 false |
+| `clear_file_cache` | 否 | BE 重启时是否删除之前的缓存数据，默认 false |
 
 ### 查看 File Cache 命中情况
 
 执行 `set enable_profile=true` 打开会话变量，可以在 FE 的 web 页面的 Queris 标签中查看到作业的 Profile。File Cache 相关的指标如下:
 
 ```
--  FileCache:
-  -  IOHitCacheNum:  552
-  -  IOTotalNum:  835
-  -  ReadFromFileCacheBytes:  19.98  MB
-  -  ReadFromWriteCacheBytes:  0.00  
-  -  ReadTotalBytes:  29.52  MB
-  -  WriteInFileCacheBytes:  915.77  MB
-  -  WriteInFileCacheNum:  283 
+-  FileCache:  0ns
+    -  BytesScannedFromCache:  2.02  GB
+    -  BytesScannedFromRemote:  0.00  
+    -  BytesWriteIntoCache:  0.00  
+    -  LocalIOUseTimer:  2s723ms
+    -  NumLocalIOTotal:  444
+    -  NumRemoteIOTotal:  0
+    -  NumSkipCacheIOTotal:  0
+    -  RemoteIOUseTimer:  0ns
+    -  WriteCacheIOUseTimer:  0ns
 ```
 
-- `IOTotalNum`:  远程访问的次数
-- `IOHitCacheNum`: 命中缓存的次数
-- `ReadFromFileCacheBytes`: 从缓存文件中读取的数据量
-- `ReadTotalBytes`: 总共读取的数据量
-- `SkipCacheBytes`: 创建缓存文件失败，或者缓存文件被删，需要再次从远程读取的数据量
-- `WriteInFileCacheBytes`: 保存到缓存文件中的数据量
-- `WriteInFileCacheNum`: 保存的 Block 数量，所以 `WriteInFileCacheBytes`/`WriteInFileCacheBytes` 为 Block 的平均大小
+- `BytesScannedFromCache`：从本地缓存中读取的数据量。
+- `BytesScannedFromRemote`：从远端读取的数据量。
+- `BytesWriteIntoCache`：写入缓存的数据量。
+- `LocalIOUseTimer`：本地缓存的 IO 时间。
+- `RemoteIOUseTimer`：远端读取的 IO 时间。
+- `NumLocalIOTotal`：本地缓存的 IO 次数。
+- `NumRemoteIOTotal`：远端 IO 次数。
+- `WriteCacheIOUseTimer`：写入缓存的 IO 时间。
 
-`IOHitCacheNum` / `IOTotalNum` 等于1，表示缓存完全命中
-
-`ReadFromFileCacheBytes` / `ReadTotalBytes` 等于1，表示缓存完全命中
+如果 `BytesScannedFromRemote` 为 0，表示全部命中缓存。
 

From 6d817bc253018027798a86c4e0584e0bc7bb8098 Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Wed, 27 Dec 2023 20:31:03 +0800
Subject: [PATCH 085/299] [fix](topn opt) avoid using topn runtime predicate
 which segment does not contain such column(column unique id) when pruning
 segment (#29148)

---
 be/src/olap/rowset/segment_v2/segment.cpp                    | 3 ++-
 .../scalar_types/test_key_topn_merge_block_columns.groovy    | 5 +++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index de2593447c5175..1f9956e5e692fe 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -150,7 +150,8 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
             AndBlockColumnPredicate and_predicate;
             auto single_predicate = new SingleColumnBlockPredicate(runtime_predicate.get());
             and_predicate.add_column_predicate(single_predicate);
-            if (can_apply_predicate_safely(runtime_predicate->column_id(), runtime_predicate.get(),
+            if (_column_readers.count(uid) >= 1 &&
+                can_apply_predicate_safely(runtime_predicate->column_id(), runtime_predicate.get(),
                                            *schema, read_options.io_ctx.reader_type) &&
                 !_column_readers.at(uid)->match_condition(&and_predicate)) {
                 // any condition not satisfied, return.
diff --git a/regression-test/suites/datatype_p0/scalar_types/test_key_topn_merge_block_columns.groovy b/regression-test/suites/datatype_p0/scalar_types/test_key_topn_merge_block_columns.groovy
index 92af65e09ac259..b917bd4214f797 100644
--- a/regression-test/suites/datatype_p0/scalar_types/test_key_topn_merge_block_columns.groovy
+++ b/regression-test/suites/datatype_p0/scalar_types/test_key_topn_merge_block_columns.groovy
@@ -73,4 +73,9 @@ suite("test_key_topn_merge_block_columns", "p0") {
 
     // will cause be core if bug not fix by #20820
     sql "SELECT /*+ SET_VAR(topn_opt_limit_threshold=1024) */ * FROM ${testTable} WHERE c_int = 100 or k1 = 1000 ORDER BY k1 LIMIT 10"
+    sql "insert into ${testTable} select * from ${testTable}"
+    sql "insert into ${testTable} select * from ${testTable}"
+    sql "insert into ${testTable} select * from ${testTable}"
+    sql """ alter table ${testTable} ADD COLUMN (new_k1 INT DEFAULT '1', new_k2 INT DEFAULT '2');"""
+    sql "select * from ${testTable} order by new_k1 limit 1"
 }

From 9715db61d48249fa900e3b5f42ef5386ae17d8c9 Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Wed, 27 Dec 2023 20:38:44 +0800
Subject: [PATCH 086/299] [FIX](complextype)fix count func with complex type
 (#28873)

---
 .../doris/analysis/FunctionCallExpr.java      |   25 +-
 gensrc/script/doris_builtins_functions.py     |    3 +-
 .../query/test_nested_type_with_count.out     | 1990 +++++++++++++++++
 .../query/test_nested_type_with_count.groovy  |  121 +
 4 files changed, 2136 insertions(+), 3 deletions(-)
 create mode 100644 regression-test/data/datatype_p0/nested_types/query/test_nested_type_with_count.out
 create mode 100644 regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index f38f984b7172f3..cb638482f8ccf7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1648,8 +1648,25 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
                 // now first find function in built-in functions
                 if (Strings.isNullOrEmpty(fnName.getDb())) {
                     Type[] childTypes = collectChildReturnTypes();
-                    fn = getBuiltinFunction(fnName.getFunction(), childTypes,
-                            Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+                    // when we call count<Array<T>> with nested type is not null type which is defined in FunctionSet
+                    // so here aim to make function signature to match builtln func we defined in fe code
+                    if (fnName.getFunction().equalsIgnoreCase("count") && childTypes.length > 0
+                            && childTypes[0].isComplexType()) {
+                        // get origin type to match builtln func
+                        Type[] matchFuncChildTypes = new Type[1];
+                        if (childTypes[0].isArrayType()) {
+                            matchFuncChildTypes[0] = Type.ARRAY;
+                        } else if (childTypes[0].isMapType()) {
+                            matchFuncChildTypes[0] = Type.MAP;
+                        } else if (childTypes[0].isStructType()) {
+                            matchFuncChildTypes[0] = Type.GENERIC_STRUCT;
+                        }
+                        fn = getBuiltinFunction(fnName.getFunction(), matchFuncChildTypes,
+                                Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+                    } else {
+                        fn = getBuiltinFunction(fnName.getFunction(), childTypes,
+                                Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+                    }
                 }
 
                 // find user defined functions
@@ -1888,6 +1905,10 @@ && collectChildReturnTypes()[0].isDecimalV3()) {
                     continue;
                 }
 
+                if (fnName.getFunction().equalsIgnoreCase("count") && args[i].isComplexType()) {
+                    continue;
+                }
+
                 if ((fnName.getFunction().equalsIgnoreCase("money_format") || fnName.getFunction()
                         .equalsIgnoreCase("histogram")
                         || fnName.getFunction().equalsIgnoreCase("hist"))
diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py
index 358327783746a0..5e45a1a78af057 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -93,7 +93,7 @@
         [['struct'], 'STRUCT<TYPES>', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['TYPES...']],
         [['named_struct'], 'ANY_STRUCT_TYPE', ['TYPES'], 'ALWAYS_NOT_NULLABLE', ['TYPES...']],
         [['struct_element'], 'ANY_ELEMENT_TYPE', ['ANY_STRUCT_TYPE', 'INT'], 'ALWAYS_NULLABLE', ['TYPES...']],
-        [['struct_element'], 'ANY_ELEMENT_TYPE', ['ANY_STRUCT_TYPE', 'VARCHAR'], 'ALWAYS_NULLABLE', ['TYPES...']]
+        [['struct_element'], 'ANY_ELEMENT_TYPE', ['ANY_STRUCT_TYPE', 'VARCHAR'], 'ALWAYS_NULLABLE', ['TYPES...']],
     ],
 
     # array functions
@@ -1551,6 +1551,7 @@
         [['null_or_empty'], 'BOOLEAN', ['VARCHAR'], 'ALWAYS_NOT_NULLABLE'],
         [['not_null_or_empty'], 'BOOLEAN', ['VARCHAR'], 'ALWAYS_NOT_NULLABLE'],
         [['space'], 'VARCHAR', ['INT'], ''],
+        [['repeat'], 'VARCHAR', ['VARCHAR', 'INT'], 'ALWAYS_NULLABLE'],
         [['lpad'], 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'], 'ALWAYS_NULLABLE'],
         [['rpad'], 'VARCHAR', ['VARCHAR', 'INT', 'VARCHAR'], 'ALWAYS_NULLABLE'],
         [['append_trailing_char_if_absent'], 'VARCHAR', ['VARCHAR', 'VARCHAR'], 'ALWAYS_NULLABLE'],
diff --git a/regression-test/data/datatype_p0/nested_types/query/test_nested_type_with_count.out b/regression-test/data/datatype_p0/nested_types/query/test_nested_type_with_count.out
new file mode 100644
index 00000000000000..03bb01a4b49492
--- /dev/null
+++ b/regression-test/data/datatype_p0/nested_types/query/test_nested_type_with_count.out
@@ -0,0 +1,1990 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_one_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_two_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
+-- !sql_three_level --
+0
+
diff --git a/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
new file mode 100644
index 00000000000000..2b5bf97795e66b
--- /dev/null
+++ b/regression-test/suites/datatype_p0/nested_types/query/test_nested_type_with_count.groovy
@@ -0,0 +1,121 @@
+import org.apache.commons.lang3.StringUtils
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_nested_type_with_count") {
+    // this test case aim to test nested type with old planner
+    sql """set enable_nereids_planner=false"""
+    sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false');"""
+
+    def table_names = ["test_array_one_level", "test_map_one_level", "test_struct_one_level"]
+
+    def colNameArr = ["c_bool", "c_tinyint", "c_smallint", "c_int", "c_bigint", "c_largeint", "c_float",
+                      "c_double", "c_decimal", "c_decimalv3", "c_date", "c_datetime", "c_datev2", "c_datetimev2",
+                      "c_char", "c_varchar", "c_string"]
+
+    // create tables
+    for (int i = 0; i < table_names.size(); ++i) {
+        sql """ DROP TABLE IF EXISTS ${table_names[i]} """
+        String result = create_table_with_nested_type(1, [i], table_names[i])
+        sql result
+        for (int j = 0; j < colNameArr.size(); ++j) {
+            qt_sql_one_level "select count(${colNameArr[j]}) from ${table_names[i]}"
+        }
+    }
+
+    // two-level
+    def table_names_level2 = [
+                                        "two_level_array_array",
+                                        "two_level_array_map",
+                                        "two_level_array_struct",
+
+                                        "two_level_map_array",
+                                        "two_level_map_map",
+                                        "two_level_map_struct",
+
+                                        "two_level_struct_array",
+                                        "two_level_struct_map",
+                                        "two_level_struct_struct"
+    ]
+    // create tables
+    // (0,0) (0,1) (0,2) (1,0) (1,1) (1,2) (2,0) (2,1) (2,2)
+    for (int i = 0; i < 3; ++i) {
+        for (int j = 0; j < 3; ++j) {
+            sql """ DROP TABLE IF EXISTS ${table_names_level2[i*3+j]} """
+            String result = create_table_with_nested_type(2, [i, j], table_names_level2[i*3+j])
+            sql result
+            for (int c = 0; c < colNameArr.size(); ++c) {
+                qt_sql_two_level "select count(${colNameArr[c]}) from ${table_names_level2[i*3+j]}"
+            }
+        }
+    }
+
+    // three-level
+    def table_names_level3 = [
+            "three_level_array_array_array",
+            "three_level_array_array_map",
+            "three_level_array_array_struct",
+            "three_level_array_map_array",
+            "three_level_array_map_map",
+            "three_level_array_map_struct",
+            "three_level_array_struct_array",
+            "three_level_array_struct_map",
+            "three_level_array_struct_struct",
+
+            "three_level_map_array_array",
+            "three_level_map_array_map",
+            "three_level_map_array_struct",
+            "three_level_map_map_array",
+            "three_level_map_map_map",
+            "three_level_map_map_struct",
+            "three_level_map_struct_array",
+            "three_level_map_struct_map",
+            "three_level_map_struct_struct",
+
+            "three_level_struct_array_array",
+            "three_level_struct_array_map",
+            "three_level_struct_array_struct",
+            "three_level_struct_map_array",
+            "three_level_struct_map_map",
+            "three_level_struct_map_struct",
+            "three_level_struct_struct_array",
+            "three_level_struct_struct_map",
+            "three_level_struct_struct_struct"
+    ]
+
+    // create tables
+    // array (0,0,0) (0,0,1) (0,0,2) (0,1,0) (0,1,1) (0,1,2) (0,2,0) (0,2,1) (0,2,2)
+    // map (1,0,0) (1,0,1) (1,0,2) (1,1,0) (1,1,1) (1,1,2) (1,2,0) (1,2,1) (1,2,2)
+    // struct (2,0,0) (2,0,1) (2,0,2) (2,1,0) (2,1,1) (2,1,2) (2,2,0) (2,2,1) (2,2,2)
+    def table_idx = 0
+    for (int i = 0; i < 3; ++i) {
+        for (int j = 0; j < 3; ++j) {
+            for ( int k = 0; k < 3; ++k) {
+                sql """ DROP TABLE IF EXISTS ${table_names_level3[table_idx]} """
+                String result = create_table_with_nested_type(3, [i, j, k], table_names_level3[table_idx])
+                sql result
+                for (int c = 0; c < colNameArr.size(); ++c) {
+                    qt_sql_three_level "select count(${colNameArr[c]}) from ${table_names_level3[table_idx]}"
+                }
+                ++ table_idx
+            }
+        }
+    }
+
+
+}

From 5f71691401b0096e0ca2cf460337b72df10e0a2e Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Wed, 27 Dec 2023 20:58:02 +0800
Subject: [PATCH 087/299] [fix](read) fix unexpected overflow of uninitialized
 column data in VStatisticsIterator::next_batch (#29141)

---
 be/src/vec/olap/vgeneric_iterators.cpp                      | 2 +-
 .../data/datatype_p0/decimalv3/test_decimalv3.out           | 6 ++++++
 .../suites/datatype_p0/decimalv3/test_decimalv3.groovy      | 5 +++++
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/be/src/vec/olap/vgeneric_iterators.cpp b/be/src/vec/olap/vgeneric_iterators.cpp
index 155bb4ee076be8..0e97f1d1720ec7 100644
--- a/be/src/vec/olap/vgeneric_iterators.cpp
+++ b/be/src/vec/olap/vgeneric_iterators.cpp
@@ -76,7 +76,7 @@ Status VStatisticsIterator::next_batch(Block* block) {
         if (_push_down_agg_type_opt == TPushAggOp::COUNT) {
             size = std::min(_target_rows - _output_rows, MAX_ROW_SIZE_IN_COUNT);
             for (int i = 0; i < columns.size(); ++i) {
-                columns[i]->resize(size);
+                columns[i]->insert_many_defaults(size);
             }
         } else {
             for (int i = 0; i < columns.size(); ++i) {
diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimalv3.out b/regression-test/data/datatype_p0/decimalv3/test_decimalv3.out
index 026424e76f867b..b137c2c882d72e 100644
--- a/regression-test/data/datatype_p0/decimalv3/test_decimalv3.out
+++ b/regression-test/data/datatype_p0/decimalv3/test_decimalv3.out
@@ -5,6 +5,12 @@
 -- !decimalv3 --
 100.00000000000000000000
 
+-- !decimalv3_view1 --
+1
+
+-- !decimalv3_view2 --
+1
+
 -- !aEb_test1 --
 0
 
diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3.groovy
index f0535ad86892fe..2bcdc58f4b7d48 100644
--- a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3.groovy
+++ b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3.groovy
@@ -27,6 +27,11 @@ suite("test_decimalv3") {
 
     qt_decimalv3 "select * from test5_v"
     qt_decimalv3 "select cast(a as decimalv3(12,10)) * cast(b as decimalv3(18,10)) from test5"
+    qt_decimalv3_view1 "select count(*) from test5_v;"
+
+    sql "drop view if exists test5_v2"
+    sql "create view test5_v2 (amout) as select cast(a as decimalv3(18,6)) from test5"
+    qt_decimalv3_view2 "select count(*) from test5_v2;"
 
     /*
     sql "drop table if exists test_decimal256;"

From 0436013baf3620a8a5842afa01d9da45e0a988f1 Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Wed, 27 Dec 2023 20:58:37 +0800
Subject: [PATCH 088/299] [fix](decimal) fix cast decimal overflow and add test
 cases for casting decimalv2 to decimalv3 (#29165)

---
 be/src/vec/functions/function_cast.h          |    4 +-
 .../decimalv3/test_decimalv2_v3.out           |  302 ++++
 .../decimalv3/test_decimalv3_cast.out         |    8 +-
 .../decimalv3/test_decimalv2_v3.groovy        | 1227 +++++++++++++++++
 .../decimalv3/test_decimalv3_cast.groovy      |   37 +
 5 files changed, 1573 insertions(+), 5 deletions(-)
 create mode 100644 regression-test/data/datatype_p0/decimalv3/test_decimalv2_v3.out
 create mode 100644 regression-test/suites/datatype_p0/decimalv3/test_decimalv2_v3.groovy

diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 2e1e48db14d3d3..474851e80d104d 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -322,8 +322,10 @@ struct ConvertImpl {
             vec_to.resize(size);
 
             if constexpr (IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>) {
+                // the result is rounded when doing cast, so it may still overflow after rounding
+                // if destination integer digit count is the same as source integer digit count.
                 bool narrow_integral = context->check_overflow_for_decimal() &&
-                                       (to_precision - to_scale) < (from_precision - from_scale);
+                                       (to_precision - to_scale) <= (from_precision - from_scale);
 
                 bool multiply_may_overflow = context->check_overflow_for_decimal();
                 if (to_scale > from_scale) {
diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimalv2_v3.out b/regression-test/data/datatype_p0/decimalv3/test_decimalv2_v3.out
new file mode 100644
index 00000000000000..b42b05d3e1d6ff
--- /dev/null
+++ b/regression-test/data/datatype_p0/decimalv3/test_decimalv2_v3.out
@@ -0,0 +1,302 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !cast_v2_to_v3_1 --
+1234.5670
+
+-- !cast_v2_to_v3_2 --
+1234.5670
+
+-- !cast_v2_to_v3_3 --
+1234.5670
+
+-- !cast_v2_to_v3_4 --
+1234.5670
+
+-- !cast32_select_all --
+99999.9999
+
+-- !cast32_wider_scale_1 --
+99999.99990
+
+-- !cast32_wider_scale_2 --
+99999.99990
+
+-- !cast32_wider_scale_3 --
+99999.99990
+
+-- !cast32_wider_scale_4 --
+99999.99990
+
+-- !cast32_select_all2 --
+9999.9999
+
+-- !cast32_wider_scale_5 --
+9999.99990
+
+-- !cast32_select_all3 --
+10000.0001
+
+-- !cast32_same_scale_1 --
+99999.9999
+
+-- !cast32_same_scale_2 --
+99999.9999
+
+-- !cast32_same_scale_3 --
+99999.9999
+
+-- !cast32_same_scale_4 --
+99999.9999
+
+-- !cast32_same_scale_5 --
+99999.9999
+
+-- !cast32_same_scale_6 --
+9999.9999
+
+-- !cast32_narrow_scale_1 --
+10000.000
+
+-- !cast32_narrow_scale_2 --
+10000.000
+
+-- !cast32_narrow_scale_3 --
+10000.000
+
+-- !cast32_narrow_scale_4 --
+10000.000
+
+-- !cast32_narrow_scale_5 --
+10000.000
+
+-- !cast32_narrow_scale_6 --
+9999.999
+
+-- !cast32_negative_wider_scale_1 --
+-99999.99990
+
+-- !cast32_negative_wider_scale_2 --
+-99999.99990
+
+-- !cast32_negative_wider_scale_3 --
+-99999.99990
+
+-- !cast32_negative_wider_scale_4 --
+-99999.99990
+
+-- !cast32_negative_wider_scale_5 --
+-9999.99990
+
+-- !cast32_2_wider_scale_1 --
+99.99990
+
+-- !cast32_2_wider_scale_2 --
+99.99990
+
+-- !cast32_2_wider_scale_2 --
+99.99990
+
+-- !cast32_2_wider_scale_3 --
+99.99990
+
+-- !cast32_wider_scale_4 --
+99.99990
+
+-- !cast32_wider_scale_5 --
+9.99990
+
+-- !cast32_2_same_scale_1 --
+99.9999
+
+-- !cast32_2_same_scale_2 --
+99.9999
+
+-- !cast32_2_same_scale_3 --
+99.9999
+
+-- !cast32_2_same_scale_4 --
+99.9999
+
+-- !cast32_2_same_scale_5 --
+99.9999
+
+-- !cast32_2_same_scale_6 --
+9.9999
+
+-- !cast32_2_narrow_scale_1 --
+100.000
+
+-- !cast32_2_narrow_scale_2 --
+100.000
+
+-- !cast32_2_narrow_scale_3 --
+100.000
+
+-- !cast32_2_narrow_scale_4 --
+100.000
+
+-- !cast32_2_narrow_scale_6 --
+9.999
+
+-- !cast32_3_to_much_bigger_scale_1 --
+999999999.000000000
+
+-- !cast32_3_to_much_bigger_scale_2 --
+999999999.0000000000
+
+-- !cast32_3_to_much_bigger_scale_3 --
+999999999.00000000000000000000000000000
+
+-- !cast32_3_to_much_bigger_scale_4 --
+999999999.0000000000000000000000000000000000000000000000000000000000000000000
+
+-- !cast64_select_all --
+999999999999.999999
+
+-- !cast64_wider_scale_1 --
+999999999999.9999990
+
+-- !cast64_wider_scale_2 --
+999999999999.9999990
+
+-- !cast64_wider_scale_3 --
+999999999999.9999990
+
+-- !cast64_wider_scale_4 --
+999999999999.9999990
+
+-- !cast64_wider_scale_5 --
+99999999999.9999990
+
+-- !cast64_wider_scale_6 --
+99.9999990
+
+-- !cast64_wider_scale_7 --
+9.9999990
+
+-- !cast64_wider_scale_8 --
+9.9999990
+
+-- !cast64_same_scale_1 --
+999999999999.999999
+
+-- !cast64_same_scale_2 --
+999999999999.999999
+
+-- !cast64_same_scale_3 --
+999999999999.999999
+
+-- !cast64_same_scale_4 --
+999999999999.999999
+
+-- !cast64_same_scale_5 --
+99999999999.999999
+
+-- !cast64_same_scale_6 --
+999.999999
+
+-- !cast64_same_scale_7 --
+99.999999
+
+-- !cast64_same_scale_8 --
+99.999999
+
+-- !cast64_narrow_scale_1 --
+1000000000000.00000
+
+-- !cast64_narrow_scale_2 --
+1000000000000.00000
+
+-- !cast64_narrow_scale_3 --
+1000000000000.00000
+
+-- !cast64_narrow_scale_5 --
+99999999999.99999
+
+-- !cast64_narrow_scale_6 --
+9999.99999
+
+-- !cast64_narrow_scale_7 --
+1000.00000
+
+-- !cast64_narrow_scale_8 --
+999.99999
+
+-- !cast64_narrow_scale_9 --
+999.99999
+
+-- !cast128_wider_scale_1 --
+-999999999999999999.9999999990
+999999999999999999.9999999990
+
+-- !cast128_wider_scale_2 --
+-999999999999999999.9999999990
+999999999999999999.9999999990
+
+-- !cast128_wider_scale_3 --
+-999999999999999999.9999999990
+999999999999999999.9999999990
+
+-- !cast128_wider_scale_4 --
+-999999999999999999.9999999990
+999999999999999999.9999999990
+
+-- !cast128_wider_scale_5 --
+-999999999999999999.9999999990
+999999999999999999.9999999990
+
+-- !cast128_wider_scale_6 --
+-99999999999999999.9999999990
+99999999999999999.9999999990
+
+-- !cast128_wider_scale_7 --
+-99999999.9999999990
+99999999.9999999990
+
+-- !cast128_same_scale_1 --
+-999999999999999999.999999999
+999999999999999999.999999999
+
+-- !cast128_same_scale_2 --
+-999999999999999999.999999999
+999999999999999999.999999999
+
+-- !cast128_same_scale_3 --
+-999999999999999999.999999999
+999999999999999999.999999999
+
+-- !cast128_same_scale_4 --
+-999999999999999999.999999999
+999999999999999999.999999999
+
+-- !cast128_same_scale_5 --
+-99999999999999999.999999999
+99999999999999999.999999999
+
+-- !cast128_same_scale_6 --
+-999999999.999999999
+999999999.999999999
+
+-- !cast128_same_scale_7 --
+-999999999.999999999
+999999999.999999999
+
+-- !cast128_narrow_scale_1 --
+-1000000000000000000.00000000
+1000000000000000000.00000000
+
+-- !cast128_narrow_scale_2 --
+-1000000000000000000.00000000
+1000000000000000000.00000000
+
+-- !cast128_narrow_scale_3 --
+-1000000000000000000.00000000
+1000000000000000000.00000000
+
+-- !cast128_narrow_scale_4 --
+-999999999999999999.99999999
+999999999999999999.99999999
+
+-- !cast128_narrow_scale_5 --
+-99999999999999999.99999999
+99999999999999999.99999999
+
diff --git a/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast.out b/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast.out
index 68bc757d5c3dce..5bec55ddf66d54 100644
--- a/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast.out
+++ b/regression-test/data/datatype_p0/decimalv3/test_decimalv3_cast.out
@@ -138,7 +138,7 @@
 100.000
 
 -- !cast32_2_narrow_scale_5 --
-100.000
+99.999
 
 -- !cast32_2_narrow_scale_6 --
 9.999
@@ -216,7 +216,7 @@
 1000000000000.00000
 
 -- !cast64_narrow_scale_4 --
-1000000000000.00000
+999999999999.99999
 
 -- !cast64_narrow_scale_5 --
 99999999999.99999
@@ -314,8 +314,8 @@
 100000000000000000000000000000000.00000
 
 -- !cast128_narrow_scale_3 --
--100000000000000000000000000000000.00000
-100000000000000000000000000000000.00000
+-99999999999999999999999999999999.99999
+99999999999999999999999999999999.99999
 
 -- !cast128_narrow_scale_4 --
 -9999999999999999999999999999999.99999
diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimalv2_v3.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimalv2_v3.groovy
new file mode 100644
index 00000000000000..b44d04a9c3814b
--- /dev/null
+++ b/regression-test/suites/datatype_p0/decimalv3/test_decimalv2_v3.groovy
@@ -0,0 +1,1227 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_decimalv2_v3", "nonConcurrent") {
+    sql """
+        admin set frontend config("enable_decimal_conversion" = "false");
+    """
+
+    // test cast decimalv2 to decimal32
+    def prepare_test_decimalv2_v3_1 = {
+        sql " drop table if exists test_decimalv2_v3_1; "
+        sql """
+            create table test_decimalv2_v3_1 (
+                k1 decimalv2(7, 3)
+            ) distributed by hash(k1)
+            properties (
+                "replication_num" = "1"
+            );
+        """
+    }
+
+    prepare_test_decimalv2_v3_1 ()
+    sql """
+        insert into test_decimalv2_v3_1 values (1234.567);
+    """
+
+    // old type 4.3
+    //////////////////////////////////////////
+    // cast to wider scale: x.4
+    //////////////////////////////////////////
+
+    //======= wider integral: 5.4
+    qt_cast_v2_to_v3_1 """
+        select cast(k1 as decimalv3(9,4)) from test_decimalv2_v3_1 order by 1;
+    """
+    qt_cast_v2_to_v3_2 """
+        select cast(k1 as decimalv3(18,4)) from test_decimalv2_v3_1 order by 1;
+    """
+    qt_cast_v2_to_v3_3 """
+        select cast(k1 as decimalv3(38,4)) from test_decimalv2_v3_1 order by 1;
+    """
+
+    //======= same integral: 4.4
+    qt_cast_v2_to_v3_4 """
+        select cast(k1 as decimalv3(8,4)) from test_decimalv2_v3_1 order by 1;
+    """
+
+    //======= narrow integral: 3.4
+    test {
+        // test multiply result overflow
+        sql """
+        select cast(k1 as decimalv3(7,4)) from test_decimalv2_v3_1 order by 1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    sql "drop table test_decimalv2_v3_1; "
+
+    // 5.4
+    // cast decimalv2 who's precision is within decimal32
+    def prepare_test_decimalv2_v3_decimal32_cast1 = {
+        sql "drop table if exists test_decimalv2_v3_decimal32_cast1;"
+        sql """
+            CREATE TABLE test_decimalv2_v3_decimal32_cast1(
+              k1 decimalv2(9, 4)
+            )
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 4
+            PROPERTIES (
+            "replication_num" = "1"
+            );
+        """
+    }
+    prepare_test_decimalv2_v3_decimal32_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (99999.9999);
+    """
+    qt_cast32_select_all """
+        select * from test_decimalv2_v3_decimal32_cast1 order by 1;
+    """
+
+    // old type: 5.4
+    //////////////////////////////////////////
+    // cast to wider scale: x.5
+    //////////////////////////////////////////
+
+    //======= wider integral: 6.5
+    qt_cast32_wider_scale_1 """
+        select cast(k1 as decimalv3(11, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    qt_cast32_wider_scale_2 """
+        select cast(k1 as decimalv3(38, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_wider_scale_3 """
+        select cast(k1 as decimalv3(76, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 5.4
+    //======= same integral: 5.5
+    qt_cast32_wider_scale_4 """
+        select cast(k1 as decimalv3(10, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+
+    //======= narrow integral: 4.5
+    test {
+        // test multiply result overflow
+        sql """
+            select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal32_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    // test multiply result not overflow
+    prepare_test_decimalv2_v3_decimal32_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (9999.9999);
+    """
+    qt_cast32_select_all2 """
+        select * from test_decimalv2_v3_decimal32_cast1 order by 1;
+    """
+    qt_cast32_wider_scale_5 """
+        select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+
+    // old type: 5.4
+    // test multiply result not overflow, but cast result overflow
+    prepare_test_decimalv2_v3_decimal32_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (10000.0001);
+    """
+    qt_cast32_select_all3 """
+        select * from test_decimalv2_v3_decimal32_cast1 order by 1;
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal32_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    prepare_test_decimalv2_v3_decimal32_cast1();
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (99999.9999);
+    """
+
+    // old type: 5.4
+    //////////////////////////////////////////
+    // cast to same scale: x.4
+    //////////////////////////////////////////
+
+    //======= wider integral: 6.4
+    qt_cast32_same_scale_1 """
+        select cast(k1 as decimalv3(10, 4)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    qt_cast32_same_scale_2 """
+        select cast(k1 as decimalv3(18, 4)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    qt_cast32_same_scale_3 """
+        select cast(k1 as decimalv3(38, 4)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_same_scale_4 """
+        select cast(k1 as decimalv3(76, 4)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=false;"
+
+    //======= same integral: 5.4
+    qt_cast32_same_scale_5 """
+        select cast(k1 as decimalv3(9, 4)) from test_decimalv2_v3_decimal32_cast1;
+    """
+
+    // old type: 5.4
+    // cast to same scale: x.4
+    //======= narrow integral: 4.4
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 4)) from test_decimalv2_v3_decimal32_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // not overflow
+    prepare_test_decimalv2_v3_decimal32_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (9999.9999);
+    """
+    qt_cast32_same_scale_6 """
+        select cast(k1 as decimalv3(8, 4)) from test_decimalv2_v3_decimal32_cast1;
+    """
+
+    // old type: 5.4
+    //////////////////////////////////////////
+    // cast to narrow scale: x.3
+    //////////////////////////////////////////
+
+    //======= wider integral: 6.3
+    qt_cast32_narrow_scale_1 """
+        select cast(k1 as decimalv3(9, 3)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    qt_cast32_narrow_scale_2 """
+        select cast(k1 as decimalv3(18, 3)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    qt_cast32_narrow_scale_3 """
+        select cast(k1 as decimalv3(38, 3)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_narrow_scale_4 """
+        select cast(k1 as decimalv3(76, 3)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 5.4
+    // cast to narrow scale: x.3
+    //======= same integral: 5.3
+    qt_cast32_narrow_scale_5 """
+        select cast(k1 as decimalv3(8, 3)) from test_decimalv2_v3_decimal32_cast1;
+    """
+
+    // old type: 5.4
+    // cast to narrow scale: x.3
+    //======= narrow integral: 4.3
+    // integral overflow: overflow after round
+    test {
+        sql """
+            select cast(k1 as decimalv3(7, 3)) from test_decimalv2_v3_decimal32_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // integral not overflow
+    prepare_test_decimalv2_v3_decimal32_cast1();
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (9999.9989);
+    """
+    qt_cast32_narrow_scale_6 """
+        select cast(k1 as decimalv3(7, 3)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "drop table test_decimalv2_v3_decimal32_cast1; "
+
+
+    // cast negative decimal32
+    prepare_test_decimalv2_v3_decimal32_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (-99999.9999);
+    """
+    // old type: 5.4
+    //////////////////////////////////////////
+    // cast to wider scale: x.5
+    //////////////////////////////////////////
+    //======= wider integral: 6.5
+    qt_cast32_negative_wider_scale_1 """
+        select cast(k1 as decimalv3(11, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    qt_cast32_negative_wider_scale_2 """
+        select cast(k1 as decimalv3(38, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_negative_wider_scale_3 """
+        select cast(k1 as decimalv3(76, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+    sql "set enable_decimal256=false;"
+
+    //======= same integral: 5.5
+    qt_cast32_negative_wider_scale_4 """
+        select cast(k1 as decimalv3(10, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+
+    //======= narrow integral: 4.5
+    test {
+        // test multiply result overflow
+        sql """
+            select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal32_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    // test multiply result not overflow
+    prepare_test_decimalv2_v3_decimal32_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (-9999.9999);
+    """
+    qt_cast32_negative_wider_scale_5 """
+        select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal32_cast1;
+    """
+
+    // old type: 5.4
+    // cast to wider scale: x.5
+    // narrow integral: 4.5
+    // test multiply result not overflow, but cast result overflow
+    prepare_test_decimalv2_v3_decimal32_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast1 values (-10000.0001);
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal32_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    sql "drop table test_decimalv2_v3_decimal32_cast1; "
+
+    def prepare_test_decimalv2_v3_decimal32_cast2 = {
+        // type: 2.4
+        sql "drop table if exists test_decimalv2_v3_decimal32_cast2;"
+        sql """
+            CREATE TABLE test_decimalv2_v3_decimal32_cast2(
+              k1 decimalv2(6, 4)
+            )
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 4
+            PROPERTIES (
+            "replication_num" = "1"
+            );
+        """
+    }
+    prepare_test_decimalv2_v3_decimal32_cast2()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast2 values (99.9999);
+    """
+
+    // old type: 2.4
+    //////////////////////////////////////////
+    // cast to wider scale: x.5
+    //////////////////////////////////////////
+
+    //======= wider integral: 3.5
+    qt_cast32_2_wider_scale_1 """
+        select cast(k1 as decimalv3(8, 5)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    qt_cast32_2_wider_scale_2 """
+        select cast(k1 as decimalv3(18, 5)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    qt_cast32_2_wider_scale_2 """
+        select cast(k1 as decimalv3(38, 5)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_2_wider_scale_3 """
+        select cast(k1 as decimalv3(76, 5)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 2.4
+    //======= same integral: 2.5
+    qt_cast32_wider_scale_4 """
+        select cast(k1 as decimalv3(7, 5)) from test_decimalv2_v3_decimal32_cast2;
+    """
+
+    //======= narrow integral: 1.5
+    // integral part overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(6, 5)) from test_decimalv2_v3_decimal32_cast2;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // integral part not overflow
+    prepare_test_decimalv2_v3_decimal32_cast2()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast2 values (9.9999);
+    """
+    qt_cast32_wider_scale_5 """
+        select cast(k1 as decimalv3(6, 5)) from test_decimalv2_v3_decimal32_cast2;
+    """
+
+    // old type: 2.4
+    //////////////////////////////////////////
+    // cast to same scale: x.4
+    //////////////////////////////////////////
+    prepare_test_decimalv2_v3_decimal32_cast2()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast2 values (99.9999);
+    """
+
+    //======= wider integral: 3.4
+    qt_cast32_2_same_scale_1 """
+        select cast(k1 as decimalv3(7, 4)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    qt_cast32_2_same_scale_2 """
+        select cast(k1 as decimalv3(18, 4)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    qt_cast32_2_same_scale_3 """
+        select cast(k1 as decimalv3(38, 4)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_2_same_scale_4 """
+        select cast(k1 as decimalv3(76, 4)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 2.4
+    // cast to same scale: x.4
+    //======= same integral: 2.4
+    qt_cast32_2_same_scale_5 """
+        select cast(k1 as decimalv3(6, 4)) from test_decimalv2_v3_decimal32_cast2;
+    """
+
+    // cast to same scale: x.4
+    //======= narrow integral: 1.4
+    // integral part overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(5, 4)) from test_decimalv2_v3_decimal32_cast2;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // integral part not overflow
+    prepare_test_decimalv2_v3_decimal32_cast2()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast2 values (9.9999);
+    """
+    qt_cast32_2_same_scale_6 """
+        select cast(k1 as decimalv3(5, 4)) from test_decimalv2_v3_decimal32_cast2;
+    """
+
+    // old type: 2.4
+    //////////////////////////////////////////
+    // cast to narrow scale: x.3
+    //////////////////////////////////////////
+    prepare_test_decimalv2_v3_decimal32_cast2()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast2 values (99.9999);
+    """
+
+    // wider integral: 3.3
+    qt_cast32_2_narrow_scale_1 """
+        select cast(k1 as decimalv3(6, 3)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    qt_cast32_2_narrow_scale_2 """
+        select cast(k1 as decimalv3(18, 3)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    qt_cast32_2_narrow_scale_3 """
+        select cast(k1 as decimalv3(38, 3)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_2_narrow_scale_4 """
+        select cast(k1 as decimalv3(76, 3)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 2.4
+    // cast to narrow scale: x.3
+    // same integral: 2.3
+    // overflow after round
+    test {
+        sql """
+        select cast(k1 as decimalv3(5, 3)) from test_decimalv2_v3_decimal32_cast2;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 2.4
+    // cast to narrow scale: x.3
+    // narrow integral: 1.3
+    // integral overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(4, 3)) from test_decimalv2_v3_decimal32_cast2;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // integral overflow: overflow after round
+    prepare_test_decimalv2_v3_decimal32_cast2();
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast2 values (9.9999);
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(4, 3)) from test_decimalv2_v3_decimal32_cast2;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // integral not overflow
+    prepare_test_decimalv2_v3_decimal32_cast2();
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast2 values (9.9989);
+    """
+    qt_cast32_2_narrow_scale_6 """
+        select cast(k1 as decimalv3(4, 3)) from test_decimalv2_v3_decimal32_cast2;
+    """
+    sql "drop table test_decimalv2_v3_decimal32_cast2;"
+
+    def prepare_test_decimalv2_v3_decimal32_cast3 = {
+        // type: 9.0
+        sql "drop table if exists test_decimalv2_v3_decimal32_cast3;"
+        sql """
+            CREATE TABLE test_decimalv2_v3_decimal32_cast3(
+              k1 decimalv2(9, 0)
+            )
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 4
+            PROPERTIES (
+            "replication_num" = "1"
+            );
+        """
+    }
+    prepare_test_decimalv2_v3_decimal32_cast3()
+    sql """
+        insert into test_decimalv2_v3_decimal32_cast3 values(999999999.0);
+    """
+    test {
+        // multiply not overflow, but result integral part overflow
+        sql """
+            select cast(k1 as decimalv3(17, 9)) from test_decimalv2_v3_decimal32_cast3;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        // multiply overflow: 999999999 * 10^10, result digit count: 19
+        sql """
+            select cast(k1 as decimalv3(18, 10)) from test_decimalv2_v3_decimal32_cast3;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(38, 30)) from test_decimalv2_v3_decimal32_cast3;
+        """
+        exception "Arithmetic overflow"
+    }
+    sql "set enable_decimal256=true;"
+    test {
+        sql """
+            select cast(k1 as decimalv3(76, 68)) from test_decimalv2_v3_decimal32_cast3;
+        """
+        exception "Arithmetic overflow"
+    }
+    sql "set enable_decimal256=false;"
+    qt_cast32_3_to_much_bigger_scale_1 """
+        select cast(k1 as decimalv3(18, 9)) from test_decimalv2_v3_decimal32_cast3;
+    """
+    qt_cast32_3_to_much_bigger_scale_2 """
+        select cast(k1 as decimalv3(19, 10)) from test_decimalv2_v3_decimal32_cast3;
+    """
+    qt_cast32_3_to_much_bigger_scale_3 """
+        select cast(k1 as decimalv3(38, 29)) from test_decimalv2_v3_decimal32_cast3;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast32_3_to_much_bigger_scale_4 """
+        select cast(k1 as decimalv3(76, 67)) from test_decimalv2_v3_decimal32_cast3;
+    """
+    sql "set enable_decimal256=false;"
+
+    sql "drop table test_decimalv2_v3_decimal32_cast3; "
+
+    // cast decimalv2 who's precision is within decimal64
+    def prepare_test_decimalv2_v3_decimal64_cast1 = {
+        // 12.6
+        sql "drop table if exists test_decimalv2_v3_decimal64_cast1;"
+        sql """
+            CREATE TABLE test_decimalv2_v3_decimal64_cast1(
+              k1 decimalv2(18, 6)
+            )
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 4
+            PROPERTIES (
+            "replication_num" = "1"
+            );
+        """
+    }
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (999999999999.999999);
+    """
+    qt_cast64_select_all """
+        select * from test_decimalv2_v3_decimal64_cast1 order by 1;
+    """
+
+    // old type: 12.6
+    //////////////////////////////////////////
+    // cast to wider scale: x.7
+    //////////////////////////////////////////
+
+    //======= wider integral: 13.7
+    qt_cast64_wider_scale_1 """
+        select cast(k1 as decimalv3(20, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    qt_cast64_wider_scale_2 """
+        select cast(k1 as decimalv3(38, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast64_wider_scale_3 """
+        select cast(k1 as decimalv3(76, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    sql "set enable_decimal256=false;"
+
+    // cast to wider scale: x.7
+    //======= same integral: 12.7
+    qt_cast64_wider_scale_4 """
+        select cast(k1 as decimalv3(19, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+
+    // old type: 12.6
+    // cast to wider scale: x.7
+    //======= narrow integral: 11.7
+    // integral part overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(18, 7)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 7)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 7)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    
+    // cast to wider scale: x.7
+    //======= narrow integral: 11.7
+    // integral part not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (99999999999.999999);
+    """
+    qt_cast64_wider_scale_5 """
+        select cast(k1 as decimalv3(18, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    // cast to decimal32 overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 7)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 7)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // cast to decimal32 not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (99.999999);
+    """
+    qt_cast64_wider_scale_6 """
+        select cast(k1 as decimalv3(9, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 7)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // cast to decimal32 not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (9.999999);
+    """
+    qt_cast64_wider_scale_7 """
+        select cast(k1 as decimalv3(9, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    qt_cast64_wider_scale_8 """
+        select cast(k1 as decimalv3(8, 7)) from test_decimalv2_v3_decimal64_cast1;
+    """
+
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (999999999999.999999);
+    """
+    // old type: 12.6
+    //////////////////////////////////////////
+    // cast to same scale: x.6
+    //////////////////////////////////////////
+
+    //======= wider integral: 13.6
+    qt_cast64_same_scale_1 """
+        select cast(k1 as decimalv3(19, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    qt_cast64_same_scale_2 """
+        select cast(k1 as decimalv3(38, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast64_same_scale_3 """
+        select cast(k1 as decimalv3(76, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    sql "set enable_decimal256=false;"
+
+    // cast to same scale: x.6
+    //======= same integral: 12.6
+    qt_cast64_same_scale_4 """
+        select cast(k1 as decimalv3(18, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+
+    // old type: 12.6
+    // cast to same scale: x.6
+    //======= narrow integral: 11.6
+    // integral part overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(17, 6)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    // to decimal32 overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 6)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 6)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // cast to same scale: x.6
+    //======= narrow integral: 11.6
+    // integral part not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (99999999999.999999);
+    """
+    qt_cast64_same_scale_5 """
+        select cast(k1 as decimalv3(17, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    // to decimal32 overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 6)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 6)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // cast to same scale: x.6
+    //======= narrow integral: 3.6
+    // to decimal32 not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (999.999999);
+    """
+    qt_cast64_same_scale_6 """
+        select cast(k1 as decimalv3(9, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 6)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // to decimal32 not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (99.999999);
+    """
+    qt_cast64_same_scale_7 """
+        select cast(k1 as decimalv3(9, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    qt_cast64_same_scale_8 """
+        select cast(k1 as decimalv3(8, 6)) from test_decimalv2_v3_decimal64_cast1;
+    """
+
+    // old type: 12.6
+    //////////////////////////////////////////
+    // cast to narrow scale: x.5
+    //////////////////////////////////////////
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (999999999999.999999);
+    """
+
+    //======= wider integral: 13.5
+    qt_cast64_narrow_scale_1 """
+        select cast(k1 as decimalv3(18, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    qt_cast64_narrow_scale_2 """
+        select cast(k1 as decimalv3(38, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast64_narrow_scale_3 """
+        select cast(k1 as decimalv3(76, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 12.6
+    // cast to narrow scale: x.5
+    //======= same integral: 12.5
+    // overflow after round
+    test {
+        sql """
+        select cast(k1 as decimalv3(17, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 12.6
+    // cast to narrow scale: x.5
+    //======= narrow integral: 11.5
+    // integral overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(16, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // integral overflow after round
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (99999999999.999999);
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(16, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // integral not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (99999999999.999989);
+    """
+    qt_cast64_narrow_scale_5 """
+        select cast(k1 as decimalv3(16, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+
+    // to decimal32 overflow
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // to decimal32 overflow after round
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (9999.999999);
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // to decimal32 not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (9999.999989);
+    """
+    qt_cast64_narrow_scale_6 """
+        select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // to decimal32 overflow after round
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (999.999999);
+    """
+    qt_cast64_narrow_scale_7 """
+        select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    test {
+        sql """
+            select cast(k1 as decimalv3(8, 5)) from test_decimalv2_v3_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // to decimal32 not overflow
+    prepare_test_decimalv2_v3_decimal64_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal64_cast1 values (999.999989);
+    """
+    qt_cast64_narrow_scale_8 """
+        select cast(k1 as decimalv3(9, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    qt_cast64_narrow_scale_9 """
+        select cast(k1 as decimalv3(8, 5)) from test_decimalv2_v3_decimal64_cast1;
+    """
+    sql "drop table test_decimalv2_v3_decimal64_cast1;"
+
+    // cast decimalv2 who's precision is within decimal128
+    def prepare_test_decimalv2_v3_decimal128_cast1 = {
+        // 18.9
+        sql "drop table if exists test_decimalv2_v3_decimal128_cast1;"
+        sql """
+            CREATE TABLE test_decimalv2_v3_decimal128_cast1(
+              k1 decimalv2(27, 9)
+            )
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 4
+            PROPERTIES (
+            "replication_num" = "1"
+            );
+        """
+    }
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (999999999999999999.999999999),
+            (-999999999999999999.999999999);
+    """
+    // old type: 18.9
+    //////////////////////////////////////////
+    // cast to wider scale: x.10
+    //////////////////////////////////////////
+
+    //======= wider integral: 19.10
+    qt_cast128_wider_scale_1 """
+        select cast(k1 as decimalv3(29, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    qt_cast128_wider_scale_2 """
+        select cast(k1 as decimalv3(38, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+
+    sql "set enable_decimal256=true;"
+    qt_cast128_wider_scale_3 """
+        select cast(k1 as decimalv3(39, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    qt_cast128_wider_scale_4 """
+        select cast(k1 as decimalv3(76, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    sql "set enable_decimal256=false;"
+
+    //======= same integral: 18.10
+    qt_cast128_wider_scale_5 """
+        select cast(k1 as decimalv3(28, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+
+    //======= narrow integral: 17.10
+    test {
+        sql """
+        select cast(k1 as decimalv3(27, 10)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+        select cast(k1 as decimalv3(11, 10)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 18.9
+    // cast to wider scale: x.10
+    //======= narrow integral: 17.10, not overflow
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (99999999999999999.999999999),
+            (-99999999999999999.999999999);
+    """
+    qt_cast128_wider_scale_6 """
+        select cast(k1 as decimalv3(27, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    // to much narrower integer, still overflow
+    test {
+        sql """
+        select cast(k1 as decimalv3(26, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+        select cast(k1 as decimalv3(11, 10)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 18.9
+    // cast to wider scale: x.10
+    //======= narrow integral
+    // narrow integral 64 not overflow
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (99999999.999999999),
+            (-99999999.999999999);
+    """
+    qt_cast128_wider_scale_7 """
+        select cast(k1 as decimalv3(18, 10)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    test {
+        sql """
+        select cast(k1 as decimalv3(11, 10)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 18.9
+    //////////////////////////////////////////
+    // cast to same scale: x.9
+    //////////////////////////////////////////
+
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (999999999999999999.999999999),
+            (-999999999999999999.999999999);
+    """
+    //======= wider integral: 19.9
+    qt_cast128_same_scale_1 """
+        select cast(k1 as decimalv3(28, 9)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    qt_cast128_same_scale_2 """
+        select cast(k1 as decimalv3(38, 9)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast128_same_scale_3 """
+        select cast(k1 as decimalv3(76, 9)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 18.9
+    // cast to same scale: x.9
+    //======= same integral: 18.9
+    qt_cast128_same_scale_4 """
+        select cast(k1 as decimalv3(27, 9)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+
+    // old type: 18.9
+    // cast to same scale: x.9
+    //======= narrow integral: 17.9
+    test {
+        sql """
+        select cast(k1 as decimalv3(26, 9)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    // to decimal64 and decimal32 overflow
+    test {
+        sql """
+        select cast(k1 as decimalv3(18, 9)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+        select cast(k1 as decimalv3(9, 9)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 18.9
+    // cast to same scale: x.9
+    //======= narrow integral: 17.9
+    // not overflow
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (99999999999999999.999999999),
+            (-99999999999999999.999999999);
+    """
+    qt_cast128_same_scale_5 """
+        select cast(k1 as decimalv3(26, 9)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+
+    // to decimal64 and decimal32 overflow
+    test {
+        sql """
+        select cast(k1 as decimalv3(18, 9)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    test {
+        sql """
+        select cast(k1 as decimalv3(9, 9)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 18.9
+    // cast to same scale: x.9
+    //======= narrow integral: 17.9
+    // not overflow
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (999999999.999999999),
+            (-999999999.999999999);
+    """
+    qt_cast128_same_scale_6 """
+        select cast(k1 as decimalv3(26, 9)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+
+    // to decimal64 not overflow
+    qt_cast128_same_scale_7 """
+        select cast(k1 as decimalv3(18, 9)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    // to decimal32 not overflow
+    test {
+        sql """
+        select cast(k1 as decimalv3(9, 9)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // old type: 18.9
+    //////////////////////////////////////////
+    // cast to narrow scale: x.8
+    //////////////////////////////////////////
+
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (999999999999999999.999999999),
+            (-999999999999999999.999999999);
+    """
+
+    //======= wider integral: 19.8
+    qt_cast128_narrow_scale_1 """
+        select cast(k1 as decimalv3(27, 8)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    qt_cast128_narrow_scale_2 """
+        select cast(k1 as decimalv3(38, 8)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    sql "set enable_decimal256=true;"
+    qt_cast128_narrow_scale_3 """
+        select cast(k1 as decimalv3(76, 8)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+    sql "set enable_decimal256=false;"
+
+    // old type: 18.9
+    // cast to narrow scale: x.8
+    //======= same integral: 18.8
+    // overflow after round, bug
+    test {
+        sql """
+        select cast(k1 as decimalv3(26, 8)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    //======= same integral: 18.8 not overflow
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (999999999999999999.999999989),
+            (-999999999999999999.999999989);
+    """
+    qt_cast128_narrow_scale_4 """
+        select cast(k1 as decimalv3(26, 8)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+
+    // old type: 18.9
+    // cast to narrow scale: x.8
+    //======= narrow integral: 17.8
+
+    test {
+        sql """
+        select cast(k1 as decimalv3(25, 8)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // overflow after round
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (99999999999999999.999999999),
+            (-99999999999999999.999999999);
+    """
+    test {
+        sql """
+        select cast(k1 as decimalv3(25, 8)) from test_decimalv2_v3_decimal128_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+
+    // not overflow
+    prepare_test_decimalv2_v3_decimal128_cast1()
+    sql """
+        insert into test_decimalv2_v3_decimal128_cast1 values
+            (99999999999999999.999999994),
+            (-99999999999999999.999999994);
+    """
+    qt_cast128_narrow_scale_5 """
+        select cast(k1 as decimalv3(25, 8)) from test_decimalv2_v3_decimal128_cast1 order by 1;
+    """
+
+    sql """
+        admin set frontend config("enable_decimal_conversion" = "true");
+    """
+}
diff --git a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_cast.groovy b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_cast.groovy
index 8394ec5cf387f7..72fa8bc14df8ee 100644
--- a/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_cast.groovy
+++ b/regression-test/suites/datatype_p0/decimalv3/test_decimalv3_cast.groovy
@@ -418,6 +418,18 @@ suite("test_decimalv3_cast") {
     // old type: 2.4
     // cast to narrow scale: x.3
     // same integral: 2.3
+    // overflow after round
+    test {
+        sql """
+        select cast(k1 as decimalv3(5, 3)) from test_decimal32_cast2;
+        """
+        exception "Arithmetic overflow"
+    }
+    // not overflow after round
+    prepare_test_decimal32_cast2()
+    sql """
+        insert into test_decimal32_cast2 values (99.9989);
+    """
     qt_cast32_2_narrow_scale_5 """
         select cast(k1 as decimalv3(5, 3)) from test_decimal32_cast2;
     """
@@ -765,6 +777,18 @@ suite("test_decimalv3_cast") {
     // old type: 12.6
     // cast to narrow scale: x.5
     //======= same integral: 12.5
+    // overflow after round
+    test {
+        sql """
+        select cast(k1 as decimalv3(17, 5)) from test_decimal64_cast1;
+        """
+        exception "Arithmetic overflow"
+    }
+    // not overflow after round
+    prepare_test_decimal64_cast1()
+    sql """
+        insert into test_decimal64_cast1 values (999999999999.999994);
+    """
     qt_cast64_narrow_scale_4 """
         select cast(k1 as decimalv3(17, 5)) from test_decimal64_cast1;
     """
@@ -1154,6 +1178,19 @@ suite("test_decimalv3_cast") {
     sql "set enable_decimal256=false;"
 
     //======= same integral: 32.5
+    // overflow after round
+    test {
+        sql """
+        select cast(k1 as decimalv3(37, 5)) from test_decimal128_cast1 order by 1;
+        """
+        exception "Arithmetic overflow"
+    }
+    prepare_test_decimal128_cast1()
+    sql """
+        insert into test_decimal128_cast1 values
+            (99999999999999999999999999999999.999989),
+            (-99999999999999999999999999999999.999989);
+    """
     qt_cast128_narrow_scale_3 """
         select cast(k1 as decimalv3(37, 5)) from test_decimal128_cast1 order by 1;
     """

From abbd2cedff0760078a7c3253e2aa4066fadfca4c Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Wed, 27 Dec 2023 21:04:00 +0800
Subject: [PATCH 089/299] [fix](Nereids) merge limit should use bottom phase
 (#29142)

---
 .../nereids/rules/rewrite/MergeLimits.java    |  2 +-
 .../rules/rewrite/MergeLimitsTest.java        | 50 +++++++++++++++++++
 2 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java
index 0621e70b707408..e1428983c159f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java
@@ -50,7 +50,7 @@ public Rule build() {
                             Math.min(upperLimit.getLimit(),
                                     Math.max(bottomLimit.getLimit() - upperLimit.getOffset(), 0)),
                             bottomLimit.getOffset() + upperLimit.getOffset(),
-                            upperLimit.getPhase(), bottomLimit.child()
+                            upperLimit.child().getPhase(), bottomLimit.child()
                     );
                 }).toRule(RuleType.MERGE_LIMITS);
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeLimitsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeLimitsTest.java
index 9fcfd03e41d37e..dbaeef6d797b39 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeLimitsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/MergeLimitsTest.java
@@ -17,6 +17,8 @@
 
 package org.apache.doris.nereids.rules.rewrite;
 
+import org.apache.doris.nereids.trees.plans.LimitPhase;
+import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.util.LogicalPlanBuilder;
@@ -44,4 +46,52 @@ void testMergeLimits() {
                 .matches(
                         logicalLimit().when(limit -> limit.getLimit() == 2).when(limit -> limit.getOffset() == 5));
     }
+
+    @Test
+    void testMergeLimitsWithLocalUpperAndGlobalBottom() {
+        LogicalPlan logicalLimit = new LogicalLimit<>(2, 0, LimitPhase.LOCAL,
+                new LogicalLimit<>(10, 0, LimitPhase.GLOBAL, scan1));
+
+        PlanChecker.from(new ConnectContext(), logicalLimit).applyTopDown(new MergeLimits())
+                .matches(
+                        logicalLimit().when(limit -> limit.getLimit() == 2)
+                                .when(limit -> limit.getPhase() == LimitPhase.GLOBAL));
+    }
+
+    @Test
+    void testMergeLimitsWithLocalUpperAndLocalBottom() {
+        LogicalPlan logicalLimit = new LogicalLimit<>(2, 0, LimitPhase.LOCAL,
+                new LogicalLimit<>(10, 0, LimitPhase.LOCAL, scan1));
+
+        PlanChecker.from(new ConnectContext(), logicalLimit).applyTopDown(new MergeLimits())
+                .matches(
+                        logicalLimit().when(limit -> limit.getLimit() == 2)
+                                .when(limit -> limit.getPhase() == LimitPhase.LOCAL));
+    }
+
+    @Test
+    void testMergeLimitsWithGlobalUpperAndGlobalBottom() {
+        LogicalPlan logicalLimit = new LogicalLimit<>(2, 0, LimitPhase.GLOBAL,
+                new LogicalLimit<>(10, 0, LimitPhase.GLOBAL, scan1));
+
+        PlanChecker.from(new ConnectContext(), logicalLimit).applyTopDown(new MergeLimits())
+                .matches(
+                        logicalLimit().when(limit -> limit.getLimit() == 2)
+                                .when(limit -> limit.getPhase() == LimitPhase.GLOBAL));
+    }
+
+    @Test
+    void testMergeLimitsWithGlobalUpperAndLocalBottom() {
+        LogicalPlan logicalLimit = new LogicalLimit<>(2, 0, LimitPhase.GLOBAL,
+                new LogicalLimit<>(10, 0, LimitPhase.LOCAL, scan1));
+
+        PlanChecker.from(new ConnectContext(), logicalLimit).applyTopDown(new MergeLimits())
+                .matches(
+                        logicalLimit(
+                                logicalLimit()
+                                        .when(limit -> limit.getLimit() == 10)
+                                        .when(limit -> limit.getPhase() == LimitPhase.LOCAL))
+                                .when(limit -> limit.getLimit() == 2)
+                                .when(limit -> limit.getPhase() == LimitPhase.GLOBAL));
+    }
 }

From 224677af7c2bc537975ba318a8dfa2336667235a Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Wed, 27 Dec 2023 21:04:58 +0800
Subject: [PATCH 090/299] delete unused code in delta writer v2 (#29131)

---
 be/src/olap/delta_writer_v2.cpp |  8 --------
 be/src/olap/delta_writer_v2.h   | 13 -------------
 2 files changed, 21 deletions(-)

diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index c47cdc6e563470..006e8a15a6ffc7 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -209,14 +209,6 @@ Status DeltaWriterV2::cancel_with_status(const Status& st) {
     return Status::OK();
 }
 
-int64_t DeltaWriterV2::mem_consumption(MemType mem) {
-    return _memtable_writer->mem_consumption(mem);
-}
-
-int64_t DeltaWriterV2::partition_id() const {
-    return _req.partition_id;
-}
-
 void DeltaWriterV2::_build_current_tablet_schema(int64_t index_id,
                                                  const OlapTableSchemaParam* table_schema_param,
                                                  const TabletSchema& ori_tablet_schema) {
diff --git a/be/src/olap/delta_writer_v2.h b/be/src/olap/delta_writer_v2.h
index c371a8e9818bb9..f896977d842539 100644
--- a/be/src/olap/delta_writer_v2.h
+++ b/be/src/olap/delta_writer_v2.h
@@ -86,16 +86,6 @@ class DeltaWriterV2 {
     Status cancel();
     Status cancel_with_status(const Status& st);
 
-    int64_t partition_id() const;
-
-    int64_t mem_consumption(MemType mem);
-
-    int64_t tablet_id() { return _req.tablet_id; }
-
-    int32_t schema_hash() { return _req.schema_hash; }
-
-    int64_t total_received_rows() const { return _total_received_rows; }
-
 private:
     DeltaWriterV2(WriteRequest* req, const std::vector<std::shared_ptr<LoadStreamStub>>& streams,
                   StorageEngine* storage_engine);
@@ -115,9 +105,6 @@ class DeltaWriterV2 {
 
     std::mutex _lock;
 
-    // total rows num written by DeltaWriterV2
-    int64_t _total_received_rows = 0;
-
     int64_t _write_memtable_time = 0;
     int64_t _wait_flush_limit_time = 0;
     int64_t _close_wait_time = 0;

From d96278ab212ff122e9e5176ee1bcc9da2430d911 Mon Sep 17 00:00:00 2001
From: Liqf <109049295+LemonLiTree@users.noreply.github.com>
Date: Wed, 27 Dec 2023 21:17:20 +0800
Subject: [PATCH 091/299] [bug](fix) show create table show comment error
 (#28346)

---
 .../org/apache/doris/nereids/parser/LogicalPlanBuilder.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index cf670fe1aa939e..71890516628c38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2424,7 +2424,9 @@ public ColumnDefinition visitColumnDef(ColumnDefContext ctx) {
                         e.getCause());
             }
         }
-        String comment = ctx.comment != null ? ctx.comment.getText() : "";
+        //comment should remove '\' and '(") at the beginning and end
+        String comment = ctx.comment != null ? ctx.comment.getText().substring(1, ctx.comment.getText().length() - 1)
+                .replace("\\", "") : "";
         boolean isAutoInc = ctx.AUTO_INCREMENT() != null;
         return new ColumnDefinition(colName, colType, isKey, aggType, !isNotNull, isAutoInc, defaultValue,
                 onUpdateDefaultValue, comment);

From 0cc4ee52bf3dcc8cc6b928f14be8a4e3855623f5 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Wed, 27 Dec 2023 21:48:32 +0800
Subject: [PATCH 092/299] [fix](move-memtable) fix streams for node memory leak
 in sink v2 (#29146)

---
 be/src/vec/sink/writer/vtablet_writer_v2.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index ae753c8714a016..f413c00dc0c7e3 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -502,6 +502,7 @@ Status VTabletWriterV2::close(Status exec_status) {
             for (const auto& [_, streams] : _streams_for_node) {
                 streams->release();
             }
+            _streams_for_node.clear();
         });
 
         {
@@ -541,7 +542,6 @@ Status VTabletWriterV2::close(Status exec_status) {
         _state->tablet_commit_infos().insert(_state->tablet_commit_infos().end(),
                                              std::make_move_iterator(tablet_commit_infos.begin()),
                                              std::make_move_iterator(tablet_commit_infos.end()));
-        _streams_for_node.clear();
 
         // _number_input_rows don't contain num_rows_load_filtered and num_rows_load_unselected in scan node
         int64_t num_rows_load_total = _number_input_rows + _state->num_rows_load_filtered() +

From 0bff38757739d12a18fb029923f4bb1f639d3a47 Mon Sep 17 00:00:00 2001
From: yujun <yu.jun.reach@gmail.com>
Date: Wed, 27 Dec 2023 22:02:42 +0800
Subject: [PATCH 093/299] [fix](tablet stat) fix tablet stat thread block
 #29151

---
 .../main/java/org/apache/doris/catalog/TabletStatMgr.java | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index bb799ab802a999..2f566268d7d025 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -68,14 +68,18 @@ protected void runAfterCatalogReady() {
                             result.getTabletsStatsSize());
                     updateTabletStat(backend.getId(), result);
                     ok = true;
-                } catch (Exception e) {
+                } catch (Throwable e) {
                     LOG.warn("task exec error. backend[{}]", backend.getId(), e);
-                } finally {
+                }
+
+                try {
                     if (ok) {
                         ClientPool.backendPool.returnObject(address, client);
                     } else {
                         ClientPool.backendPool.invalidateObject(address, client);
                     }
+                } catch (Throwable e) {
+                    LOG.warn("client pool recyle error. backend[{}]", backend.getId(), e);
                 }
             });
         }).join();

From f4c5ce260b4ac4b3974bcd8ef7dcd059ecfdd78b Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Wed, 27 Dec 2023 23:04:37 +0800
Subject: [PATCH 094/299] [fix](statistics)Fix rowCount==0 while analyzing bug
 (#28969)

Sample analyzing need to get row count by using table.getRowCount(). This method is not updated in real time, which may cause the sample task to scan whole table.
This pr is to fix this. Set the flag that indicate the analyze job is for an empty table and skip scan the table. Meanwhile, don't reset updatedRows in this case.

Set hugeTableAutoAnalyzeIntervalInMillis = 0 because all default huge table size has been set to 0.
---
 docs/en/docs/query-acceleration/statistics.md          |  4 ++--
 docs/zh-CN/docs/query-acceleration/statistics.md       |  4 ++--
 .../main/java/org/apache/doris/qe/SessionVariable.java |  2 +-
 .../java/org/apache/doris/statistics/AnalysisInfo.java |  7 ++++++-
 .../apache/doris/statistics/AnalysisInfoBuilder.java   | 10 ++++++++--
 .../org/apache/doris/statistics/AnalysisManager.java   |  1 +
 .../org/apache/doris/statistics/OlapAnalysisTask.java  |  3 ++-
 .../apache/doris/statistics/StatisticConstants.java    |  2 +-
 .../doris/statistics/StatisticsAutoCollector.java      |  1 +
 .../org/apache/doris/statistics/TableStatsMeta.java    |  2 +-
 .../doris/statistics/StatisticsAutoCollectorTest.java  |  2 +-
 regression-test/suites/statistics/analyze_stats.groovy |  2 +-
 12 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/docs/en/docs/query-acceleration/statistics.md b/docs/en/docs/query-acceleration/statistics.md
index c7a58277580a00..4cb0891172d8fc 100644
--- a/docs/en/docs/query-acceleration/statistics.md
+++ b/docs/en/docs/query-acceleration/statistics.md
@@ -295,8 +295,8 @@ mysql> KILL ANALYZE 52357;
 |auto_analyze_end_time|End time for automatic statistics collection|23:59:59|
 |enable_auto_analyze|Enable automatic collection functionality|true|
 |huge_table_default_sample_rows|Sampling rows for large tables|4194304|
-|huge_table_lower_bound_size_in_bytes|Tables with size greater than this value will be automatically sampled during collection of statistics|5368709120|
-|huge_table_auto_analyze_interval_in_millis|Controls the minimum time interval for automatic ANALYZE on large tables. Tables with sizes greater than `huge_table_lower_bound_size_in_bytes * 5` will be ANALYZEed only once within this time interval.|43200000|
+|huge_table_lower_bound_size_in_bytes|Tables with size greater than this value will be automatically sampled during collection of statistics|0|
+|huge_table_auto_analyze_interval_in_millis|Controls the minimum time interval for automatic ANALYZE on large tables. Tables with sizes greater than `huge_table_lower_bound_size_in_bytes * 5` will be ANALYZEed only once within this time interval.|0|
 |table_stats_health_threshold|Ranges from 0 to 100. If data updates since the last statistics collection exceed `(100 - table_stats_health_threshold)%`, the table's statistics are considered outdated.|60|
 |analyze_timeout|Controls the timeout for synchronous ANALYZE in seconds|43200|
 |auto_analyze_table_width_threshold|Controls the maximum width of table that will be auto analyzed. Table with more columns than this value will not be auto analyzed.|70|
diff --git a/docs/zh-CN/docs/query-acceleration/statistics.md b/docs/zh-CN/docs/query-acceleration/statistics.md
index 20b535e357b8ad..bff100fa98ac4f 100644
--- a/docs/zh-CN/docs/query-acceleration/statistics.md
+++ b/docs/zh-CN/docs/query-acceleration/statistics.md
@@ -299,8 +299,8 @@ mysql> KILL ANALYZE 52357;
 |auto_analyze_end_time|自动统计信息收集结束时间|23:59:59|
 |enable_auto_analyze|开启自动收集功能|true|
 |huge_table_default_sample_rows|对大表的采样行数|4194304|
-|huge_table_lower_bound_size_in_bytes|大小超过该值的的表，在自动收集时将会自动通过采样收集统计信息|5368709120|
-|huge_table_auto_analyze_interval_in_millis|控制对大表的自动ANALYZE的最小时间间隔，在该时间间隔内大小超过huge_table_lower_bound_size_in_bytes * 5的表仅ANALYZE一次|43200000|
+|huge_table_lower_bound_size_in_bytes|大小超过该值的的表，在自动收集时将会自动通过采样收集统计信息|0|
+|huge_table_auto_analyze_interval_in_millis|控制对大表的自动ANALYZE的最小时间间隔，在该时间间隔内大小超过huge_table_lower_bound_size_in_bytes * 5的表仅ANALYZE一次|0|
 |table_stats_health_threshold|取值在0-100之间，当自上次统计信息收集操作之后，数据更新量达到 (100 - table_stats_health_threshold)% ，认为该表的统计信息已过时|60|
 |analyze_timeout|控制ANALYZE超时时间，单位为秒|43200|
 |auto_analyze_table_width_threshold|控制自动统计信息收集处理的最大表宽度，列数大于该值的表不会参与自动统计信息收集|70|
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 35545059901d4f..c1ea2f29ff292f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -1450,7 +1450,7 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
                     "This controls the minimum time interval for automatic ANALYZE on large tables."
                             + "Within this interval,"
                             + "tables larger than huge_table_lower_bound_size_in_bytes are analyzed only once."})
-    public long hugeTableAutoAnalyzeIntervalInMillis = TimeUnit.HOURS.toMillis(12);
+    public long hugeTableAutoAnalyzeIntervalInMillis = TimeUnit.HOURS.toMillis(0);
 
     @VariableMgr.VarAttr(name = EXTERNAL_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS, flag = VariableMgr.GLOBAL,
             description = {"控制对外表的自动ANALYZE的最小时间间隔，在该时间间隔内的外表仅ANALYZE一次",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
index 65bb4a5dd95e3d..aaff9e59927385 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -188,6 +188,9 @@ public enum ScheduleType {
 
     @SerializedName("endTime")
     public long endTime;
+
+    @SerializedName("emptyJob")
+    public final boolean emptyJob;
     /**
      *
      * Used to store the newest partition version of tbl when creating this job.
@@ -202,7 +205,7 @@ public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId,
             long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType,
             boolean isExternalTableLevelTask, boolean partitionOnly, boolean samplingPartition,
             boolean isAllPartition, long partitionCount, CronExpression cronExpression, boolean forceFull,
-            boolean usingSqlForPartitionColumn, long tblUpdateTime) {
+            boolean usingSqlForPartitionColumn, long tblUpdateTime, boolean emptyJob) {
         this.jobId = jobId;
         this.taskId = taskId;
         this.taskIds = taskIds;
@@ -238,6 +241,7 @@ public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId,
         this.forceFull = forceFull;
         this.usingSqlForPartitionColumn = usingSqlForPartitionColumn;
         this.tblUpdateTime = tblUpdateTime;
+        this.emptyJob = emptyJob;
     }
 
     @Override
@@ -279,6 +283,7 @@ public String toString() {
         }
         sj.add("forceFull: " + forceFull);
         sj.add("usingSqlForPartitionColumn: " + usingSqlForPartitionColumn);
+        sj.add("emptyJob: " + emptyJob);
         return sj.toString();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
index 204aba6d0f8947..310b7816ecd5b0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfoBuilder.java
@@ -61,8 +61,8 @@ public class AnalysisInfoBuilder {
     private CronExpression cronExpression;
     private boolean forceFull;
     private boolean usingSqlForPartitionColumn;
-
     private long tblUpdateTime;
+    private boolean emptyJob;
 
     public AnalysisInfoBuilder() {
     }
@@ -100,6 +100,7 @@ public AnalysisInfoBuilder(AnalysisInfo info) {
         forceFull = info.forceFull;
         usingSqlForPartitionColumn = info.usingSqlForPartitionColumn;
         tblUpdateTime = info.tblUpdateTime;
+        emptyJob = info.emptyJob;
     }
 
     public AnalysisInfoBuilder setJobId(long jobId) {
@@ -262,12 +263,17 @@ public AnalysisInfoBuilder setTblUpdateTime(long tblUpdateTime) {
         return this;
     }
 
+    public AnalysisInfoBuilder setEmptyJob(boolean emptyJob) {
+        this.emptyJob = emptyJob;
+        return this;
+    }
+
     public AnalysisInfo build() {
         return new AnalysisInfo(jobId, taskId, taskIds, catalogId, dbId, tblId, colToPartitions, partitionNames,
                 colName, indexId, jobType, analysisMode, analysisMethod, analysisType, samplePercent,
                 sampleRows, maxBucketNum, periodTimeInMs, message, lastExecTimeInMs, timeCostInMs, state, scheduleType,
                 externalTableLevelTask, partitionOnly, samplingPartition, isAllPartition, partitionCount,
-                cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime);
+                cronExpression, forceFull, usingSqlForPartitionColumn, tblUpdateTime, emptyJob);
     }
 
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 0bf24e0c288071..39ae191d45a83c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -519,6 +519,7 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlExceptio
         infoBuilder.setColToPartitions(colToPartitions);
         infoBuilder.setTaskIds(Lists.newArrayList());
         infoBuilder.setTblUpdateTime(table.getUpdateTime());
+        infoBuilder.setEmptyJob(table instanceof OlapTable && table.getRowCount() == 0);
         return infoBuilder.build();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index e062e4eef850d2..81348c1f948c29 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -61,7 +61,8 @@ public OlapAnalysisTask(AnalysisInfo info) {
 
     public void doExecute() throws Exception {
         Set<String> partitionNames = info.colToPartitions.get(info.colName);
-        if (partitionNames == null || partitionNames.isEmpty()) {
+        if ((info.emptyJob && info.analysisMethod.equals(AnalysisInfo.AnalysisMethod.SAMPLE))
+                || partitionNames == null || partitionNames.isEmpty()) {
             if (partitionNames == null) {
                 LOG.warn("Table {}.{}.{}, partitionNames for column {} is null. ColToPartitions:[{}]",
                         info.catalogId, info.dbId, info.tblId, info.colName, info.colToPartitions);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index 3d6d2fe52aa937..857a50e234cea5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -88,7 +88,7 @@ public class StatisticConstants {
     public static final long HUGE_TABLE_DEFAULT_SAMPLE_ROWS = 4194304;
     public static final long HUGE_TABLE_LOWER_BOUND_SIZE_IN_BYTES = 0;
 
-    public static final long HUGE_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS = TimeUnit.HOURS.toMillis(12);
+    public static final long HUGE_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS = TimeUnit.HOURS.toMillis(0);
 
     public static final long EXTERNAL_TABLE_AUTO_ANALYZE_INTERVAL_IN_MILLIS = TimeUnit.HOURS.toMillis(24);
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index ee50471175d4f8..f799da56206095 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -170,6 +170,7 @@ protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
                 .setLastExecTimeInMs(System.currentTimeMillis())
                 .setJobType(JobType.SYSTEM)
                 .setTblUpdateTime(table.getUpdateTime())
+                .setEmptyJob(table instanceof OlapTable && table.getRowCount() == 0)
                 .build();
         analysisInfos.add(jobInfo);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index f500ab09f0b1de..eb6672ffe1856f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -149,7 +149,7 @@ public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
             if (tableIf instanceof OlapTable) {
                 rowCount = tableIf.getRowCount();
             }
-            if (analyzedJob.colToPartitions.keySet()
+            if (!analyzedJob.emptyJob && analyzedJob.colToPartitions.keySet()
                     .containsAll(tableIf.getBaseSchema().stream()
                             .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
                             .map(Column::getName).collect(Collectors.toSet()))) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 0b4b2203d0dc79..87342202fb2a27 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -299,7 +299,7 @@ public TableStatsMeta findTableStatsStatus(long tblId) {
         // A very huge table has been updated recently, so we should skip it this time
         stats.updatedTime = System.currentTimeMillis() - 1000;
         StatisticsAutoCollector autoCollector = new StatisticsAutoCollector();
-        Assertions.assertTrue(autoCollector.skip(olapTable));
+        Assertions.assertFalse(autoCollector.skip(olapTable));
         // The update of this huge table is long time ago, so we shouldn't skip it this time
         stats.updatedTime = System.currentTimeMillis()
                 - StatisticsUtil.getHugeTableAutoAnalyzeIntervalInMillis() - 10000;
diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
index e7e89f858fb945..64967280ce9274 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -1168,7 +1168,7 @@ PARTITION `p599` VALUES IN (599)
     sql """ INSERT INTO test_updated_rows SELECT * FROM test_updated_rows """
     sql """ANALYZE TABLE test_updated_rows WITH SYNC"""
     def cnt2 = sql """ SHOW TABLE STATS test_updated_rows """
-    assertEquals(Integer.valueOf(cnt2[0][0]), 0)
+    assertTrue(Integer.valueOf(cnt2[0][0]) == 0 || Integer.valueOf(cnt2[0][0]) == 8)
 
     // test analyze specific column
     sql """CREATE TABLE test_analyze_specific_column (col1 varchar(11451) not null, col2 int not null, col3 int not null)

From 1aa9ac4fe44ee23db95a757d2d2ecf368d5774b9 Mon Sep 17 00:00:00 2001
From: plat1ko <platonekosama@gmail.com>
Date: Wed, 27 Dec 2023 23:43:43 +0800
Subject: [PATCH 095/299] Prevent making snapshot on remote rowset in single
 replica compaction (#28716)

---
 be/src/common/status.h                        |   2 -
 be/src/olap/base_compaction.cpp               |   8 -
 be/src/olap/base_compaction.h                 |   2 -
 be/src/olap/compaction.h                      |   7 +-
 be/src/olap/cumulative_compaction.cpp         |   8 -
 be/src/olap/cumulative_compaction.h           |   2 -
 be/src/olap/full_compaction.cpp               |  10 -
 be/src/olap/full_compaction.h                 |   2 -
 be/src/olap/olap_server.cpp                   |  61 ++--
 be/src/olap/single_replica_compaction.cpp     |   8 -
 be/src/olap/snapshot_manager.cpp              |   4 +-
 be/src/olap/storage_engine.cpp                |  22 --
 be/src/olap/storage_engine.h                  |  12 -
 be/src/olap/tablet.cpp                        | 289 +++++++-----------
 be/src/olap/tablet.h                          |  37 +--
 be/src/olap/task/engine_clone_task.cpp        |   1 -
 .../task/engine_storage_migration_task.cpp    |   2 -
 17 files changed, 162 insertions(+), 315 deletions(-)

diff --git a/be/src/common/status.h b/be/src/common/status.h
index 14aec46cc226b6..c0d92152877574 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -160,7 +160,6 @@ namespace ErrorCode {
     E(BE_INVALID_NEED_MERGED_VERSIONS, -810, true);          \
     E(BE_ERROR_DELETE_ACTION, -811, true);                   \
     E(BE_SEGMENTS_OVERLAPPING, -812, true);                  \
-    E(BE_CLONE_OCCURRED, -813, true);                        \
     E(PUSH_INIT_ERROR, -900, true);                          \
     E(PUSH_VERSION_INCORRECT, -902, true);                   \
     E(PUSH_SCHEMA_MISMATCH, -903, true);                     \
@@ -228,7 +227,6 @@ namespace ErrorCode {
     E(CUMULATIVE_INVALID_NEED_MERGED_VERSIONS, -2004, true); \
     E(CUMULATIVE_ERROR_DELETE_ACTION, -2005, true);          \
     E(CUMULATIVE_MISS_VERSION, -2006, true);                 \
-    E(CUMULATIVE_CLONE_OCCURRED, -2007, true);               \
     E(FULL_NO_SUITABLE_VERSION, -2008, false);               \
     E(FULL_MISS_VERSION, -2009, true);                       \
     E(META_INVALID_ARGUMENT, -3000, true);                   \
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
index 6ae006709a77ab..474909cbf45b65 100644
--- a/be/src/olap/base_compaction.cpp
+++ b/be/src/olap/base_compaction.cpp
@@ -56,7 +56,6 @@ Status BaseCompaction::prepare_compact() {
     // 1. pick rowsets to compact
     RETURN_IF_ERROR(pick_rowsets_to_compact());
     COUNTER_UPDATE(_input_rowsets_counter, _input_rowsets.size());
-    _tablet->set_clone_occurred(false);
 
     return Status::OK();
 }
@@ -73,13 +72,6 @@ Status BaseCompaction::execute_compact_impl() {
                 "another base compaction is running. tablet={}", _tablet->tablet_id());
     }
 
-    // Clone task may happen after compaction task is submitted to thread pool, and rowsets picked
-    // for compaction may change. In this case, current compaction task should not be executed.
-    if (_tablet->get_clone_occurred()) {
-        _tablet->set_clone_occurred(false);
-        return Status::Error<BE_CLONE_OCCURRED, false>("get_clone_occurred failed");
-    }
-
     SCOPED_ATTACH_TASK(_mem_tracker);
 
     // 2. do base compaction, merge rowsets
diff --git a/be/src/olap/base_compaction.h b/be/src/olap/base_compaction.h
index 73aca0d5e1f0ec..e86cb3330a56c4 100644
--- a/be/src/olap/base_compaction.h
+++ b/be/src/olap/base_compaction.h
@@ -43,8 +43,6 @@ class BaseCompaction : public Compaction {
     Status prepare_compact() override;
     Status execute_compact_impl() override;
 
-    std::vector<RowsetSharedPtr> get_input_rowsets() { return _input_rowsets; }
-
 protected:
     Status pick_rowsets_to_compact() override;
     std::string compaction_name() const override { return "base compaction"; }
diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h
index 2aff05ced83748..a8e3b1a5c284ae 100644
--- a/be/src/olap/compaction.h
+++ b/be/src/olap/compaction.h
@@ -58,6 +58,8 @@ class Compaction {
     virtual Status prepare_compact() = 0;
     Status execute_compact();
     virtual Status execute_compact_impl() = 0;
+
+    const std::vector<RowsetSharedPtr>& input_rowsets() { return _input_rowsets; }
 #ifdef BE_TEST
     void set_input_rowset(const std::vector<RowsetSharedPtr>& rowsets);
     RowsetSharedPtr output_rowset();
@@ -65,10 +67,11 @@ class Compaction {
 
     RuntimeProfile* runtime_profile() const { return _profile.get(); }
 
+    virtual ReaderType compaction_type() const = 0;
+    virtual std::string compaction_name() const = 0;
+
 protected:
     virtual Status pick_rowsets_to_compact() = 0;
-    virtual std::string compaction_name() const = 0;
-    virtual ReaderType compaction_type() const = 0;
 
     Status do_compaction(int64_t permits);
     Status do_compaction_impl(int64_t permits);
diff --git a/be/src/olap/cumulative_compaction.cpp b/be/src/olap/cumulative_compaction.cpp
index 1c65df768d00e5..1f54c1f32855c9 100644
--- a/be/src/olap/cumulative_compaction.cpp
+++ b/be/src/olap/cumulative_compaction.cpp
@@ -61,7 +61,6 @@ Status CumulativeCompaction::prepare_compact() {
     // 2. pick rowsets to compact
     RETURN_IF_ERROR(pick_rowsets_to_compact());
     COUNTER_UPDATE(_input_rowsets_counter, _input_rowsets.size());
-    _tablet->set_clone_occurred(false);
 
     return Status::OK();
 }
@@ -73,13 +72,6 @@ Status CumulativeCompaction::execute_compact_impl() {
                 "The tablet is under cumulative compaction. tablet={}", _tablet->tablet_id());
     }
 
-    // Clone task may happen after compaction task is submitted to thread pool, and rowsets picked
-    // for compaction may change. In this case, current compaction task should not be executed.
-    if (_tablet->get_clone_occurred()) {
-        _tablet->set_clone_occurred(false);
-        return Status::Error<CUMULATIVE_CLONE_OCCURRED, false>("get_clone_occurred failed");
-    }
-
     SCOPED_ATTACH_TASK(_mem_tracker);
 
     // 3. do cumulative compaction, merge rowsets
diff --git a/be/src/olap/cumulative_compaction.h b/be/src/olap/cumulative_compaction.h
index d74542a2ffe070..7ea7fb383f1ebb 100644
--- a/be/src/olap/cumulative_compaction.h
+++ b/be/src/olap/cumulative_compaction.h
@@ -39,8 +39,6 @@ class CumulativeCompaction : public Compaction {
     Status prepare_compact() override;
     Status execute_compact_impl() override;
 
-    std::vector<RowsetSharedPtr> get_input_rowsets() { return _input_rowsets; }
-
 protected:
     Status pick_rowsets_to_compact() override;
 
diff --git a/be/src/olap/full_compaction.cpp b/be/src/olap/full_compaction.cpp
index 01bd5e3dc613e7..927b4a33198b15 100644
--- a/be/src/olap/full_compaction.cpp
+++ b/be/src/olap/full_compaction.cpp
@@ -51,29 +51,19 @@ Status FullCompaction::prepare_compact() {
         return Status::Error<INVALID_ARGUMENT, false>("Full compaction init failed");
     }
 
-    std::unique_lock full_lock(_tablet->get_full_compaction_lock());
     std::unique_lock base_lock(_tablet->get_base_compaction_lock());
     std::unique_lock cumu_lock(_tablet->get_cumulative_compaction_lock());
 
     // 1. pick rowsets to compact
     RETURN_IF_ERROR(pick_rowsets_to_compact());
-    _tablet->set_clone_occurred(false);
 
     return Status::OK();
 }
 
 Status FullCompaction::execute_compact_impl() {
-    std::unique_lock full_lock(_tablet->get_full_compaction_lock());
     std::unique_lock base_lock(_tablet->get_base_compaction_lock());
     std::unique_lock cumu_lock(_tablet->get_cumulative_compaction_lock());
 
-    // Clone task may happen after compaction task is submitted to thread pool, and rowsets picked
-    // for compaction may change. In this case, current compaction task should not be executed.
-    if (_tablet->get_clone_occurred()) {
-        _tablet->set_clone_occurred(false);
-        return Status::Error<BE_CLONE_OCCURRED, false>("get_clone_occurred failed");
-    }
-
     SCOPED_ATTACH_TASK(_mem_tracker);
 
     // 2. do full compaction, merge rowsets
diff --git a/be/src/olap/full_compaction.h b/be/src/olap/full_compaction.h
index bce9ac745b6bb2..631d901e8466cf 100644
--- a/be/src/olap/full_compaction.h
+++ b/be/src/olap/full_compaction.h
@@ -39,8 +39,6 @@ class FullCompaction : public Compaction {
     Status execute_compact_impl() override;
     Status modify_rowsets(const Merger::Statistics* stats = nullptr) override;
 
-    std::vector<RowsetSharedPtr> get_input_rowsets() { return _input_rowsets; }
-
 protected:
     Status pick_rowsets_to_compact() override;
     std::string compaction_name() const override { return "full compaction"; }
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index f7caa7d8d02967..7268d7959f998a 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -57,6 +57,7 @@
 #include "olap/olap_common.h"
 #include "olap/rowset/segcompaction.h"
 #include "olap/schema_change.h"
+#include "olap/single_replica_compaction.h"
 #include "olap/storage_engine.h"
 #include "olap/tablet.h"
 #include "olap/tablet_manager.h"
@@ -593,11 +594,6 @@ void StorageEngine::_compaction_tasks_producer_callback() {
                 continue;
             }
 
-            /// Regardless of whether the tablet is submitted for compaction or not,
-            /// we need to call 'reset_compaction' to clean up the base_compaction or cumulative_compaction objects
-            /// in the tablet, because these two objects store the tablet's own shared_ptr.
-            /// If it is not cleaned up, the reference count of the tablet will always be greater than 1,
-            /// thus cannot be collected by the garbage collector. (TabletManager::start_trash_sweep)
             for (const auto& tablet : tablets_compaction) {
                 if (compaction_type == CompactionType::BASE_COMPACTION) {
                     tablet->set_last_base_compaction_schedule_time(UnixMillis());
@@ -717,33 +713,39 @@ Status StorageEngine::_submit_single_replica_compaction_task(TabletSharedPtr tab
         return Status::AlreadyExist<false>(
                 "compaction task has already been submitted, tablet_id={}", tablet->tablet_id());
     }
-    Status st = tablet->prepare_single_replica_compaction(tablet, compaction_type);
+
+    auto compaction = std::make_shared<SingleReplicaCompaction>(tablet, compaction_type);
+    auto st = compaction->prepare_compact();
+
     auto clean_single_replica_compaction = [tablet, this]() {
-        tablet->reset_single_replica_compaction();
         _pop_tablet_from_submitted_compaction(tablet, CompactionType::CUMULATIVE_COMPACTION);
         _pop_tablet_from_submitted_compaction(tablet, CompactionType::BASE_COMPACTION);
     };
 
-    if (st.ok()) {
-        auto submit_st = _single_replica_compaction_thread_pool->submit_func(
-                [tablet, compaction_type, clean_single_replica_compaction]() {
-                    tablet->execute_single_replica_compaction(compaction_type);
-                    clean_single_replica_compaction();
-                });
-        if (!submit_st.ok()) {
-            clean_single_replica_compaction();
-            return Status::InternalError(
-                    "failed to submit single replica compaction task to thread pool, "
-                    "tablet_id={} ",
-                    tablet->tablet_id());
+    if (!st.ok()) {
+        clean_single_replica_compaction();
+        if (!st.is<ErrorCode::CUMULATIVE_NO_SUITABLE_VERSION>()) {
+            LOG(WARNING) << "failed to prepare single replica compaction, tablet_id="
+                         << tablet->tablet_id() << " : " << st;
+            return st;
         }
-        return Status::OK();
-    } else {
+        return Status::OK(); // No suitable version, regard as OK
+    }
+
+    auto submit_st = _single_replica_compaction_thread_pool->submit_func(
+            [tablet, compaction = std::move(compaction),
+             clean_single_replica_compaction]() mutable {
+                tablet->execute_single_replica_compaction(*compaction);
+                clean_single_replica_compaction();
+            });
+    if (!submit_st.ok()) {
         clean_single_replica_compaction();
         return Status::InternalError(
-                "failed to prepare single replica compaction task tablet_id={} ",
+                "failed to submit single replica compaction task to thread pool, "
+                "tablet_id={}",
                 tablet->tablet_id());
     }
+    return Status::OK();
 }
 
 void StorageEngine::get_tablet_rowset_versions(const PGetTabletVersionsRequest* request,
@@ -917,8 +919,10 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet,
                 "compaction task has already been submitted, tablet_id={}, compaction_type={}.",
                 tablet->tablet_id(), compaction_type);
     }
+    std::shared_ptr<Compaction> compaction;
     int64_t permits = 0;
-    Status st = tablet->prepare_compaction_and_calculate_permits(compaction_type, tablet, &permits);
+    Status st = Tablet::prepare_compaction_and_calculate_permits(compaction_type, tablet,
+                                                                 compaction, permits);
     bool is_low_priority_task = [&]() {
         // Can add more strategies to determine whether a task is a low priority task in the future
         if (!config::enable_compaction_priority_scheduling) {
@@ -938,27 +942,24 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet,
                 (compaction_type == CompactionType::CUMULATIVE_COMPACTION)
                         ? _cumu_compaction_thread_pool
                         : _base_compaction_thread_pool;
-        auto st = thread_pool->submit_func([tablet, compaction_type, permits, is_low_priority_task,
+        auto st = thread_pool->submit_func([tablet, compaction = std::move(compaction),
+                                            compaction_type, permits, is_low_priority_task,
                                             this]() {
             if (is_low_priority_task && !_increase_low_priority_task_nums(tablet->data_dir())) {
                 VLOG_DEBUG << "skip low priority compaction task for tablet: "
                            << tablet->tablet_id();
                 // Todo: push task back
             } else {
-                tablet->execute_compaction(compaction_type);
+                tablet->execute_compaction(*compaction);
                 if (is_low_priority_task) {
                     _decrease_low_priority_task_nums(tablet->data_dir());
                 }
             }
             _permit_limiter.release(permits);
-            // reset compaction
-            tablet->reset_compaction(compaction_type);
             _pop_tablet_from_submitted_compaction(tablet, compaction_type);
         });
         if (!st.ok()) {
             _permit_limiter.release(permits);
-            // reset compaction
-            tablet->reset_compaction(compaction_type);
             _pop_tablet_from_submitted_compaction(tablet, compaction_type);
             return Status::InternalError(
                     "failed to submit compaction task to thread pool, "
@@ -967,8 +968,6 @@ Status StorageEngine::_submit_compaction_task(TabletSharedPtr tablet,
         }
         return Status::OK();
     } else {
-        // reset compaction
-        tablet->reset_compaction(compaction_type);
         _pop_tablet_from_submitted_compaction(tablet, compaction_type);
         if (!st.ok()) {
             return Status::InternalError(
diff --git a/be/src/olap/single_replica_compaction.cpp b/be/src/olap/single_replica_compaction.cpp
index a5e060147d6b36..2a8edb60471154 100644
--- a/be/src/olap/single_replica_compaction.cpp
+++ b/be/src/olap/single_replica_compaction.cpp
@@ -70,7 +70,6 @@ Status SingleReplicaCompaction::prepare_compact() {
 
     // 1. pick rowsets to compact
     RETURN_IF_ERROR(pick_rowsets_to_compact());
-    _tablet->set_clone_occurred(false);
     if (_input_rowsets.size() == 1) {
         return Status::Error<CUMULATIVE_NO_SUITABLE_VERSION>("_input_rowsets.size() is 1");
     }
@@ -105,13 +104,6 @@ Status SingleReplicaCompaction::execute_compact_impl() {
                 "another base compaction is running. tablet={}", _tablet->tablet_id());
     }
 
-    // Clone task may happen after compaction task is submitted to thread pool, and rowsets picked
-    // for compaction may change. In this case, current compaction task should not be executed.
-    if (_tablet->get_clone_occurred()) {
-        _tablet->set_clone_occurred(false);
-        return Status::Error<BE_CLONE_OCCURRED, false>("get_clone_occurred failed");
-    }
-
     SCOPED_ATTACH_TASK(_mem_tracker);
 
     // 2. do single replica compaction
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index fd865712c5f70f..b5090f0e280cbe 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -426,10 +426,10 @@ Status SnapshotManager::_create_snapshot_files(const TabletSharedPtr& ref_tablet
                           << ref_tablet->tablet_id();
                 Version version(request.start_version, request.end_version);
                 const RowsetSharedPtr rowset = ref_tablet->get_rowset_by_version(version, false);
-                if (rowset != nullptr) {
+                if (rowset && rowset->is_local()) {
                     consistent_rowsets.push_back(rowset);
                 } else {
-                    LOG(WARNING) << "failed to find version when do compaction snapshot. "
+                    LOG(WARNING) << "failed to find local version when do compaction snapshot. "
                                  << " tablet=" << request.tablet_id
                                  << " schema_hash=" << request.schema_hash
                                  << " version=" << version;
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 4e0fb2eddf19f6..cf4687caf3e6bc 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -1256,28 +1256,6 @@ PendingRowsetGuard StorageEngine::add_pending_rowset(const RowsetWriterContext&
     return _pending_remote_rowsets.add(ctx.rowset_id);
 }
 
-void StorageEngine::create_cumulative_compaction(
-        TabletSharedPtr best_tablet, std::shared_ptr<CumulativeCompaction>& cumulative_compaction) {
-    cumulative_compaction.reset(new CumulativeCompaction(best_tablet));
-}
-
-void StorageEngine::create_base_compaction(TabletSharedPtr best_tablet,
-                                           std::shared_ptr<BaseCompaction>& base_compaction) {
-    base_compaction.reset(new BaseCompaction(best_tablet));
-}
-
-void StorageEngine::create_full_compaction(TabletSharedPtr best_tablet,
-                                           std::shared_ptr<FullCompaction>& full_compaction) {
-    full_compaction.reset(new FullCompaction(best_tablet));
-}
-
-void StorageEngine::create_single_replica_compaction(
-        TabletSharedPtr best_tablet,
-        std::shared_ptr<SingleReplicaCompaction>& single_replica_compaction,
-        CompactionType compaction_type) {
-    single_replica_compaction.reset(new SingleReplicaCompaction(best_tablet, compaction_type));
-}
-
 bool StorageEngine::get_peer_replica_info(int64_t tablet_id, TReplicaInfo* replica,
                                           std::string* token) {
     TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id);
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 6b028b01734e2d..77d3efeaaf8e69 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -180,18 +180,6 @@ class StorageEngine {
     void get_tablet_rowset_versions(const PGetTabletVersionsRequest* request,
                                     PGetTabletVersionsResponse* response);
 
-    void create_cumulative_compaction(TabletSharedPtr best_tablet,
-                                      std::shared_ptr<CumulativeCompaction>& cumulative_compaction);
-    void create_base_compaction(TabletSharedPtr best_tablet,
-                                std::shared_ptr<BaseCompaction>& base_compaction);
-
-    void create_full_compaction(TabletSharedPtr best_tablet,
-                                std::shared_ptr<FullCompaction>& full_compaction);
-
-    void create_single_replica_compaction(
-            TabletSharedPtr best_tablet,
-            std::shared_ptr<SingleReplicaCompaction>& single_replica_compaction,
-            CompactionType compaction_type);
     bool get_peer_replica_info(int64_t tablet_id, TReplicaInfo* replica, std::string* token);
 
     bool should_fetch_from_peer(int64_t tablet_id);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 5f915ad1de16c9..8cfce2d35ec523 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -69,6 +69,7 @@
 #include "io/fs/file_writer.h"
 #include "io/fs/path.h"
 #include "io/fs/remote_file_system.h"
+#include "io/io_common.h"
 #include "olap/base_compaction.h"
 #include "olap/base_tablet.h"
 #include "olap/binlog.h"
@@ -143,21 +144,39 @@ using std::string;
 using std::vector;
 using io::FileSystemSPtr;
 
-static bvar::LatencyRecorder g_tablet_lookup_rowkey_latency("doris_pk", "tablet_lookup_rowkey");
-static bvar::LatencyRecorder g_tablet_commit_phase_update_delete_bitmap_latency(
+namespace {
+
+bvar::LatencyRecorder g_tablet_lookup_rowkey_latency("doris_pk", "tablet_lookup_rowkey");
+bvar::LatencyRecorder g_tablet_commit_phase_update_delete_bitmap_latency(
         "doris_pk", "commit_phase_update_delete_bitmap");
-static bvar::LatencyRecorder g_tablet_update_delete_bitmap_latency("doris_pk",
-                                                                   "update_delete_bitmap");
-static bvar::Adder<uint64_t> g_tablet_pk_not_found("doris_pk", "lookup_not_found");
-static bvar::PerSecond<bvar::Adder<uint64_t>> g_tablet_pk_not_found_per_second(
+bvar::LatencyRecorder g_tablet_update_delete_bitmap_latency("doris_pk", "update_delete_bitmap");
+bvar::Adder<uint64_t> g_tablet_pk_not_found("doris_pk", "lookup_not_found");
+bvar::PerSecond<bvar::Adder<uint64_t>> g_tablet_pk_not_found_per_second(
         "doris_pk", "lookup_not_found_per_second", &g_tablet_pk_not_found, 60);
 
-const std::chrono::seconds TRACE_TABLET_LOCK_THRESHOLD = 1s;
-
 bvar::Adder<uint64_t> exceed_version_limit_counter;
 bvar::Window<bvar::Adder<uint64_t>> exceed_version_limit_counter_minute(
         &exceed_version_limit_counter, 60);
 
+void set_last_failure_time(Tablet* tablet, const Compaction& compaction, int64_t ms) {
+    switch (compaction.compaction_type()) {
+    case ReaderType::READER_CUMULATIVE_COMPACTION:
+        tablet->set_last_cumu_compaction_failure_time(ms);
+        return;
+    case ReaderType::READER_BASE_COMPACTION:
+        tablet->set_last_base_compaction_failure_time(ms);
+        return;
+    case ReaderType::READER_FULL_COMPACTION:
+        tablet->set_last_full_compaction_failure_time(ms);
+        return;
+    default:
+        LOG(FATAL) << "invalid compaction type " << compaction.compaction_name()
+                   << " tablet_id: " << tablet->tablet_id();
+    }
+};
+
+} // namespace
+
 struct WriteCooldownMetaExecutors {
     WriteCooldownMetaExecutors(size_t executor_nums = 5);
 
@@ -251,7 +270,6 @@ Tablet::Tablet(TabletMetaSharedPtr tablet_meta, DataDir* data_dir,
           _newly_created_rowset_num(0),
           _last_checkpoint_time(0),
           _cumulative_compaction_type(cumulative_compaction_type),
-          _is_clone_occurred(false),
           _is_tablet_path_exists(true),
           _last_missed_version(-1),
           _last_missed_time_s(0) {
@@ -475,6 +493,22 @@ Status Tablet::modify_rowsets(std::vector<RowsetSharedPtr>& to_add,
         return Status::OK();
     }
 
+    if (check_delete) {
+        for (auto&& rs : to_delete) {
+            if (auto it = _rs_version_map.find(rs->version()); it == _rs_version_map.end()) {
+                return Status::Error<DELETE_VERSION_ERROR>(
+                        "try to delete not exist version {} from {}", rs->version().to_string(),
+                        tablet_id());
+            } else if (rs->rowset_id() != it->second->rowset_id()) {
+                return Status::Error<DELETE_VERSION_ERROR>(
+                        "try to delete version {} from {}, but rowset id changed, delete rowset id "
+                        "is {}, exists rowsetid is {}",
+                        rs->version().to_string(), tablet_id(), rs->rowset_id().to_string(),
+                        it->second->rowset_id().to_string());
+            }
+        }
+    }
+
     bool same_version = true;
     std::sort(to_add.begin(), to_add.end(), Rowset::comparator);
     std::sort(to_delete.begin(), to_delete.end(), Rowset::comparator);
@@ -489,23 +523,6 @@ Status Tablet::modify_rowsets(std::vector<RowsetSharedPtr>& to_add,
         same_version = false;
     }
 
-    if (check_delete) {
-        for (auto& rs : to_delete) {
-            auto find_rs = _rs_version_map.find(rs->version());
-            if (find_rs == _rs_version_map.end()) {
-                return Status::Error<DELETE_VERSION_ERROR>(
-                        "try to delete not exist version {} from {}", rs->version().to_string(),
-                        tablet_id());
-            } else if (find_rs->second->rowset_id() != rs->rowset_id()) {
-                return Status::Error<DELETE_VERSION_ERROR>(
-                        "try to delete version {} from {}, but rowset id changed, delete rowset id "
-                        "is {}, exists rowsetid is {}",
-                        rs->version().to_string(), tablet_id(), rs->rowset_id().to_string(),
-                        find_rs->second->rowset_id().to_string());
-            }
-        }
-    }
-
     std::vector<RowsetMetaSharedPtr> rs_metas_to_delete;
     for (auto& rs : to_delete) {
         rs_metas_to_delete.push_back(rs->rowset_meta());
@@ -1755,138 +1772,100 @@ void Tablet::generate_tablet_meta_copy_unlocked(TabletMetaSharedPtr new_tablet_m
 }
 
 Status Tablet::prepare_compaction_and_calculate_permits(CompactionType compaction_type,
-                                                        TabletSharedPtr tablet, int64_t* permits) {
-    std::vector<RowsetSharedPtr> compaction_rowsets;
+                                                        const TabletSharedPtr& tablet,
+                                                        std::shared_ptr<Compaction>& compaction,
+                                                        int64_t& permits) {
     if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
         MonotonicStopWatch watch;
         watch.start();
-        SCOPED_CLEANUP({
-            if (!config::disable_compaction_trace_log &&
-                watch.elapsed_time() / 1e9 > config::cumulative_compaction_trace_threshold) {
-                std::stringstream ss;
-                _cumulative_compaction->runtime_profile()->pretty_print(&ss);
-                LOG(WARNING) << "prepare cumulative compaction cost " << watch.elapsed_time() / 1e9
-                             << std::endl
-                             << ss.str();
-            }
-        });
 
-        StorageEngine::instance()->create_cumulative_compaction(tablet, _cumulative_compaction);
+        compaction = std::make_shared<CumulativeCompaction>(tablet);
         DorisMetrics::instance()->cumulative_compaction_request_total->increment(1);
-        Status res = _cumulative_compaction->prepare_compact();
+        Status res = compaction->prepare_compact();
+        if (!config::disable_compaction_trace_log &&
+            watch.elapsed_time() / 1e9 > config::cumulative_compaction_trace_threshold) {
+            std::stringstream ss;
+            compaction->runtime_profile()->pretty_print(&ss);
+            LOG(WARNING) << "prepare cumulative compaction cost " << watch.elapsed_time() / 1e9
+                         << std::endl
+                         << ss.str();
+        }
+
         if (!res.ok()) {
-            set_last_cumu_compaction_failure_time(UnixMillis());
-            *permits = 0;
+            tablet->set_last_cumu_compaction_failure_time(UnixMillis());
+            permits = 0;
             if (!res.is<CUMULATIVE_NO_SUITABLE_VERSION>()) {
                 DorisMetrics::instance()->cumulative_compaction_request_failed->increment(1);
-                return Status::InternalError("prepare cumulative compaction with err: {}",
-                                             res.to_string());
+                return Status::InternalError("prepare cumulative compaction with err: {}", res);
             }
             // return OK if OLAP_ERR_CUMULATIVE_NO_SUITABLE_VERSION, so that we don't need to
             // print too much useless logs.
             // And because we set permits to 0, so even if we return OK here, nothing will be done.
             return Status::OK();
         }
-        compaction_rowsets = _cumulative_compaction->get_input_rowsets();
     } else if (compaction_type == CompactionType::BASE_COMPACTION) {
-        DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION);
         MonotonicStopWatch watch;
         watch.start();
-        SCOPED_CLEANUP({
-            if (!config::disable_compaction_trace_log &&
-                watch.elapsed_time() / 1e9 > config::base_compaction_trace_threshold) {
-                std::stringstream ss;
-                _base_compaction->runtime_profile()->pretty_print(&ss);
-                LOG(WARNING) << "prepare base compaction cost " << watch.elapsed_time() / 1e9
-                             << std::endl
-                             << ss.str();
-            }
-        });
 
-        StorageEngine::instance()->create_base_compaction(tablet, _base_compaction);
+        compaction = std::make_shared<BaseCompaction>(tablet);
         DorisMetrics::instance()->base_compaction_request_total->increment(1);
-        Status res = _base_compaction->prepare_compact();
-        set_last_base_compaction_status(res.to_string());
+        Status res = compaction->prepare_compact();
+        if (!config::disable_compaction_trace_log &&
+            watch.elapsed_time() / 1e9 > config::base_compaction_trace_threshold) {
+            std::stringstream ss;
+            compaction->runtime_profile()->pretty_print(&ss);
+            LOG(WARNING) << "prepare base compaction cost " << watch.elapsed_time() / 1e9
+                         << std::endl
+                         << ss.str();
+        }
+
+        tablet->set_last_base_compaction_status(res.to_string());
         if (!res.ok()) {
-            set_last_base_compaction_failure_time(UnixMillis());
-            *permits = 0;
+            tablet->set_last_base_compaction_failure_time(UnixMillis());
+            permits = 0;
             if (!res.is<BE_NO_SUITABLE_VERSION>()) {
                 DorisMetrics::instance()->base_compaction_request_failed->increment(1);
-                return Status::InternalError("prepare base compaction with err: {}",
-                                             res.to_string());
+                return Status::InternalError("prepare base compaction with err: {}", res);
             }
             // return OK if OLAP_ERR_BE_NO_SUITABLE_VERSION, so that we don't need to
             // print too much useless logs.
             // And because we set permits to 0, so even if we return OK here, nothing will be done.
             return Status::OK();
         }
-        compaction_rowsets = _base_compaction->get_input_rowsets();
     } else {
         DCHECK_EQ(compaction_type, CompactionType::FULL_COMPACTION);
-        MonotonicStopWatch watch;
-        watch.start();
-        StorageEngine::instance()->create_full_compaction(tablet, _full_compaction);
-        Status res = _full_compaction->prepare_compact();
+
+        compaction = std::make_shared<FullCompaction>(tablet);
+        Status res = compaction->prepare_compact();
         if (!res.ok()) {
-            set_last_full_compaction_failure_time(UnixMillis());
-            *permits = 0;
+            tablet->set_last_full_compaction_failure_time(UnixMillis());
+            permits = 0;
             if (!res.is<FULL_NO_SUITABLE_VERSION>()) {
-                return Status::InternalError("prepare full compaction with err: {}",
-                                             res.to_string());
+                return Status::InternalError("prepare full compaction with err: {}", res);
             }
             // return OK if OLAP_ERR_BE_NO_SUITABLE_VERSION, so that we don't need to
             // print too much useless logs.
             // And because we set permits to 0, so even if we return OK here, nothing will be done.
             return Status::OK();
         }
-        compaction_rowsets = _full_compaction->get_input_rowsets();
-    }
-    *permits = 0;
-    for (auto rowset : compaction_rowsets) {
-        *permits += rowset->rowset_meta()->get_compaction_score();
     }
-    return Status::OK();
-}
 
-Status Tablet::prepare_single_replica_compaction(TabletSharedPtr tablet,
-                                                 CompactionType compaction_type) {
-    StorageEngine::instance()->create_single_replica_compaction(tablet, _single_replica_compaction,
-                                                                compaction_type);
-    Status res = _single_replica_compaction->prepare_compact();
-    if (!res.ok()) {
-        if (!res.is<CUMULATIVE_NO_SUITABLE_VERSION>()) {
-            return Status::InternalError("prepare single replica compaction with err: {}",
-                                         res.to_string());
-        }
+    permits = 0;
+    for (auto&& rowset : compaction->input_rowsets()) {
+        permits += rowset->rowset_meta()->get_compaction_score();
     }
     return Status::OK();
 }
 
-void Tablet::execute_single_replica_compaction(CompactionType compaction_type) {
-    Status res = _single_replica_compaction->execute_compact();
+void Tablet::execute_single_replica_compaction(SingleReplicaCompaction& compaction) {
+    Status res = compaction.execute_compact();
     if (!res.ok()) {
-        if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
-            set_last_cumu_compaction_failure_time(UnixMillis());
-        } else if (compaction_type == CompactionType::BASE_COMPACTION) {
-            set_last_base_compaction_failure_time(UnixMillis());
-        } else if (compaction_type == CompactionType::FULL_COMPACTION) {
-            set_last_full_compaction_failure_time(UnixMillis());
-        }
+        set_last_failure_time(this, compaction, UnixMillis());
         LOG(WARNING) << "failed to do single replica compaction. res=" << res
                      << ", tablet=" << tablet_id();
         return;
     }
-    if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
-        set_last_cumu_compaction_failure_time(0);
-    } else if (compaction_type == CompactionType::BASE_COMPACTION) {
-        set_last_base_compaction_failure_time(0);
-    } else if (compaction_type == CompactionType::FULL_COMPACTION) {
-        set_last_full_compaction_failure_time(0);
-    }
-}
-
-void Tablet::reset_single_replica_compaction() {
-    _single_replica_compaction.reset();
+    set_last_failure_time(this, compaction, 0);
 }
 
 std::vector<Version> Tablet::get_all_versions() {
@@ -1903,78 +1882,38 @@ std::vector<Version> Tablet::get_all_versions() {
     return local_versions;
 }
 
-void Tablet::execute_compaction(CompactionType compaction_type) {
+void Tablet::execute_compaction(Compaction& compaction) {
     signal::tablet_id = tablet_id();
-    if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
-        MonotonicStopWatch watch;
-        watch.start();
-        SCOPED_CLEANUP({
-            if (!config::disable_compaction_trace_log &&
-                watch.elapsed_time() / 1e9 > config::cumulative_compaction_trace_threshold) {
-                std::stringstream ss;
-                _cumulative_compaction->runtime_profile()->pretty_print(&ss);
-                LOG(WARNING) << "execute cumulative compaction cost " << watch.elapsed_time() / 1e9
-                             << std::endl
-                             << ss.str();
-            }
-        });
 
-        Status res = _cumulative_compaction->execute_compact();
-        if (!res.ok()) {
-            set_last_cumu_compaction_failure_time(UnixMillis());
-            DorisMetrics::instance()->cumulative_compaction_request_failed->increment(1);
-            LOG(WARNING) << "failed to do cumulative compaction. res=" << res
-                         << ", tablet=" << tablet_id();
-            return;
-        }
-        set_last_cumu_compaction_failure_time(0);
-    } else if (compaction_type == CompactionType::BASE_COMPACTION) {
-        DCHECK_EQ(compaction_type, CompactionType::BASE_COMPACTION);
-        MonotonicStopWatch watch;
-        watch.start();
-        SCOPED_CLEANUP({
-            if (!config::disable_compaction_trace_log &&
-                watch.elapsed_time() / 1e9 > config::base_compaction_trace_threshold) {
-                std::stringstream ss;
-                _base_compaction->runtime_profile()->pretty_print(&ss);
-                LOG(WARNING) << "execute base compaction cost " << watch.elapsed_time() / 1e9
-                             << std::endl
-                             << ss.str();
-            }
-        });
+    MonotonicStopWatch watch;
+    watch.start();
 
-        Status res = _base_compaction->execute_compact();
-        set_last_base_compaction_status(res.to_string());
-        if (!res.ok()) {
-            set_last_base_compaction_failure_time(UnixMillis());
-            DorisMetrics::instance()->base_compaction_request_failed->increment(1);
-            LOG(WARNING) << "failed to do base compaction. res=" << res
-                         << ", tablet=" << tablet_id();
-            return;
-        }
-        set_last_base_compaction_failure_time(0);
+    Status res = compaction.execute_compact();
+
+    if (!res.ok()) [[unlikely]] {
+        set_last_failure_time(this, compaction, UnixMillis());
+        LOG(WARNING) << "failed to do " << compaction.compaction_name()
+                     << ", tablet=" << tablet_id() << " : " << res;
     } else {
-        DCHECK_EQ(compaction_type, CompactionType::FULL_COMPACTION);
-        MonotonicStopWatch watch;
-        watch.start();
-        Status res = _full_compaction->execute_compact();
-        if (!res.ok()) {
-            set_last_full_compaction_failure_time(UnixMillis());
-            LOG(WARNING) << "failed to do full compaction. res=" << res
-                         << ", tablet=" << tablet_id();
-            return;
-        }
-        set_last_full_compaction_failure_time(0);
+        set_last_failure_time(this, compaction, 0);
     }
-}
 
-void Tablet::reset_compaction(CompactionType compaction_type) {
-    if (compaction_type == CompactionType::CUMULATIVE_COMPACTION) {
-        _cumulative_compaction.reset();
-    } else if (compaction_type == CompactionType::BASE_COMPACTION) {
-        _base_compaction.reset();
-    } else {
-        _full_compaction.reset();
+    if (!config::disable_compaction_trace_log) {
+        auto need_trace = [&compaction, &watch] {
+            return compaction.compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION
+                           ? watch.elapsed_time() / 1e9 >
+                                     config::cumulative_compaction_trace_threshold
+                   : compaction.compaction_type() == ReaderType::READER_BASE_COMPACTION
+                           ? watch.elapsed_time() / 1e9 > config::base_compaction_trace_threshold
+                           : false;
+        };
+        if (need_trace()) {
+            std::stringstream ss;
+            compaction.runtime_profile()->pretty_print(&ss);
+            LOG(WARNING) << "execute " << compaction.compaction_name() << " cost "
+                         << watch.elapsed_time() / 1e9 << std::endl
+                         << ss.str();
+        }
     }
 }
 
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 4174a0ec26db13..614288921ed3f8 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -59,9 +59,7 @@ namespace doris {
 
 class Tablet;
 class CumulativeCompactionPolicy;
-class CumulativeCompaction;
-class BaseCompaction;
-class FullCompaction;
+class Compaction;
 class SingleReplicaCompaction;
 class RowsetWriter;
 struct RowsetWriterContext;
@@ -85,7 +83,7 @@ enum SortType : int;
 
 enum TabletStorageType { STORAGE_TYPE_LOCAL, STORAGE_TYPE_REMOTE, STORAGE_TYPE_REMOTE_AND_LOCAL };
 
-extern const std::chrono::seconds TRACE_TABLET_LOCK_THRESHOLD;
+static inline constexpr auto TRACE_TABLET_LOCK_THRESHOLD = std::chrono::seconds(1);
 
 class Tablet final : public BaseTablet {
 public:
@@ -146,6 +144,8 @@ class Tablet final : public BaseTablet {
     // operation in rowsets
     Status add_rowset(RowsetSharedPtr rowset);
     Status create_initial_rowset(const int64_t version);
+
+    // MUST hold EXCLUSIVE `_meta_lock`.
     Status modify_rowsets(std::vector<RowsetSharedPtr>& to_add,
                           std::vector<RowsetSharedPtr>& to_delete, bool check_delete = false);
 
@@ -194,7 +194,6 @@ class Tablet final : public BaseTablet {
     std::mutex& get_push_lock() { return _ingest_lock; }
     std::mutex& get_base_compaction_lock() { return _base_compaction_lock; }
     std::mutex& get_cumulative_compaction_lock() { return _cumulative_compaction_lock; }
-    std::mutex& get_full_compaction_lock() { return _full_compaction_lock; }
 
     std::shared_mutex& get_migration_lock() { return _migration_lock; }
 
@@ -296,18 +295,13 @@ class Tablet final : public BaseTablet {
     // return a json string to show the compaction status of this tablet
     void get_compaction_status(std::string* json_result);
 
-    Status prepare_compaction_and_calculate_permits(CompactionType compaction_type,
-                                                    TabletSharedPtr tablet, int64_t* permits);
-
-    Status prepare_single_replica_compaction(TabletSharedPtr tablet,
-                                             CompactionType compaction_type);
-    void execute_compaction(CompactionType compaction_type);
-    void reset_compaction(CompactionType compaction_type);
-    void execute_single_replica_compaction(CompactionType compaction_type);
-    void reset_single_replica_compaction();
+    static Status prepare_compaction_and_calculate_permits(CompactionType compaction_type,
+                                                           const TabletSharedPtr& tablet,
+                                                           std::shared_ptr<Compaction>& compaction,
+                                                           int64_t& permits);
 
-    void set_clone_occurred(bool clone_occurred) { _is_clone_occurred = clone_occurred; }
-    bool get_clone_occurred() { return _is_clone_occurred; }
+    void execute_compaction(Compaction& compaction);
+    void execute_single_replica_compaction(SingleReplicaCompaction& compaction);
 
     void set_cumulative_compaction_policy(
             std::shared_ptr<CumulativeCompactionPolicy> cumulative_compaction_policy) {
@@ -319,7 +313,7 @@ class Tablet final : public BaseTablet {
     }
 
     void set_last_base_compaction_status(std::string status) {
-        _last_base_compaction_status = status;
+        _last_base_compaction_status = std::move(status);
     }
 
     std::string get_last_base_compaction_status() { return _last_base_compaction_status; }
@@ -622,7 +616,6 @@ class Tablet final : public BaseTablet {
     std::mutex _ingest_lock;
     std::mutex _base_compaction_lock;
     std::mutex _cumulative_compaction_lock;
-    std::mutex _full_compaction_lock;
     std::mutex _schema_change_lock;
     std::shared_mutex _migration_lock;
     std::mutex _build_inverted_index_lock;
@@ -666,14 +659,6 @@ class Tablet final : public BaseTablet {
     std::shared_ptr<CumulativeCompactionPolicy> _cumulative_compaction_policy;
     std::string_view _cumulative_compaction_type;
 
-    std::shared_ptr<CumulativeCompaction> _cumulative_compaction;
-    std::shared_ptr<BaseCompaction> _base_compaction;
-    std::shared_ptr<FullCompaction> _full_compaction;
-    std::shared_ptr<SingleReplicaCompaction> _single_replica_compaction;
-
-    // whether clone task occurred during the tablet is in thread pool queue to wait for compaction
-    std::atomic<bool> _is_clone_occurred;
-
     // use a seperate thread to check all tablets paths existance
     std::atomic<bool> _is_tablet_path_exists;
 
diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp
index 068e87233547d1..e27db6188bd2be 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -702,7 +702,6 @@ Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_d
     std::lock_guard cumulative_compaction_lock(tablet->get_cumulative_compaction_lock());
     std::lock_guard cold_compaction_lock(tablet->get_cold_compaction_lock());
     std::lock_guard build_inverted_index_lock(tablet->get_build_inverted_index_lock());
-    tablet->set_clone_occurred(true);
     std::lock_guard<std::mutex> push_lock(tablet->get_push_lock());
     std::lock_guard<std::mutex> rwlock(tablet->get_rowset_update_lock());
     std::lock_guard<std::shared_mutex> wrlock(tablet->get_header_lock());
diff --git a/be/src/olap/task/engine_storage_migration_task.cpp b/be/src/olap/task/engine_storage_migration_task.cpp
index 8fba16c67df85c..9e8fc39abdef04 100644
--- a/be/src/olap/task/engine_storage_migration_task.cpp
+++ b/be/src/olap/task/engine_storage_migration_task.cpp
@@ -201,12 +201,10 @@ Status EngineStorageMigrationTask::_migrate() {
     // compaction will be prohibited for the mow table when migration. Moreover, it is useless
     // to perform a compaction operation on the migration data, as the migration still migrates
     // the data of rowsets before the compaction operation.
-    std::unique_lock full_compaction_lock(_tablet->get_full_compaction_lock(), std::defer_lock);
     std::unique_lock base_compaction_lock(_tablet->get_base_compaction_lock(), std::defer_lock);
     std::unique_lock cumu_compaction_lock(_tablet->get_cumulative_compaction_lock(),
                                           std::defer_lock);
     if (_tablet->enable_unique_key_merge_on_write()) {
-        full_compaction_lock.lock();
         base_compaction_lock.lock();
         cumu_compaction_lock.lock();
     }

From 8b225c6c3c4872c6a99a2044f2b3345e5d560af2 Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Thu, 28 Dec 2023 10:04:51 +0800
Subject: [PATCH 096/299] [pipelineX](fix) Fix core dump if cancelled (#29138)

---
 .../exec/analytic_source_operator.cpp         |  3 --
 be/src/pipeline/pipeline_x/dependency.h       | 32 +++++++------------
 .../local_exchange_source_operator.cpp        |  2 +-
 be/src/pipeline/pipeline_x/operator.cpp       |  6 ----
 .../pipeline/pipeline_x/pipeline_x_task.cpp   |  1 +
 be/src/vec/exec/vaggregation_node.cpp         |  3 --
 be/src/vec/exec/vanalytic_eval_node.cpp       |  3 --
 be/src/vec/exprs/vectorized_agg_fn.cpp        |  2 --
 be/src/vec/exprs/vectorized_agg_fn.h          |  2 --
 9 files changed, 13 insertions(+), 41 deletions(-)

diff --git a/be/src/pipeline/exec/analytic_source_operator.cpp b/be/src/pipeline/exec/analytic_source_operator.cpp
index 3c99d5bbdd1c9c..a95c2a1225ab43 100644
--- a/be/src/pipeline/exec/analytic_source_operator.cpp
+++ b/be/src/pipeline/exec/analytic_source_operator.cpp
@@ -559,9 +559,6 @@ Status AnalyticLocalState::close(RuntimeState* state) {
     if (_closed) {
         return Status::OK();
     }
-    for (auto* agg_function : _agg_functions) {
-        agg_function->close(state);
-    }
 
     static_cast<void>(_destroy_agg_status());
     _agg_arena_pool = nullptr;
diff --git a/be/src/pipeline/pipeline_x/dependency.h b/be/src/pipeline/pipeline_x/dependency.h
index 5b57f48dae8401..116a9f3e87c364 100644
--- a/be/src/pipeline/pipeline_x/dependency.h
+++ b/be/src/pipeline/pipeline_x/dependency.h
@@ -59,9 +59,6 @@ struct BasicSharedState {
     Dependency* source_dep = nullptr;
     Dependency* sink_dep = nullptr;
 
-    std::atomic<int> ref_count = 0;
-
-    void ref() { ref_count++; }
     virtual Status close(RuntimeState* state) { return Status::OK(); }
     virtual ~BasicSharedState() = default;
 };
@@ -296,24 +293,17 @@ struct AggSharedState : public BasicSharedState {
         agg_data = std::make_unique<vectorized::AggregatedDataVariants>();
         agg_arena_pool = std::make_unique<vectorized::Arena>();
     }
-    ~AggSharedState() override = default;
+    ~AggSharedState() override {
+        if (probe_expr_ctxs.empty()) {
+            _close_without_key();
+        } else {
+            _close_with_serialized_key();
+        }
+    }
     void init_spill_partition_helper(size_t spill_partition_count_bits) {
         spill_partition_helper =
                 std::make_unique<vectorized::SpillPartitionHelper>(spill_partition_count_bits);
     }
-    Status close(RuntimeState* state) override {
-        if (ref_count.fetch_sub(1) == 1) {
-            for (auto* aggregate_evaluator : aggregate_evaluators) {
-                aggregate_evaluator->close(state);
-            }
-            if (probe_expr_ctxs.empty()) {
-                _close_without_key();
-            } else {
-                _close_with_serialized_key();
-            }
-        }
-        return Status::OK();
-    }
 
     vectorized::AggregatedDataVariantsUPtr agg_data = nullptr;
     std::unique_ptr<vectorized::AggregateDataContainer> aggregate_data_container;
@@ -620,25 +610,25 @@ struct LocalExchangeSharedState : public BasicSharedState {
 public:
     ENABLE_FACTORY_CREATOR(LocalExchangeSharedState);
     std::unique_ptr<Exchanger> exchanger {};
-    std::vector<Dependency*> source_dependencies;
+    std::vector<DependencySPtr> source_dependencies;
     Dependency* sink_dependency;
     std::vector<MemTracker*> mem_trackers;
     std::atomic<size_t> mem_usage = 0;
     std::mutex le_lock;
     void sub_running_sink_operators();
     void _set_ready_for_read() {
-        for (auto* dep : source_dependencies) {
+        for (auto& dep : source_dependencies) {
             DCHECK(dep);
             dep->set_ready();
         }
     }
 
-    void set_dep_by_channel_id(Dependency* dep, int channel_id) {
+    void set_dep_by_channel_id(DependencySPtr dep, int channel_id) {
         source_dependencies[channel_id] = dep;
     }
 
     void set_ready_to_read(int channel_id) {
-        auto* dep = source_dependencies[channel_id];
+        auto& dep = source_dependencies[channel_id];
         DCHECK(dep) << channel_id;
         dep->set_ready();
     }
diff --git a/be/src/pipeline/pipeline_x/local_exchange/local_exchange_source_operator.cpp b/be/src/pipeline/pipeline_x/local_exchange/local_exchange_source_operator.cpp
index c1933f93c716fa..029dcb15a480c5 100644
--- a/be/src/pipeline/pipeline_x/local_exchange/local_exchange_source_operator.cpp
+++ b/be/src/pipeline/pipeline_x/local_exchange/local_exchange_source_operator.cpp
@@ -37,7 +37,7 @@ Status LocalExchangeSourceLocalState::init(RuntimeState* state, LocalStateInfo&
     SCOPED_TIMER(exec_time_counter());
     SCOPED_TIMER(_open_timer);
     _channel_id = info.task_idx;
-    _shared_state->set_dep_by_channel_id(_dependency, _channel_id);
+    _shared_state->set_dep_by_channel_id(info.dependency, _channel_id);
     _shared_state->mem_trackers[_channel_id] = _mem_tracker.get();
     _exchanger = _shared_state->exchanger.get();
     DCHECK(_exchanger != nullptr);
diff --git a/be/src/pipeline/pipeline_x/operator.cpp b/be/src/pipeline/pipeline_x/operator.cpp
index e763a6eee1d467..c5dbd34619f417 100644
--- a/be/src/pipeline/pipeline_x/operator.cpp
+++ b/be/src/pipeline/pipeline_x/operator.cpp
@@ -335,7 +335,6 @@ Status PipelineXLocalState<DependencyType>::init(RuntimeState* state, LocalState
             _dependency->set_shared_state(info.le_state_map[_parent->operator_id()].first);
             _shared_state =
                     (typename DependencyType::SharedState*)_dependency->shared_state().get();
-            _shared_state->ref();
 
             _shared_state->source_dep = _dependency;
             _shared_state->sink_dep = deps.front().get();
@@ -343,7 +342,6 @@ Status PipelineXLocalState<DependencyType>::init(RuntimeState* state, LocalState
             _dependency->set_shared_state(deps.front()->shared_state());
             _shared_state =
                     (typename DependencyType::SharedState*)_dependency->shared_state().get();
-            _shared_state->ref();
 
             _shared_state->source_dep = _dependency;
             _shared_state->sink_dep = deps.front().get();
@@ -419,10 +417,6 @@ Status PipelineXSinkLocalState<DependencyType>::init(RuntimeState* state,
             _wait_for_dependency_timer = ADD_TIMER_WITH_LEVEL(
                     _profile, "WaitForDependency[" + _dependency->name() + "]Time", 1);
         }
-        if constexpr (!is_fake_shared) {
-            _shared_state->ref();
-        }
-
     } else {
         auto& deps = info.dependencys;
         deps.front() = std::make_shared<FakeDependency>(0, 0, state->get_query_ctx());
diff --git a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
index 1a0c0749e8b342..4c0c0af1cd428b 100644
--- a/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
+++ b/be/src/pipeline/pipeline_x/pipeline_x_task.cpp
@@ -300,6 +300,7 @@ void PipelineXTask::finalize() {
     _finished = true;
     std::vector<DependencySPtr> {}.swap(_downstream_dependency);
     DependencyMap {}.swap(_upstream_dependency);
+    std::map<int, DependencySPtr> {}.swap(_source_dependency);
 
     _le_state_map.clear();
 }
diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp
index 69a50ba891b5bf..020763ecff75a4 100644
--- a/be/src/vec/exec/vaggregation_node.cpp
+++ b/be/src/vec/exec/vaggregation_node.cpp
@@ -534,9 +534,6 @@ Status AggregationNode::sink(doris::RuntimeState* state, vectorized::Block* in_b
 }
 
 void AggregationNode::release_resource(RuntimeState* state) {
-    for (auto* aggregate_evaluator : _aggregate_evaluators) {
-        aggregate_evaluator->close(state);
-    }
     if (_executor.close) {
         _executor.close();
     }
diff --git a/be/src/vec/exec/vanalytic_eval_node.cpp b/be/src/vec/exec/vanalytic_eval_node.cpp
index f279352dd9052b..918d86fb7533f3 100644
--- a/be/src/vec/exec/vanalytic_eval_node.cpp
+++ b/be/src/vec/exec/vanalytic_eval_node.cpp
@@ -292,9 +292,6 @@ void VAnalyticEvalNode::release_resource(RuntimeState* state) {
     if (is_closed()) {
         return;
     }
-    for (auto* agg_function : _agg_functions) {
-        agg_function->close(state);
-    }
 
     static_cast<void>(_destroy_agg_status());
     _release_mem();
diff --git a/be/src/vec/exprs/vectorized_agg_fn.cpp b/be/src/vec/exprs/vectorized_agg_fn.cpp
index 0e63838e87e733..166ad9bc2b2846 100644
--- a/be/src/vec/exprs/vectorized_agg_fn.cpp
+++ b/be/src/vec/exprs/vectorized_agg_fn.cpp
@@ -216,8 +216,6 @@ Status AggFnEvaluator::open(RuntimeState* state) {
     return VExpr::open(_input_exprs_ctxs, state);
 }
 
-void AggFnEvaluator::close(RuntimeState* state) {}
-
 void AggFnEvaluator::create(AggregateDataPtr place) {
     _function->create(place);
 }
diff --git a/be/src/vec/exprs/vectorized_agg_fn.h b/be/src/vec/exprs/vectorized_agg_fn.h
index 3cabd275614c1e..546b939ddf4299 100644
--- a/be/src/vec/exprs/vectorized_agg_fn.h
+++ b/be/src/vec/exprs/vectorized_agg_fn.h
@@ -63,8 +63,6 @@ class AggFnEvaluator {
 
     Status open(RuntimeState* state);
 
-    void close(RuntimeState* state);
-
     // create/destroy AGG Data
     void create(AggregateDataPtr place);
     void destroy(AggregateDataPtr place);

From 1284975b9bc24ef0515f4e7af80169c212851348 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Thu, 28 Dec 2023 10:24:39 +0800
Subject: [PATCH 097/299] [Improve](Job) Create task adds concurrency control
 (#29144)

---
 .../apache/doris/job/base/AbstractJob.java    | 28 +++++++++++++------
 .../job/extensions/insert/InsertJob.java      |  2 +-
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
index bc8f921d841dbe..9d7b817f73222c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/base/AbstractJob.java
@@ -50,6 +50,8 @@
 import java.util.Comparator;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
 
 @Data
@@ -105,7 +107,7 @@ public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
                        UserIdentity createUser,
                        JobExecutionConfiguration jobConfig) {
         this(jobId, jobName, jobStatus, currentDbName, comment,
-                createUser, jobConfig, System.currentTimeMillis(), null, null);
+                createUser, jobConfig, System.currentTimeMillis(), null);
     }
 
     public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
@@ -114,8 +116,7 @@ public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
                        UserIdentity createUser,
                        JobExecutionConfiguration jobConfig,
                        Long createTimeMs,
-                       String executeSql,
-                       List<T> runningTasks) {
+                       String executeSql) {
         this.jobId = jobId;
         this.jobName = jobName;
         this.jobStatus = jobStatus;
@@ -125,11 +126,12 @@ public AbstractJob(Long jobId, String jobName, JobStatus jobStatus,
         this.jobConfig = jobConfig;
         this.createTimeMs = createTimeMs;
         this.executeSql = executeSql;
-        this.runningTasks = runningTasks;
     }
 
     private List<T> runningTasks = new ArrayList<>();
 
+    private Lock createTaskLock = new ReentrantLock();
+
     @Override
     public void cancelAllTasks() throws JobException {
         if (CollectionUtils.isEmpty(runningTasks)) {
@@ -201,13 +203,22 @@ public List<T> commonCreateTasks(TaskType taskType, C taskContext) {
             log.info("job is not ready for scheduling, job id is {}", jobId);
             return new ArrayList<>();
         }
-        return createTasks(taskType, taskContext);
+        try {
+            //it's better to use tryLock and add timeout limit
+            createTaskLock.lock();
+            if (!isReadyForScheduling(taskContext)) {
+                log.info("job is not ready for scheduling, job id is {}", jobId);
+                return new ArrayList<>();
+            }
+            List<T> tasks = createTasks(taskType, taskContext);
+            tasks.forEach(task -> log.info("common create task, job id is {}, task id is {}", jobId, task.getTaskId()));
+            return tasks;
+        } finally {
+            createTaskLock.unlock();
+        }
     }
 
     public void initTasks(Collection<? extends T> tasks, TaskType taskType) {
-        if (CollectionUtils.isEmpty(getRunningTasks())) {
-            runningTasks = new ArrayList<>();
-        }
         tasks.forEach(task -> {
             task.setTaskType(taskType);
             task.setJobId(getJobId());
@@ -260,6 +271,7 @@ public static AbstractJob readFields(DataInput in) throws IOException {
         String jsonJob = Text.readString(in);
         AbstractJob job = GsonUtils.GSON.fromJson(jsonJob, AbstractJob.class);
         job.runningTasks = new ArrayList<>();
+        job.createTaskLock = new ReentrantLock();
         return job;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
index 90fc126b723c27..44cc63ee9992d8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/InsertJob.java
@@ -217,7 +217,7 @@ public InsertJob(String jobName,
                      Long createTimeMs,
                      String executeSql) {
         super(getNextJobId(), jobName, jobStatus, dbName, comment, createUser,
-                jobConfig, createTimeMs, executeSql, null);
+                jobConfig, createTimeMs, executeSql);
         this.dbId = ConnectContext.get().getCurrentDbId();
     }
 

From e5b6826de6bd89cc1e909d4d6b74078632b25df8 Mon Sep 17 00:00:00 2001
From: zhannngchen <48427519+zhannngchen@users.noreply.github.com>
Date: Thu, 28 Dec 2023 10:33:03 +0800
Subject: [PATCH 098/299] [fix](partial update) update error code when failed
 to fill the missing fields (#29103)

1. InternalError is not clear for such error, use InvalidSchema Error instead
2. avoid some useless stacktrace on InternalError when load failed
---
 be/src/olap/memtable_flush_executor.cpp       |  3 ++-
 .../olap/rowset/segment_v2/segment_writer.cpp | 15 +++++++++++---
 .../segment_v2/vertical_segment_writer.cpp    | 15 +++++++++++---
 be/src/vec/sink/writer/vtablet_writer.cpp     | 20 ++++++++++---------
 .../insert_into_table/partial_update.groovy   |  2 +-
 ...t_partial_update_native_insert_stmt.groovy |  2 +-
 6 files changed, 39 insertions(+), 18 deletions(-)

diff --git a/be/src/olap/memtable_flush_executor.cpp b/be/src/olap/memtable_flush_executor.cpp
index e02048b4c88e8c..ff2d16bd1f508f 100644
--- a/be/src/olap/memtable_flush_executor.cpp
+++ b/be/src/olap/memtable_flush_executor.cpp
@@ -161,7 +161,8 @@ void FlushToken::_flush_memtable(MemTable* mem_table, int32_t segment_id,
     }
     if (!s.ok()) {
         std::lock_guard wrlk(_flush_status_lock);
-        LOG(WARNING) << "Flush memtable failed with res = " << s;
+        LOG(WARNING) << "Flush memtable failed with res = " << s
+                     << ", load_id: " << print_id(_rowset_writer->load_id());
         _flush_status = s;
         return;
     }
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index a36b83f23b9f1d..e2fd6d5f503ee4 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -471,9 +471,18 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block*
             }
 
             if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) {
-                return Status::InternalError(
-                        "the unmentioned columns should have default value or be nullable for "
-                        "newly inserted rows in non-strict mode partial update");
+                std::string error_column;
+                for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) {
+                    const TabletColumn& col = _tablet_schema->column(cid);
+                    if (!col.has_default_value() && !col.is_nullable()) {
+                        error_column = col.name();
+                        break;
+                    }
+                }
+                return Status::Error<INVALID_SCHEMA, false>(
+                        "the unmentioned column `{}` should have default value or be nullable for "
+                        "newly inserted rows in non-strict mode partial update",
+                        error_column);
             }
             has_default_or_nullable = true;
             use_default_or_null_flag.emplace_back(true);
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index bc45092442eded..fe3204e242e436 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -405,9 +405,18 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da
             }
 
             if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) {
-                return Status::InternalError(
-                        "the unmentioned columns should have default value or be nullable for "
-                        "newly inserted rows in non-strict mode partial update");
+                std::string error_column;
+                for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) {
+                    const TabletColumn& col = _tablet_schema->column(cid);
+                    if (!col.has_default_value() && !col.is_nullable()) {
+                        error_column = col.name();
+                        break;
+                    }
+                }
+                return Status::Error<INVALID_SCHEMA, false>(
+                        "the unmentioned column `{}` should have default value or be nullable for "
+                        "newly inserted rows in non-strict mode partial update",
+                        error_column);
             }
             has_default_or_nullable = true;
             use_default_or_null_flag.emplace_back(true);
diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index a6615c2592dbe7..3093dbc5f6b632 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -176,16 +176,16 @@ void IndexChannel::mark_as_failed(const VNodeChannel* node_channel, const std::s
                 _failed_channels_msgs.emplace(the_tablet_id,
                                               err + ", host: " + node_channel->host());
                 if (_failed_channels[the_tablet_id].size() >= ((_parent->_num_replicas + 1) / 2)) {
-                    _intolerable_failure_status =
-                            Status::InternalError(_failed_channels_msgs[the_tablet_id]);
+                    _intolerable_failure_status = Status::Error<ErrorCode::INTERNAL_ERROR, false>(
+                            _failed_channels_msgs[the_tablet_id]);
                 }
             }
         } else {
             _failed_channels[tablet_id].insert(node_id);
             _failed_channels_msgs.emplace(tablet_id, err + ", host: " + node_channel->host());
             if (_failed_channels[tablet_id].size() >= ((_parent->_num_replicas + 1) / 2)) {
-                _intolerable_failure_status =
-                        Status::InternalError(_failed_channels_msgs[tablet_id]);
+                _intolerable_failure_status = Status::Error<ErrorCode::INTERNAL_ERROR, false>(
+                        _failed_channels_msgs[tablet_id]);
             }
         }
     }
@@ -432,7 +432,7 @@ Status VNodeChannel::open_wait() {
             _cancelled = true;
             auto error_code = open_callback->cntl_->ErrorCode();
             auto error_text = open_callback->cntl_->ErrorText();
-            return Status::InternalError(
+            return Status::Error<ErrorCode::INTERNAL_ERROR, false>(
                     "failed to open tablet writer, error={}, error_text={}, info={}",
                     berror(error_code), error_text, channel_info());
         }
@@ -466,7 +466,8 @@ Status VNodeChannel::add_block(vectorized::Block* block, const Payload* payload,
     if (!st.ok()) {
         if (_cancelled) {
             std::lock_guard<doris::SpinLock> l(_cancel_msg_lock);
-            return Status::InternalError("add row failed. {}", _cancel_msg);
+            return Status::Error<ErrorCode::INTERNAL_ERROR, false>("add row failed. {}",
+                                                                   _cancel_msg);
         } else {
             return std::move(st.prepend("already stopped, can't add row. cancelled/eos: "));
         }
@@ -872,7 +873,8 @@ Status VNodeChannel::close_wait(RuntimeState* state) {
     if (!st.ok()) {
         if (_cancelled) {
             std::lock_guard<doris::SpinLock> l(_cancel_msg_lock);
-            return Status::InternalError("wait close failed. {}", _cancel_msg);
+            return Status::Error<ErrorCode::INTERNAL_ERROR, false>("wait close failed. {}",
+                                                                   _cancel_msg);
         } else {
             return std::move(
                     st.prepend("already stopped, skip waiting for close. cancelled/!eos: "));
@@ -902,7 +904,7 @@ Status VNodeChannel::close_wait(RuntimeState* state) {
         return Status::OK();
     }
 
-    return Status::InternalError(get_cancel_msg());
+    return Status::Error<ErrorCode::INTERNAL_ERROR, false>(get_cancel_msg());
 }
 
 void VNodeChannel::_close_check() {
@@ -1049,7 +1051,7 @@ Status VTabletWriter::open(doris::RuntimeState* state, doris::RuntimeProfile* pr
 
     // start to send batch continually. this must be called after _init
     if (bthread_start_background(&_sender_thread, nullptr, periodic_send_batch, (void*)this) != 0) {
-        return Status::Error<INTERNAL_ERROR>("bthread_start_backgroud failed");
+        return Status::Error<ErrorCode::INTERNAL_ERROR>("bthread_start_backgroud failed");
     }
     return Status::OK();
 }
diff --git a/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy b/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy
index 5fcf4e63deeb5c..da697fa20e6083 100644
--- a/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy
+++ b/regression-test/suites/nereids_p0/insert_into_table/partial_update.groovy
@@ -122,7 +122,7 @@ suite("nereids_partial_update_native_insert_stmt", "p0") {
             // but field `name` is not nullable and doesn't have default value
             test {
                 sql """insert into ${tableName3}(id,score) values(2,400),(1,200),(4,400)"""
-                exception "INTERNAL_ERROR"
+                exception "the unmentioned column `name` should have default value or be nullable"
             }
             sql "set enable_unique_key_partial_update=false;"
             sql "sync;"
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy
index 7cd9916527234c..bf128ba26b4861 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_native_insert_stmt.groovy
@@ -122,7 +122,7 @@ suite("test_partial_update_native_insert_stmt", "p0") {
             // but field `name` is not nullable and doesn't have default value
             test {
                 sql """insert into ${tableName3}(id,score) values(2,400),(1,200),(4,400)"""
-                exception "INTERNAL_ERROR"
+                exception "the unmentioned column `name` should have default value or be nullable"
             }
             sql "set enable_unique_key_partial_update=false;"
             sql "sync;"

From 8a169b9906006f538a5ba1f0131e4f79607b45df Mon Sep 17 00:00:00 2001
From: HowardQin <hao.qin@esgyn.cn>
Date: Thu, 28 Dec 2023 10:49:19 +0800
Subject: [PATCH 099/299] [case](regression) Test enable pipeline load (#28172)

Co-authored-by: qinhao <qinhao@newland.com.cn>
---
 .../java/org/apache/doris/common/Config.java  |   2 +-
 .../stream_load/test_pipeline_load.out        |   7 +
 .../stream_load/test_pipeline_load.groovy     | 171 ++++++++++++++++++
 3 files changed, 179 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/load_p0/stream_load/test_pipeline_load.out
 create mode 100644 regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 4c92449cb5736f..eae58ddaafd4ab 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1565,7 +1565,7 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true, masterOnly = true)
     public static boolean enable_quantile_state_type = true;
 
-    @ConfField
+    @ConfField(mutable = true)
     public static boolean enable_pipeline_load = false;
 
     /*---------------------- JOB CONFIG START------------------------*/
diff --git a/regression-test/data/load_p0/stream_load/test_pipeline_load.out b/regression-test/data/load_p0/stream_load/test_pipeline_load.out
new file mode 100644
index 00000000000000..f3d24fe05d763d
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_pipeline_load.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !pipeline_load_enabled --
+2500
+
+-- !pipeline_load_enabled_exception --
+0
+
diff --git a/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy b/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
new file mode 100644
index 00000000000000..ad0aa89b17daa6
--- /dev/null
+++ b/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
@@ -0,0 +1,171 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_pipeline_load", "nonConcurrent") {
+
+    // Stream load with enable_pipeline_load = true
+    def config_row = sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'enable_pipeline_load'; """
+    String old_value = config_row[0][1]
+    sql """ ADMIN SET FRONTEND CONFIG ("enable_pipeline_load" = "true"); """
+    def tableName = "pipeline_all_types"
+    try {
+        // Using table definations and data from test_stream_load.groovy
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                `k1` int(11) NULL,
+                `k2` tinyint(4) NULL,
+                `k3` smallint(6) NULL,
+                `k4` bigint(20) NULL,
+                `k5` largeint(40) NULL,
+                `k6` float NULL,
+                `k7` double NULL,
+                `k8` decimal(9, 0) NULL,
+                `k9` char(10) NULL,
+                `k10` varchar(1024) NULL,
+                `k11` text NULL,
+                `k12` date NULL,
+                `k13` datetime NULL
+            ) ENGINE=OLAP
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+            PROPERTIES (
+                "replication_allocation" = "tag.location.default: 1"
+            );
+        """
+
+        streamLoad {
+            table "${tableName}"
+            set 'column_separator', ','
+
+            file 'all_types.csv'
+            time 10000 // limit inflight 10s
+
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                    throw exception
+                }
+                log.info("Stream load result: ${result}".toString())
+                def json = parseJson(result)
+                assertEquals("success", json.Status.toLowerCase())
+                assertEquals(2500, json.NumberTotalRows)
+                assertEquals(0, json.NumberFilteredRows)
+            }
+        }
+
+        sql "sync"
+        qt_pipeline_load_enabled """ SELECT COUNT(*) FROM ${tableName} """
+
+    } finally {
+        sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""
+    }
+
+    // Stream load with enable_pipeline_load = true and fail
+    tableName = "pipeline_input_too_long"
+    try {
+        sql """ DROP TABLE IF EXISTS ${tableName} """
+        sql """
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                `c1` varchar(48) NULL,
+                `c2` varchar(48) NULL,
+                `c3` varchar(48) NULL,
+                `c4` varchar(48) NULL,
+                `c5` varchar(48) NULL,
+                `c6` varchar(48) NULL,
+                `c7` varchar(48) NULL,
+                `c8` varchar(48) NULL,
+                `c9` varchar(48) NULL,
+                `c10` varchar(48) NULL,
+                `c11` varchar(48) NULL,
+                `c12` varchar(48) NULL,
+                `c13` varchar(48) NULL,
+                `c14` varchar(48) NULL,
+                `c15` varchar(48) NULL,
+                `c16` varchar(48) NULL,
+                `c17` varchar(48) NULL,
+                `c18` varchar(48) NULL,
+                `c19` varchar(48) NULL,
+                `c20` varchar(48) NULL,
+                `c21` varchar(48) NULL,
+                `c22` varchar(48) NULL,
+                `c23` varchar(48) NULL,
+                `c24` varchar(48) NULL,
+                `c25` varchar(48) NULL,
+                `c26` varchar(48) NULL,
+                `c27` varchar(48) NULL,
+                `c28` varchar(48) NULL,
+                `c29` varchar(48) NULL,
+                `c30` varchar(48) NULL,
+                `c31` varchar(48) NULL,
+                `c32` varchar(48) NULL,
+                `c33` varchar(48) NULL,
+                `c34` varchar(48) NULL,
+                `c35` varchar(48) NULL,
+                `c36` varchar(48) NULL,
+                `c37` varchar(48) NULL,
+                `c38` varchar(48) NULL,
+                `c39` varchar(48) NULL,
+                `c40` varchar(48) NULL,
+                `c41` varchar(48) NULL,
+                `c42` varchar(48) NULL,
+                `c43` varchar(48) NULL,
+                `c44` varchar(48) NULL,
+                `c45` varchar(48) NULL,
+                `c46` varchar(48) NULL,
+                `c47` varchar(48) NULL,
+                `c48` varchar(48) NULL,
+                `c49` varchar(48) NULL,
+                `c50` varchar(48) NULL,
+            ) ENGINE=OLAP
+            DUPLICATE KEY(`c1`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY HASH(`c1`) BUCKETS AUTO
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "disable_auto_compaction" = "false"
+            );
+        """
+        // Trigger error which is different from non-pipeline load
+        streamLoad {
+            table "${tableName}"
+            set 'column_separator', ','
+            file 'test_input_long_than_schema.csv'
+
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                    throw exception
+                }
+                log.info("Stream load result: ${result}".toString())
+                def json = parseJson(result)
+                assertEquals("fail", json.Status.toLowerCase())
+                assertTrue(json.Message.contains("[END_OF_FILE]Encountered unqualified data"))
+                assertEquals(0, json.NumberTotalRows)
+                assertEquals(0, json.NumberFilteredRows)
+                assertEquals(0, json.NumberUnselectedRows)
+            }
+        }
+
+        sql "sync"
+        qt_pipeline_load_enabled_exception """SELECT COUNT(*) FROM ${tableName}"""
+
+    } finally {
+        sql """ DROP TABLE IF EXISTS ${tableName} FORCE"""
+    }
+    // restore enable_pipeline_load to old_value
+    sql """ ADMIN SET FRONTEND CONFIG ("enable_pipeline_load" = "${old_value}"); """
+
+}
+

From 3dc3e81734c830e4790485a4d88acd5a3e83aae2 Mon Sep 17 00:00:00 2001
From: yangshijie <sjyang2022@zju.edu.cn>
Date: Thu, 28 Dec 2023 11:00:38 +0800
Subject: [PATCH 100/299] [Improvement](datatype) Update Parser for IPv4/v6
 data types (#29044)

Transforming from parsing std:: string to parsing char * to accelerate the parsing of ipv4/v6 data types.
---
 be/src/util/types.h             |   5 -
 be/src/vec/common/format_ip.h   | 162 ++++++++++++++++++++++-
 be/src/vec/runtime/ipv4_value.h |  89 +++----------
 be/src/vec/runtime/ipv6_value.h | 228 +++-----------------------------
 4 files changed, 201 insertions(+), 283 deletions(-)

diff --git a/be/src/util/types.h b/be/src/util/types.h
index b04c0644f6fa7f..7688dd60390855 100644
--- a/be/src/util/types.h
+++ b/be/src/util/types.h
@@ -44,9 +44,4 @@ inline int128_t get_int128_from_unalign(const void* address) {
     return value;
 }
 
-inline uint128_t get_uint128_from_unalign(const void* address) {
-    uint128_t value = 0;
-    memcpy(&value, address, sizeof(uint128_t));
-    return value;
-}
 } // namespace doris
diff --git a/be/src/vec/common/format_ip.h b/be/src/vec/common/format_ip.h
index ab9ac4b6595032..58322ecd9fc5f7 100644
--- a/be/src/vec/common/format_ip.h
+++ b/be/src/vec/common/format_ip.h
@@ -20,6 +20,7 @@
 
 #pragma once
 
+#include <vec/common/hex.h>
 #include <vec/common/string_utils/string_utils.h>
 
 #include <algorithm>
@@ -34,7 +35,7 @@ constexpr size_t IPV4_MAX_TEXT_LENGTH = 15; /// Does not count tail zero byte.
 constexpr size_t IPV6_MAX_TEXT_LENGTH = 39;
 constexpr size_t IPV4_MIN_NUM_VALUE = 0;          //num value of '0.0.0.0'
 constexpr size_t IPV4_MAX_NUM_VALUE = 4294967295; //num value of '255.255.255.255'
-constexpr int IPV4_MAX_OCTET_VALUE = 255;         //max vulue of octet
+constexpr int IPV4_MAX_OCTET_VALUE = 255;         //max value of octet
 constexpr size_t IPV4_OCTET_BITS = 8;
 constexpr size_t DECIMAL_BASE = 10;
 constexpr size_t IPV6_BINARY_LENGTH = 16;
@@ -198,4 +199,163 @@ inline bool parseIPv4whole(const char* src, unsigned char* dst) {
   */
 void formatIPv6(const unsigned char* src, char*& dst, uint8_t zeroed_tail_bytes_count = 0);
 
+/** Unsafe (no bounds-checking for src nor dst), optimized version of parsing IPv6 string.
+*
+* Parses the input string `src` and stores binary big-endian value into buffer pointed by `dst`,
+* which should be long enough. In case of failure zeroes IPV6_BINARY_LENGTH bytes of buffer pointed by `dst`.
+*
+* WARNING - this function is adapted to work with ReadBuffer, where src is the position reference (ReadBuffer::position())
+*           and eof is the ReadBuffer::eof() - therefore algorithm below does not rely on buffer's continuity.
+*           To parse strings use overloads below.
+*
+* @param src         - iterator (reference to pointer) over input string - warning - continuity is not guaranteed.
+* @param eof         - function returning true if iterator riched the end - warning - can break iterator's continuity.
+* @param dst         - where to put output bytes, expected to be non-null and at IPV6_BINARY_LENGTH-long.
+* @param first_block - preparsed first block
+* @return            - true if parsed successfully, false otherwise.
+*/
+template <typename T, typename EOFfunction>
+    requires(std::is_same<typename std::remove_cv<T>::type, char>::value)
+inline bool parseIPv6(T*& src, EOFfunction eof, unsigned char* dst, int32_t first_block = -1) {
+    const auto clear_dst = [dst]() {
+        std::memset(dst, '\0', IPV6_BINARY_LENGTH);
+        return false;
+    };
+
+    if (src == nullptr || eof()) return clear_dst();
+
+    int groups = 0;            /// number of parsed groups
+    unsigned char* iter = dst; /// iterator over dst buffer
+    unsigned char* zptr =
+            nullptr; /// pointer into dst buffer array where all-zeroes block ("::") is started
+
+    std::memset(dst, '\0', IPV6_BINARY_LENGTH);
+
+    if (first_block >= 0) {
+        *iter++ = static_cast<unsigned char>((first_block >> 8) & 0xffu);
+        *iter++ = static_cast<unsigned char>(first_block & 0xffu);
+        if (*src == ':') {
+            zptr = iter;
+            ++src;
+        }
+        ++groups;
+    }
+
+    bool group_start = true;
+
+    while (!eof() && groups < 8) {
+        if (*src == ':') {
+            ++src;
+            if (eof()) /// trailing colon is not allowed
+                return clear_dst();
+
+            group_start = true;
+
+            if (*src == ':') {
+                if (zptr != nullptr) /// multiple all-zeroes blocks are not allowed
+                    return clear_dst();
+                zptr = iter;
+                ++src;
+                continue;
+            }
+            if (groups == 0) /// leading colon is not allowed
+                return clear_dst();
+        }
+
+        if (*src == '.') /// mixed IPv4 parsing
+        {
+            if (groups <= 1 && zptr == nullptr) /// IPv4 block can't be the first
+                return clear_dst();
+
+            if (group_start) /// first octet of IPv4 should be already parsed as an IPv6 group
+                return clear_dst();
+
+            ++src;
+            if (eof()) return clear_dst();
+
+            /// last parsed group should be reinterpreted as a decimal value - it's the first octet of IPv4
+            --groups;
+            iter -= 2;
+
+            UInt16 num = 0;
+            for (int i = 0; i < 2; ++i) {
+                unsigned char first = (iter[i] >> 4) & 0x0fu;
+                unsigned char second = iter[i] & 0x0fu;
+                if (first > 9 || second > 9) return clear_dst();
+                (num *= 100) += first * 10 + second;
+            }
+            if (num > 255) return clear_dst();
+
+            /// parse IPv4 with known first octet
+            if (!parseIPv4(src, eof, iter, num)) return clear_dst();
+
+            if constexpr (std::endian::native == std::endian::little)
+                std::reverse(iter, iter + IPV4_BINARY_LENGTH);
+
+            iter += 4;
+            groups += 2;
+            break; /// IPv4 block is the last - end of parsing
+        }
+
+        if (!group_start) /// end of parsing
+            break;
+        group_start = false;
+
+        UInt16 val = 0;  /// current decoded group
+        int xdigits = 0; /// number of decoded hex digits in current group
+
+        for (; !eof() && xdigits < 4; ++src, ++xdigits) {
+            UInt8 num = unhex(*src);
+            if (num == 0xFF) break;
+            (val <<= 4) |= num;
+        }
+
+        if (xdigits == 0) /// end of parsing
+            break;
+
+        *iter++ = static_cast<unsigned char>((val >> 8) & 0xffu);
+        *iter++ = static_cast<unsigned char>(val & 0xffu);
+        ++groups;
+    }
+
+    /// either all 8 groups or all-zeroes block should be present
+    if (groups < 8 && zptr == nullptr) return clear_dst();
+
+    if (zptr != nullptr) /// process all-zeroes block
+    {
+        size_t msize = iter - zptr;
+        std::memmove(dst + IPV6_BINARY_LENGTH - msize, zptr, msize);
+        std::memset(zptr, '\0', IPV6_BINARY_LENGTH - (iter - dst));
+    }
+
+    return true;
+}
+
+/// returns pointer to the right after parsed sequence or null on failed parsing
+inline const char* parseIPv6(const char* src, const char* end, unsigned char* dst) {
+    if (parseIPv6(
+                src, [&src, end]() { return src == end; }, dst))
+        return src;
+    return nullptr;
+}
+
+/// returns true if whole buffer was parsed successfully
+inline bool parseIPv6whole(const char* src, const char* end, unsigned char* dst) {
+    return parseIPv6(src, end, dst) == end;
+}
+
+/// returns pointer to the right after parsed sequence or null on failed parsing
+inline const char* parseIPv6(const char* src, unsigned char* dst) {
+    if (parseIPv6(
+                src, []() { return false; }, dst))
+        return src;
+    return nullptr;
+}
+
+/// returns true if whole null-terminated string was parsed successfully
+inline bool parseIPv6whole(const char* src, unsigned char* dst) {
+    const char* end = parseIPv6(src, dst);
+    return end != nullptr && *end == '\0';
+}
+
 } // namespace doris::vectorized
diff --git a/be/src/vec/runtime/ipv4_value.h b/be/src/vec/runtime/ipv4_value.h
index fb42ed66f9ca5c..9304aa2c18a846 100644
--- a/be/src/vec/runtime/ipv4_value.h
+++ b/be/src/vec/runtime/ipv4_value.h
@@ -17,14 +17,13 @@
 
 #pragma once
 
-#include <stdint.h>
-
 #include <algorithm>
 #include <regex>
 #include <sstream>
 #include <string>
 
 #include "util/string_parser.hpp"
+#include "vec/common/format_ip.h"
 
 namespace doris {
 
@@ -34,87 +33,41 @@ class IPv4Value {
 
     explicit IPv4Value(vectorized::IPv4 ipv4) { _value = ipv4; }
 
-    explicit IPv4Value(std::string ipv4) {}
-
-    [[nodiscard]] const vectorized::IPv4& value() const { return _value; }
+    const vectorized::IPv4& value() const { return _value; }
 
     vectorized::IPv4& value() { return _value; }
 
     void set_value(vectorized::IPv4 ipv4) { _value = ipv4; }
 
-    bool from_string(std::string ipv4) { return from_string(_value, ipv4); }
-
-    [[nodiscard]] std::string to_string() const { return to_string(_value); }
+    bool from_string(const std::string& ipv4_str) { return from_string(_value, ipv4_str); }
 
-    static bool from_string(vectorized::IPv4& value, std::string ipv4) {
-        remove_ipv4_space(ipv4);
+    std::string to_string() const { return to_string(_value); }
 
-        // shortest ipv4 string is `0.0.0.0` whose length is 7
-        if (ipv4.size() < 7 || !is_valid_string(ipv4)) {
+    static bool from_string(vectorized::IPv4& value, const std::string& ipv4_str) {
+        if (ipv4_str.empty()) {
             return false;
         }
-
-        vectorized::IPv4 octets[4] = {0};
-        std::istringstream iss(ipv4);
-        std::string octet;
-        uint8_t octet_index = 0;
-
-        while (getline(iss, octet, '.')) {
-            if (octet_index >= 4) {
-                return false;
-            }
-
-            StringParser::ParseResult result;
-            vectorized::IPv4 val = StringParser::string_to_unsigned_int<vectorized::IPv4>(
-                    octet.c_str(), octet.length(), &result);
-            if (result != StringParser::PARSE_SUCCESS || val > 255) {
-                return false;
-            }
-
-            octets[octet_index++] = val;
-        }
-
-        if (octet_index != 4) {
+        int64_t parse_value;
+        const char* src = ipv4_str.c_str();
+        const char* end = ipv4_str.c_str() + ipv4_str.size() - 1;
+        while (std::isspace(*src)) ++src;
+        while (std::isspace(*end)) --end;
+        if (!vectorized::parseIPv4whole(src, ++end,
+                                        reinterpret_cast<unsigned char*>(&parse_value))) {
             return false;
         }
-
-        value = (octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3];
+        value = static_cast<vectorized::IPv4>(parse_value);
         return true;
     }
 
     static std::string to_string(vectorized::IPv4 value) {
-        std::stringstream ss;
-        ss << ((value >> 24) & 0xFF) << '.' << ((value >> 16) & 0xFF) << '.'
-           << ((value >> 8) & 0xFF) << '.' << (value & 0xFF);
-        return ss.str();
-    }
-
-    static void remove_ipv4_space(std::string& ipv4) {
-        if (ipv4.empty()) {
-            return;
-        }
-
-        std::string special_chars = "\r\n\t ";
-
-        size_t pos = ipv4.find_first_not_of(special_chars);
-        if (pos != std::string::npos) {
-            ipv4.erase(0, pos);
-        }
-
-        pos = ipv4.find_last_not_of(special_chars);
-        if (pos != std::string::npos) {
-            ipv4.erase(pos + 1);
-        }
-    }
-
-    static bool is_valid_string(std::string ipv4) {
-        static std::regex IPV4_STD_REGEX(
-                "^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-"
-                "9]?)$");
-        if (ipv4.size() > 15 || !std::regex_match(ipv4, IPV4_STD_REGEX)) {
-            return false;
-        }
-        return true;
+        char buf[IPV4_MAX_TEXT_LENGTH + 1];
+        char* start = buf;
+        char* end = buf;
+        const auto* src = reinterpret_cast<const unsigned char*>(&value);
+        vectorized::formatIPv4(src, end);
+        size_t len = end - start;
+        return {buf, len};
     }
 
 private:
diff --git a/be/src/vec/runtime/ipv6_value.h b/be/src/vec/runtime/ipv6_value.h
index 8839ab4e2b06d2..8aaa8a26b699c7 100644
--- a/be/src/vec/runtime/ipv6_value.h
+++ b/be/src/vec/runtime/ipv6_value.h
@@ -17,12 +17,11 @@
 
 #pragma once
 
-#include <stdint.h>
-
 #include <regex>
 #include <sstream>
 #include <string>
 
+#include "vec/common/format_ip.h"
 #include "vec/core/types.h"
 #include "vec/data_types/data_type.h"
 #include "vec/data_types/data_type_number_base.h"
@@ -35,224 +34,35 @@ class IPv6Value {
 
     explicit IPv6Value(vectorized::IPv6 ipv6) { _value = ipv6; }
 
-    [[nodiscard]] const vectorized::IPv6& value() const { return _value; }
+    const vectorized::IPv6& value() const { return _value; }
 
     vectorized::IPv6& value() { return _value; }
 
     void set_value(vectorized::IPv6 ipv6) { _value = ipv6; }
 
-    bool from_string(std::string ipv6) { return from_string(_value, ipv6); }
-
-    bool from_binary_string(std::string ipv6_binary) {
-        return from_binary_string(_value, ipv6_binary);
-    }
-
-    static bool from_string(vectorized::IPv6& x, std::string ipv6) {
-        remove_ipv6_space(ipv6);
-
-        if (ipv6.empty() || !is_valid_string(ipv6)) {
-            return false;
-        }
-
-        std::transform(ipv6.begin(), ipv6.end(), ipv6.begin(),
-                       [](unsigned char ch) { return std::tolower(ch); });
-        std::istringstream iss(ipv6);
-        std::string field;
-        uint16_t fields[8] = {0};
-        uint8_t zero_index = 0;
-        uint8_t num_field = 0;
-        uint8_t right_field_num = 0;
-
-        while (num_field < 8) {
-            if (!getline(iss, field, ':')) {
-                break;
-            }
-
-            if (field.empty()) {
-                zero_index = num_field;
-                fields[num_field++] = 0;
-            } else {
-                try {
-                    if (field.size() > 4 || field > "ffff") {
-                        return false;
-                    }
-
-                    fields[num_field++] = std::stoi(field, nullptr, 16);
-                } catch (const std::exception& /*e*/) {
-                    return false;
-                }
-            }
-        }
-
-        if (zero_index != 0) {
-            right_field_num = num_field - zero_index - 1;
-
-            for (uint8_t i = 7; i > 7 - right_field_num; --i) {
-                fields[i] = fields[zero_index + right_field_num + i - 7];
-                fields[zero_index + right_field_num + i - 7] = 0;
-            }
-        }
-
-        uint64_t high = (static_cast<uint64_t>(fields[0]) << 48) |
-                        (static_cast<uint64_t>(fields[1]) << 32) |
-                        (static_cast<uint64_t>(fields[2]) << 16) | static_cast<uint64_t>(fields[3]);
-        uint64_t low = (static_cast<uint64_t>(fields[4]) << 48) |
-                       (static_cast<uint64_t>(fields[5]) << 32) |
-                       (static_cast<uint64_t>(fields[6]) << 16) | static_cast<uint64_t>(fields[7]);
-
-        x = static_cast<vectorized::IPv6>(high) << 64 | low;
-        return true;
-    }
+    bool from_string(const std::string& ipv6_str) { return from_string(_value, ipv6_str); }
 
-    static bool from_binary_string(vectorized::IPv6& x, std::string ipv6_binary_str) {
-        // Accepts a FixedString(16) value containing the IPv6 address in binary format
-        if (ipv6_binary_str.size() != 16) {
+    static bool from_string(vectorized::IPv6& value, const std::string& ipv6_str) {
+        if (ipv6_str.empty()) {
             return false;
         }
-
-        uint64_t high = 0;
-        uint64_t low = 0;
-
-        const uint8_t* ipv6_binary = reinterpret_cast<const uint8_t*>(ipv6_binary_str.c_str());
-
-        for (int i = 0; i < 8; ++i) {
-            high |= (static_cast<uint64_t>(ipv6_binary[i]) << (56 - i * 8));
-        }
-
-        for (int i = 8; i < 16; ++i) {
-            low |= (static_cast<uint64_t>(ipv6_binary[i]) << (56 - (i - 8) * 8));
-        }
-
-        x = static_cast<vectorized::IPv6>(high) << 64 | low;
-        return true;
-    }
-
-    [[nodiscard]] std::string to_string() const { return to_string(_value); }
-
-    static std::string to_string(vectorized::IPv6 x) {
-        // "0000:0000:0000:0000:0000:0000:0000:0000"
-        if (x == 0) {
-            return "::";
-        }
-
-        uint64_t low = static_cast<uint64_t>(x);
-        uint64_t high = static_cast<uint64_t>(x >> 64);
-
-        uint16_t fields[8] = {static_cast<uint16_t>((high >> 48) & 0xFFFF),
-                              static_cast<uint16_t>((high >> 32) & 0xFFFF),
-                              static_cast<uint16_t>((high >> 16) & 0xFFFF),
-                              static_cast<uint16_t>(high & 0xFFFF),
-                              static_cast<uint16_t>((low >> 48) & 0xFFFF),
-                              static_cast<uint16_t>((low >> 32) & 0xFFFF),
-                              static_cast<uint16_t>((low >> 16) & 0xFFFF),
-                              static_cast<uint16_t>(low & 0xFFFF)};
-
-        uint8_t zero_start = 0, zero_end = 0;
-
-        while (zero_start < 8 && zero_end < 8) {
-            if (fields[zero_start] != 0) {
-                zero_start++;
-                zero_end = zero_start;
-                continue;
-            }
-
-            while (zero_end < 7 && fields[zero_end + 1] == 0) {
-                zero_end++;
-            }
-
-            if (zero_end > zero_start) {
-                break;
-            }
-
-            zero_start++;
-            zero_end = zero_start;
-        }
-
-        std::stringstream ss;
-
-        if (zero_start == zero_end) {
-            for (uint8_t i = 0; i < 7; ++i) {
-                ss << std::hex << fields[i] << ":";
-            }
-            ss << std::hex << fields[7];
-        } else {
-            for (uint8_t i = 0; i < zero_start; ++i) {
-                ss << std::hex << fields[i] << ":";
-            }
-
-            if (zero_end == 7) {
-                ss << ":";
-            } else {
-                for (uint8_t j = zero_end + 1; j < 8; ++j) {
-                    ss << std::hex << ":" << fields[j];
-                }
-            }
-        }
-
-        return ss.str();
-    }
-
-    [[nodiscard]] std::string to_binary_string() const { return to_binary_string(_value); }
-
-    static std::string to_binary_string(vectorized::IPv6 x) {
-        uint64_t low = static_cast<uint64_t>(x);
-        uint64_t high = static_cast<uint64_t>(x >> 64);
-
-        uint8_t fields[16] = {static_cast<uint8_t>((high >> 56) & 0xFF),
-                              static_cast<uint8_t>((high >> 48) & 0xFF),
-                              static_cast<uint8_t>((high >> 40) & 0xFF),
-                              static_cast<uint8_t>((high >> 32) & 0xFF),
-                              static_cast<uint8_t>((high >> 24) & 0xFF),
-                              static_cast<uint8_t>((high >> 16) & 0xFF),
-                              static_cast<uint8_t>((high >> 8) & 0xFF),
-                              static_cast<uint8_t>(high & 0xFF),
-                              static_cast<uint8_t>((low >> 56) & 0xFF),
-                              static_cast<uint8_t>((low >> 48) & 0xFF),
-                              static_cast<uint8_t>((low >> 40) & 0xFF),
-                              static_cast<uint8_t>((low >> 32) & 0xFF),
-                              static_cast<uint8_t>((low >> 24) & 0xFF),
-                              static_cast<uint8_t>((low >> 16) & 0xFF),
-                              static_cast<uint8_t>((low >> 8) & 0xFF),
-                              static_cast<uint8_t>(low & 0xFF)};
-
-        std::stringstream ss;
-
-        for (int i = 0; i < 16; ++i) {
-            ss << (char)fields[i];
-        }
-
-        return ss.str();
+        const char* src = ipv6_str.c_str();
+        const char* end = ipv6_str.c_str() + ipv6_str.size() - 1;
+        while (std::isspace(*src)) ++src;
+        while (std::isspace(*end)) --end;
+        return vectorized::parseIPv6whole(src, ++end, reinterpret_cast<unsigned char*>(&value));
     }
 
-    static void remove_ipv6_space(std::string& ipv6) {
-        if (ipv6.empty()) {
-            return;
-        }
-
-        std::string special_chars = "\r\n\t ";
+    std::string to_string() const { return to_string(_value); }
 
-        size_t pos = ipv6.find_first_not_of(special_chars);
-        if (pos != std::string::npos) {
-            ipv6.erase(0, pos);
-        }
-
-        pos = ipv6.find_last_not_of(special_chars);
-        if (pos != std::string::npos) {
-            ipv6.erase(pos + 1);
-        }
-    }
-
-    static bool is_valid_string(std::string ipv6) {
-        static std::regex IPV6_STD_REGEX("^([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}$");
-        static std::regex IPV6_COMPRESS_REGEX(
-                "^(([0-9A-Fa-f]{1,4}(:[0-9A-Fa-f]{1,4})*)?)::((([0-9A-Fa-f]{1,4}:)*[0-9A-Fa-f]{1,4}"
-                ")?)$");
-
-        if (ipv6.size() > 39 || !(std::regex_match(ipv6, IPV6_STD_REGEX) ||
-                                  std::regex_match(ipv6, IPV6_COMPRESS_REGEX))) {
-            return false;
-        }
-        return true;
+    static std::string to_string(vectorized::IPv6 value) {
+        char buf[IPV6_MAX_TEXT_LENGTH + 1];
+        char* start = buf;
+        char* end = buf;
+        const auto* src = reinterpret_cast<const unsigned char*>(&value);
+        vectorized::formatIPv6(src, end);
+        size_t len = end - start;
+        return {buf, len};
     }
 
 private:

From 0562999f917e95f6847c81d350232b96a5ddb496 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Thu, 28 Dec 2023 10:12:35 +0700
Subject: [PATCH 101/299] [fix](doc) spell errors fixes and align with code log
 for memory tracker. (#28000)

Spell corrected for LastestSuccessChannelCache and aligned that with the docs
---
 be/src/runtime/load_channel_mgr.cpp                           | 2 +-
 .../maint-monitor/memory-management/be-oom-analysis.md        | 4 ++--
 .../memory-management/memory-limit-exceeded-analysis.md       | 2 +-
 .../maint-monitor/memory-management/memory-tracker.md         | 2 +-
 .../maint-monitor/memory-management/be-oom-analysis.md        | 4 ++--
 .../memory-management/memory-limit-exceeded-analysis.md       | 2 +-
 .../maint-monitor/memory-management/memory-tracker.md         | 2 +-
 .../admin-manual/maint-monitor/monitor-metrics/metrics.md     | 2 +-
 8 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/be/src/runtime/load_channel_mgr.cpp b/be/src/runtime/load_channel_mgr.cpp
index 82001b74119de1..2eb6812ac52a59 100644
--- a/be/src/runtime/load_channel_mgr.cpp
+++ b/be/src/runtime/load_channel_mgr.cpp
@@ -83,7 +83,7 @@ void LoadChannelMgr::stop() {
 
 Status LoadChannelMgr::init(int64_t process_mem_limit) {
     _last_success_channel =
-            std::unique_ptr<Cache>(new ShardedLRUCache("LastestSuccessChannelCache", 1024));
+            std::unique_ptr<Cache>(new ShardedLRUCache("LastSuccessChannelCache", 1024));
     RETURN_IF_ERROR(_start_bg_worker());
     return Status::OK();
 }
diff --git a/docs/en/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md b/docs/en/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
index 1994cefe489e05..165b60259f91b0 100644
--- a/docs/en/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
+++ b/docs/en/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
@@ -63,7 +63,7 @@ Memory Tracker Summary:
     MemTrackerLimiter Label=SegmentCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DiskIO, Type=global, Limit=2.47 GB(2655423201 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=ChunkAllocator, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
-    MemTrackerLimiter Label=LastestSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
+    MemTrackerLimiter Label=LastSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DeleteBitmap AggCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
 ```
 
@@ -75,7 +75,7 @@ Memory Tracker Summary:
 
 6. `type=load` imports a lot of memory.
 
-7. When the `type=global` memory is used for a long time, continue to check the `type=global` detailed statistics in the second half of the `Memory Tracker Summary` log. When DataPageCache, IndexPageCache, SegmentCache, ChunkAllocator, LastestSuccessChannelCache, etc. use a lot of memory, refer to [BE Configuration Item](../../../admin-manual/config/be-config.md) to consider modifying the size of the cache; when Orphan memory usage is too large, Continue the analysis as follows.
+7. When the `type=global` memory is used for a long time, continue to check the `type=global` detailed statistics in the second half of the `Memory Tracker Summary` log. When DataPageCache, IndexPageCache, SegmentCache, ChunkAllocator, LastSuccessChannelCache, etc. use a lot of memory, refer to [BE Configuration Item](../../../admin-manual/config/be-config.md) to consider modifying the size of the cache; when Orphan memory usage is too large, Continue the analysis as follows.
   - If the sum of the tracker statistics of `Parent Label=Orphan` only accounts for a small part of the Orphan memory, it means that there is currently a large amount of memory that has no accurate statistics, such as the memory of the brpc process. At this time, you can consider using the heap profile [Memory Tracker]( https://doris.apache.org/community/developer-guide/debug-tool) to further analyze memory locations.
   - If the tracker statistics of `Parent Label=Orphan` account for most of Orphan’s memory, when `Label=TabletManager` uses a lot of memory, further check the number of tablets in the cluster. If there are too many tablets, delete them and they will not be used table or data; when `Label=StorageEngine` uses too much memory, further check the number of segment files in the cluster, and consider manually triggering compaction if the number of segment files is too large;
 
diff --git a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
index b1611cb593e52f..2a4118a2361267 100644
--- a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
+++ b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
@@ -100,7 +100,7 @@ Memory Tracker Summary:
     MemTrackerLimiter Label=SegmentCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DiskIO, Type=global, Limit=2.47 GB(2655423201 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=ChunkAllocator, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
-    MemTrackerLimiter Label=LastestSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
+    MemTrackerLimiter Label=LastSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DeleteBitmap AggCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
 ```
 
diff --git a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
index 46eb9335561a43..984042fbdc8c5c 100644
--- a/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
+++ b/docs/en/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
@@ -78,7 +78,7 @@ For the memory statistics results of historical queries, you can view the `peakM
 - SegmentCache: Used to cache opened Segments, such as index information.
 - DiskIO: Used to cache Disk IO data, only used in non-vectorization.
 - ChunkAllocator: Used to cache power-of-2 memory blocks, and reuse memory at the application layer.
-- LastestSuccessChannelCache: Used to cache the LoadChannel of the import receiver.
+- LastSuccessChannelCache: Used to cache the LoadChannel of the import receiver.
 - DeleteBitmap AggCache: Gets aggregated delete_bitmap on rowset_id and version.
 
 ### Query Type `/mem_tracker?type=query`
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
index 1cd64d2b6d2fb6..7ce67603b19e1c 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/be-oom-analysis.md
@@ -63,7 +63,7 @@ Memory Tracker Summary:
     MemTrackerLimiter Label=SegmentCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DiskIO, Type=global, Limit=2.47 GB(2655423201 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=ChunkAllocator, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
-    MemTrackerLimiter Label=LastestSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
+    MemTrackerLimiter Label=LastSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DeleteBitmap AggCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
 ```
 
@@ -75,7 +75,7 @@ Memory Tracker Summary:
 
 6. `type=load`导入内存使用多时。
 
-7. `type=global`内存使用多时，继续查看`Memory Tracker Summary`日志后半部分已经打出得`type=global`详细统计。当 DataPageCache、IndexPageCache、SegmentCache、ChunkAllocator、LastestSuccessChannelCache 等内存使用多时，参考 [BE 配置项](../../../admin-manual/config/be-config.md) 考虑修改cache的大小；当 Orphan 内存使用过多时，如下继续分析。
+7. `type=global`内存使用多时，继续查看`Memory Tracker Summary`日志后半部分已经打出得`type=global`详细统计。当 DataPageCache、IndexPageCache、SegmentCache、ChunkAllocator、LastSuccessChannelCache 等内存使用多时，参考 [BE 配置项](../../../admin-manual/config/be-config.md) 考虑修改cache的大小；当 Orphan 内存使用过多时，如下继续分析。
   - 若`Parent Label=Orphan`的tracker统计值相加只占 Orphan 内存的小部分，则说明当前有大量内存没有准确统计，比如 brpc 过程的内存，此时可以考虑借助 heap profile [Memory Tracker](https://doris.apache.org/zh-CN/community/developer-guide/debug-tool) 中的方法进一步分析内存位置。
   - 若`Parent Label=Orphan`的tracker统计值相加占 Orphan 内存的大部分，当`Label=TabletManager`内存使用多时，进一步查看集群 Tablet 数量，若 Tablet 数量过多则考虑删除过时不会被使用的表或数据；当`Label=StorageEngine`内存使用过多时，进一步查看集群 Segment 文件个数，若 Segment 文件个数过多则考虑手动触发compaction；
 
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
index c4bf5d0ac908ca..40c08bb8052624 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-limit-exceeded-analysis.md
@@ -100,7 +100,7 @@ Memory Tracker Summary:
     MemTrackerLimiter Label=SegmentCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DiskIO, Type=global, Limit=2.47 GB(2655423201 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=ChunkAllocator, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
-    MemTrackerLimiter Label=LastestSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
+    MemTrackerLimiter Label=LastSuccessChannelCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
     MemTrackerLimiter Label=DeleteBitmap AggCache, Type=global, Limit=-1.00 B(-1 B), Used=0(0 B), Peak=0(0 B)
 ```
 
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
index af63a5c983971c..a0fd3cd8a68141 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/memory-management/memory-tracker.md
@@ -78,7 +78,7 @@ https://shimo.im/docs/DT6JXDRkdTvdyV3G
 - SegmentCache: 用于缓存已打开的 Segment，如索引信息。
 - DiskIO: 用于缓存 Disk IO 数据，仅在非向量化使用。
 - ChunkAllocator: 用于缓存2的幂大小的内存块，在应用层内存复用。
-- LastestSuccessChannelCache: 用于缓存导入接收端的 LoadChannel。
+- LastSuccessChannelCache: 用于缓存导入接收端的 LoadChannel。
 - DeleteBitmap AggCache: Gets aggregated delete_bitmap on rowset_id and version。
 
 ### Query Type `/mem_tracker?type=query`
diff --git a/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md b/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md
index cb8de2bf1e908e..9c0b8110651220 100644
--- a/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md
+++ b/docs/zh-CN/docs/admin-manual/maint-monitor/monitor-metrics/metrics.md
@@ -195,7 +195,7 @@ curl http://be_host:webserver_port/metrics?type=json
 | `doris_be_cache_hit_ratio`                        |                                             |      | 记录指定 LRU Cache 的命中率                                                          | 用于观测cache是否有效                                                       | P0 |
 |                                                   | {name="DataPageCache"}                      | Num  | DataPageCache 用于缓存数据的 Data Page                                              | 数据Cache，直接影响查询效率                                                    | P0 |
 |                                                   | {name="IndexPageCache"}                     | Num  | IndexPageCache 用于缓存数据的 Index Page                                            | 索引Cache，直接影响查询效率                                                    | P0 |
-|                                                   | {name="LastestSuccessChannelCache"}         | Num  | LastestSuccessChannelCache 用于缓存导入接收端的 LoadChannel                            |                                                                     |
+|                                                   | {name="LastSuccessChannelCache"}         | Num  | LastSuccessChannelCache 用于缓存导入接收端的 LoadChannel                            |                                                                     |
 |                                                   | {name="SegmentCache"}                       | Num  | SegmentCache 用于缓存已打开的 Segment，如索引信息                                          |                                                                     |
 | `doris_be_chunk_pool_local_core_alloc_count`      |                                             | Num  | ChunkAllocator中，从绑定的 core 的内存队列中分配内存的次数                                      |                                                                     |
 | `doris_be_chunk_pool_other_core_alloc_count`      |                                             | Num  | ChunkAllocator中，从其他的 core 的内存队列中分配内存的次数                                      |                                                                     |

From 82a8232c8af00ebfdb9f111806d7a1925518a5cb Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi02@gmail.com>
Date: Thu, 28 Dec 2023 11:39:54 +0800
Subject: [PATCH 102/299] [fix](expr) Fix BE core dump while common expr filter
 delete condition column (#29107)

pred column also needs to be filtered by expr, exclude delete condition column, delete condition column not need to be filtered, query engine does not need it, after _output_column_by_sel_idx, delete condition materialize column will be erase at the end of the block.
Eg:
delete from table where a = 10;
select b from table;
a column only effective in segment iterator, the block from query engine only contain the b column, so no need to filter a column by expr.
---
 .../rowset/segment_v2/segment_iterator.cpp    | 34 ++++++++++-
 .../correctness/test_pushdown_common_expr.out | 18 ++++++
 .../test_pushdown_common_expr.groovy          | 58 +++++++++++++++++++
 3 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 135e576825fb72..6bc01ea0d4be6e 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1504,6 +1504,8 @@ Status SegmentIterator::_seek_columns(const std::vector<ColumnId>& column_ids, r
 // todo(wb) need a UT here
 Status SegmentIterator::_vec_init_lazy_materialization() {
     _is_pred_column.resize(_schema->columns().size(), false);
+    std::vector<bool> is_pred_column_no_del_condition;
+    is_pred_column_no_del_condition.resize(_schema->columns().size(), false);
 
     // including short/vec/delete pred
     std::set<ColumnId> pred_column_ids;
@@ -1545,6 +1547,7 @@ Status SegmentIterator::_vec_init_lazy_materialization() {
         for (auto predicate : _col_predicates) {
             auto cid = predicate->column_id();
             _is_pred_column[cid] = true;
+            is_pred_column_no_del_condition[cid] = true;
             pred_column_ids.insert(cid);
 
             // check pred using short eval or vec eval
@@ -1598,8 +1601,16 @@ Status SegmentIterator::_vec_init_lazy_materialization() {
         if (!_common_expr_columns.empty()) {
             _is_need_expr_eval = true;
             for (auto cid : _schema->column_ids()) {
-                // pred column also needs to be filtered by expr
-                if (_is_common_expr_column[cid] || _is_pred_column[cid]) {
+                // pred column also needs to be filtered by expr, exclude delete condition column,
+                // Delete condition column not need to be filtered, query engine does not need it,
+                // after _output_column_by_sel_idx, delete condition materialize column will be erase
+                // at the end of the block.
+                // Eg:
+                //      `delete from table where a = 10;`
+                //      `select b from table;`
+                // a column only effective in segment iterator, the block from query engine only contain the b column,
+                // so no need to filter a column by expr.
+                if (_is_common_expr_column[cid] || is_pred_column_no_del_condition[cid]) {
                     auto loc = _schema_block_id_map[cid];
                     _columns_to_filter.push_back(loc);
                 }
@@ -2173,6 +2184,25 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
         _replace_version_col(_current_batch_rows_read);
     }
 
+    // If col >= block->columns(), it means col should not be filtered, there is a BUG.
+    // such as delete condition column was incorrectly put into columns_to_filter,
+    // which is usually at the end of the block. only check during the first next_batch.
+    if (_opts.stats->blocks_load == 0) {
+        for (const auto& col : _columns_to_filter) {
+            if (col >= block->columns()) {
+                std::ostringstream ss;
+                for (const auto& i : _columns_to_filter) {
+                    ss << i << "-";
+                }
+                throw Exception(
+                        ErrorCode::INTERNAL_ERROR,
+                        "filter block column id(index) greater than block->columns(), "
+                        "column id={}, all columns that need filter={}, block columns num={}",
+                        col, ss.str().substr(0, ss.str().length() - 1), block->columns());
+            }
+        }
+    }
+
     _opts.stats->blocks_load += 1;
     _opts.stats->raw_rows_read += _current_batch_rows_read;
 
diff --git a/regression-test/data/correctness/test_pushdown_common_expr.out b/regression-test/data/correctness/test_pushdown_common_expr.out
index 8da380e841b23a..0055c93b7d83de 100644
--- a/regression-test/data/correctness/test_pushdown_common_expr.out
+++ b/regression-test/data/correctness/test_pushdown_common_expr.out
@@ -71,3 +71,21 @@
 
 -- !9 --
 
+-- !1 --
+0
+
+-- !2 --
+0
+
+-- !3 --
+0
+
+-- !4 --
+0
+
+-- !5 --
+0
+
+-- !6 --
+a
+
diff --git a/regression-test/suites/correctness/test_pushdown_common_expr.groovy b/regression-test/suites/correctness/test_pushdown_common_expr.groovy
index 2395f057b1df4d..aa0e68e3e4c5f5 100644
--- a/regression-test/suites/correctness/test_pushdown_common_expr.groovy
+++ b/regression-test/suites/correctness/test_pushdown_common_expr.groovy
@@ -150,4 +150,62 @@ suite("test_pushdown_common_expr") {
             WHERE random() = 0
         ) AS t2 ON t1.c1 = t2.c1
     """
+
+    sql """ DROP TABLE IF EXISTS t_pushdown_common_expr_for_del """
+    sql """
+    CREATE TABLE `t_pushdown_common_expr_for_del` (
+        `c1` int(11) NULL,
+        `c2` varchar(100) NULL COMMENT "",
+        `c3` varchar(100) NULL COMMENT "",
+        `c4` varchar(100) NULL COMMENT "",
+        `c5` varchar(100) NULL COMMENT "",
+        `c6` varchar(100) NULL COMMENT "",
+        `c7` varchar(100) NULL COMMENT ""
+    ) ENGINE=OLAP
+    DUPLICATE KEY(`c1`)
+    COMMENT 'OLAP'
+    DISTRIBUTED BY HASH(`c1`) BUCKETS 1
+    PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "disable_auto_compaction" = "true"
+    );
+    """
+
+    sql """
+        INSERT INTO t_pushdown_common_expr_for_del VALUES
+        (1,'a','aa','aaa','aaaa','aaaaa','aaaaaa'),
+        (2,'b','bb','bbb','bbbb','bbbbb','bbbbbb'),
+        (3,'c','cc','ccc','cccc','ccccc','cccccc'),
+        (4,'d','dd','ddd','dddd','ddddd','dddddd'),
+        (5,'e','ee','eee','eeee','eeeee','eeeeee'),
+        (6,'f','ff','fff','ffff','fffff','ffffff'),
+        (7,'g','gg','ggg','gggg','ggggg','gggggg');
+     """
+
+    sql """set enable_common_expr_pushdown=true"""
+
+    // delete condition columns num > block columns num
+    order_qt_1 """
+        delete from t_pushdown_common_expr_for_del where c3 = "bb";
+    """
+
+    order_qt_2 """
+        delete from t_pushdown_common_expr_for_del where c4 = "ccc";
+    """
+
+    order_qt_3 """
+        delete from t_pushdown_common_expr_for_del where c5 = "dddd";
+    """
+
+    order_qt_4 """
+        delete from t_pushdown_common_expr_for_del where c6 = "eeeee";
+    """
+
+    order_qt_5 """
+        delete from t_pushdown_common_expr_for_del where c7 = "ffffff";
+    """
+
+    order_qt_6 """
+        select c2 from t_pushdown_common_expr_for_del where upper(c2) = "A";
+    """
 }
\ No newline at end of file

From a7c0dddbc9206ed47812ba2fb9bdadac0add067c Mon Sep 17 00:00:00 2001
From: zhiqiang <seuhezhiqiang@163.com>
Date: Thu, 28 Dec 2023 12:54:47 +0800
Subject: [PATCH 103/299] [refactor](rename) Rename some variables in pipeline
 for better readability (#29140)

* rft-rename

* format
---
 be/src/pipeline/pipeline.cpp                  |  6 +--
 be/src/pipeline/pipeline.h                    |  6 +--
 be/src/pipeline/pipeline_fragment_context.cpp | 42 ++++++++++---------
 3 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/be/src/pipeline/pipeline.cpp b/be/src/pipeline/pipeline.cpp
index d8ac73374f5e6b..7990f84df49364 100644
--- a/be/src/pipeline/pipeline.cpp
+++ b/be/src/pipeline/pipeline.cpp
@@ -71,14 +71,14 @@ Status Pipeline::prepare(RuntimeState* state) {
     return Status::OK();
 }
 
-Status Pipeline::set_sink(OperatorBuilderPtr& sink_) {
-    if (_sink) {
+Status Pipeline::set_sink_builder(OperatorBuilderPtr& sink_) {
+    if (_sink_builder) {
         return Status::InternalError("set sink twice");
     }
     if (!sink_->is_sink()) {
         return Status::InternalError("should set a sink operator but {}", typeid(sink_).name());
     }
-    _sink = sink_;
+    _sink_builder = sink_;
     return Status::OK();
 }
 
diff --git a/be/src/pipeline/pipeline.h b/be/src/pipeline/pipeline.h
index 2775c45019e726..ef0acfba2584ed 100644
--- a/be/src/pipeline/pipeline.h
+++ b/be/src/pipeline/pipeline.h
@@ -96,10 +96,10 @@ class Pipeline : public std::enable_shared_from_this<Pipeline> {
     // prepare operators for pipelineX
     Status prepare(RuntimeState* state);
 
-    Status set_sink(OperatorBuilderPtr& sink_operator);
+    Status set_sink_builder(OperatorBuilderPtr& sink_operator_builder);
     Status set_sink(DataSinkOperatorXPtr& sink_operator);
 
-    OperatorBuilderBase* sink() { return _sink.get(); }
+    OperatorBuilderBase* get_sink_builder() { return _sink_builder.get(); }
     DataSinkOperatorXBase* sink_x() { return _sink_x.get(); }
     OperatorXs& operator_xs() { return operatorXs; }
     DataSinkOperatorXPtr sink_shared_pointer() { return _sink_x; }
@@ -185,7 +185,7 @@ class Pipeline : public std::enable_shared_from_this<Pipeline> {
     void _init_profile();
 
     OperatorBuilders _operator_builders; // left is _source, right is _root
-    OperatorBuilderPtr _sink;            // put block to sink
+    OperatorBuilderPtr _sink_builder;    // put block to sink
 
     std::mutex _depend_mutex;
     std::vector<std::pair<int, std::weak_ptr<Pipeline>>> _parents;
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp b/be/src/pipeline/pipeline_fragment_context.cpp
index 034fbaf29513cd..695fd6f4d3d25d 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -35,6 +35,7 @@
 
 #include "common/config.h"
 #include "common/logging.h"
+#include "common/status.h"
 #include "exec/data_sink.h"
 #include "exec/exec_node.h"
 #include "exec/scan_node.h"
@@ -332,6 +333,7 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re
     _runtime_state->set_num_local_sink(request.num_local_sink);
 
     if (request.fragment.__isset.output_sink) {
+        // Here we build a DataSink object, which will be hold by DataSinkOperator
         RETURN_IF_ERROR_OR_CATCH_EXCEPTION(DataSink::create_data_sink(
                 _runtime_state->obj_pool(), request.fragment.output_sink,
                 request.fragment.output_exprs, request, idx, _root_plan->row_desc(),
@@ -343,6 +345,7 @@ Status PipelineFragmentContext::prepare(const doris::TPipelineFragmentParams& re
     _root_pipeline->set_collect_query_statistics_with_every_batch();
     RETURN_IF_ERROR(_build_pipelines(_root_plan, _root_pipeline));
     if (_sink) {
+        // DataSinkOperator is builded here
         RETURN_IF_ERROR(_create_sink(request.local_params[idx].sender_id,
                                      request.fragment.output_sink, _runtime_state.get()));
     }
@@ -366,14 +369,15 @@ Status PipelineFragmentContext::_build_pipeline_tasks(
     _total_tasks = 0;
     for (PipelinePtr& pipeline : _pipelines) {
         // if sink
-        auto sink = pipeline->sink()->build_operator();
+        auto sink_operator = pipeline->get_sink_builder()->build_operator();
         // TODO pipeline 1 need to add new interface for exec node and operator
-        static_cast<void>(sink->init(request.fragment.output_sink));
+        RETURN_IF_ERROR(sink_operator->init(request.fragment.output_sink));
 
         RETURN_IF_ERROR(pipeline->build_operators());
-        auto task = std::make_unique<PipelineTask>(pipeline, _total_tasks++, _runtime_state.get(),
-                                                   sink, this, pipeline->pipeline_profile());
-        static_cast<void>(sink->set_child(task->get_root()));
+        auto task =
+                std::make_unique<PipelineTask>(pipeline, _total_tasks++, _runtime_state.get(),
+                                               sink_operator, this, pipeline->pipeline_profile());
+        RETURN_IF_ERROR(sink_operator->set_child(task->get_root()));
         _tasks.emplace_back(std::move(task));
         _runtime_profile->add_child(pipeline->pipeline_profile(), true, nullptr);
     }
@@ -524,7 +528,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
                 RETURN_IF_ERROR(_build_pipelines(union_node->child(child_id), new_child_pipeline));
                 OperatorBuilderPtr child_sink_builder = std::make_shared<UnionSinkOperatorBuilder>(
                         union_node->id(), child_id, union_node, data_queue);
-                RETURN_IF_ERROR(new_child_pipeline->set_sink(child_sink_builder));
+                RETURN_IF_ERROR(new_child_pipeline->set_sink_builder(child_sink_builder));
             }
             OperatorBuilderPtr source_builder = std::make_shared<UnionSourceOperatorBuilder>(
                     node->id(), union_node, data_queue);
@@ -541,7 +545,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
             OperatorBuilderPtr pre_agg_sink =
                     std::make_shared<DistinctStreamingAggSinkOperatorBuilder>(node->id(), agg_node,
                                                                               data_queue);
-            RETURN_IF_ERROR(new_pipe->set_sink(pre_agg_sink));
+            RETURN_IF_ERROR(new_pipe->set_sink_builder(pre_agg_sink));
 
             OperatorBuilderPtr pre_agg_source =
                     std::make_shared<DistinctStreamingAggSourceOperatorBuilder>(
@@ -551,7 +555,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
             auto data_queue = std::make_shared<DataQueue>(1);
             OperatorBuilderPtr pre_agg_sink = std::make_shared<StreamingAggSinkOperatorBuilder>(
                     node->id(), agg_node, data_queue);
-            RETURN_IF_ERROR(new_pipe->set_sink(pre_agg_sink));
+            RETURN_IF_ERROR(new_pipe->set_sink_builder(pre_agg_sink));
 
             OperatorBuilderPtr pre_agg_source = std::make_shared<StreamingAggSourceOperatorBuilder>(
                     node->id(), agg_node, data_queue);
@@ -559,7 +563,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
         } else {
             OperatorBuilderPtr agg_sink =
                     std::make_shared<AggSinkOperatorBuilder>(node->id(), agg_node);
-            RETURN_IF_ERROR(new_pipe->set_sink(agg_sink));
+            RETURN_IF_ERROR(new_pipe->set_sink_builder(agg_sink));
 
             OperatorBuilderPtr agg_source =
                     std::make_shared<AggSourceOperatorBuilder>(node->id(), agg_node);
@@ -572,7 +576,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
         RETURN_IF_ERROR(_build_pipelines(node->child(0), new_pipeline));
 
         OperatorBuilderPtr sort_sink = std::make_shared<SortSinkOperatorBuilder>(node->id(), node);
-        RETURN_IF_ERROR(new_pipeline->set_sink(sort_sink));
+        RETURN_IF_ERROR(new_pipeline->set_sink_builder(sort_sink));
 
         OperatorBuilderPtr sort_source =
                 std::make_shared<SortSourceOperatorBuilder>(node->id(), node);
@@ -585,7 +589,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
 
         OperatorBuilderPtr partition_sort_sink =
                 std::make_shared<PartitionSortSinkOperatorBuilder>(node->id(), node);
-        RETURN_IF_ERROR(new_pipeline->set_sink(partition_sort_sink));
+        RETURN_IF_ERROR(new_pipeline->set_sink_builder(partition_sort_sink));
 
         OperatorBuilderPtr partition_sort_source =
                 std::make_shared<PartitionSortSourceOperatorBuilder>(node->id(), node);
@@ -598,7 +602,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
 
         OperatorBuilderPtr analytic_sink =
                 std::make_shared<AnalyticSinkOperatorBuilder>(node->id(), node);
-        RETURN_IF_ERROR(new_pipeline->set_sink(analytic_sink));
+        RETURN_IF_ERROR(new_pipeline->set_sink_builder(analytic_sink));
 
         OperatorBuilderPtr analytic_source =
                 std::make_shared<AnalyticSourceOperatorBuilder>(node->id(), node);
@@ -637,7 +641,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
         }
         OperatorBuilderPtr join_sink =
                 std::make_shared<HashJoinBuildSinkBuilder>(node->id(), join_node);
-        RETURN_IF_ERROR(new_pipe->set_sink(join_sink));
+        RETURN_IF_ERROR(new_pipe->set_sink_builder(join_sink));
 
         RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe));
         OperatorBuilderPtr join_source =
@@ -652,7 +656,7 @@ Status PipelineFragmentContext::_build_pipelines(ExecNode* node, PipelinePtr cur
         RETURN_IF_ERROR(_build_pipelines(node->child(1), new_pipe));
         OperatorBuilderPtr join_sink =
                 std::make_shared<NestLoopJoinBuildOperatorBuilder>(node->id(), node);
-        RETURN_IF_ERROR(new_pipe->set_sink(join_sink));
+        RETURN_IF_ERROR(new_pipe->set_sink_builder(join_sink));
 
         RETURN_IF_ERROR(_build_pipelines(node->child(0), cur_pipe));
         OperatorBuilderPtr join_source =
@@ -690,7 +694,7 @@ Status PipelineFragmentContext::_build_operators_for_set_operation_node(ExecNode
     RETURN_IF_ERROR(_build_pipelines(node->child(0), build_pipeline));
     OperatorBuilderPtr sink_builder =
             std::make_shared<SetSinkOperatorBuilder<is_intersect>>(node->id(), node);
-    RETURN_IF_ERROR(build_pipeline->set_sink(sink_builder));
+    RETURN_IF_ERROR(build_pipeline->set_sink_builder(sink_builder));
 
     for (int child_id = 1; child_id < node->children_count(); ++child_id) {
         auto probe_pipeline = add_pipeline();
@@ -698,7 +702,7 @@ Status PipelineFragmentContext::_build_operators_for_set_operation_node(ExecNode
         OperatorBuilderPtr probe_sink_builder =
                 std::make_shared<SetProbeSinkOperatorBuilder<is_intersect>>(node->id(), child_id,
                                                                             node);
-        RETURN_IF_ERROR(probe_pipeline->set_sink(probe_sink_builder));
+        RETURN_IF_ERROR(probe_pipeline->set_sink_builder(probe_sink_builder));
     }
 
     OperatorBuilderPtr source_builder =
@@ -827,7 +831,7 @@ Status PipelineFragmentContext::_create_sink(int sender_id, const TDataSink& thr
     case TDataSinkType::MULTI_CAST_DATA_STREAM_SINK: {
         sink_ = std::make_shared<MultiCastDataStreamSinkOperatorBuilder>(next_operator_builder_id(),
                                                                          _sink.get());
-        RETURN_IF_ERROR(_root_pipeline->set_sink(sink_));
+        RETURN_IF_ERROR(_root_pipeline->set_sink_builder(sink_));
 
         auto& multi_cast_data_streamer =
                 assert_cast<vectorized::MultiCastDataStreamSink*>(_sink.get())
@@ -862,7 +866,7 @@ Status PipelineFragmentContext::_create_sink(int sender_id, const TDataSink& thr
             // 3. create and set sink operator of data stream sender for new pipeline
             OperatorBuilderPtr sink_op_builder = std::make_shared<ExchangeSinkOperatorBuilder>(
                     next_operator_builder_id(), _multi_cast_stream_sink_senders[i].get(), i);
-            static_cast<void>(new_pipeline->set_sink(sink_op_builder));
+            static_cast<void>(new_pipeline->set_sink_builder(sink_op_builder));
 
             // 4. init and prepare the data_stream_sender of diff exchange
             TDataSink t;
@@ -876,7 +880,7 @@ Status PipelineFragmentContext::_create_sink(int sender_id, const TDataSink& thr
     default:
         return Status::InternalError("Unsuported sink type in pipeline: {}", thrift_sink.type);
     }
-    return _root_pipeline->set_sink(sink_);
+    return _root_pipeline->set_sink_builder(sink_);
 }
 
 // If all pipeline tasks binded to the fragment instance are finished, then we could

From fd90c3a6a655337b4acc59d5c7deb09098e07fc5 Mon Sep 17 00:00:00 2001
From: xy <whuxingying@163.com>
Date: Thu, 28 Dec 2023 13:03:33 +0800
Subject: [PATCH 104/299] [optimize](cooldown)Reduce the number of calls to the
 pick_cooldown_rowset (#27091)

Co-authored-by: xingying01 <xingying01@corp.netease.com>
---
 be/src/olap/olap_server.cpp    |  9 ++++++---
 be/src/olap/tablet.cpp         | 35 ++++++++++++++++++++++++----------
 be/src/olap/tablet.h           |  6 +++---
 be/src/olap/tablet_manager.cpp | 12 +++++++++---
 be/src/olap/tablet_manager.h   |  1 +
 be/test/olap/tablet_test.cpp   | 16 ++++++++--------
 6 files changed, 52 insertions(+), 27 deletions(-)

diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 7268d7959f998a..29aa4eb05fa099 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1037,6 +1037,7 @@ void StorageEngine::_cooldown_tasks_producer_callback() {
     do {
         // these tables are ordered by priority desc
         std::vector<TabletSharedPtr> tablets;
+        std::vector<RowsetSharedPtr> rowsets;
         // TODO(luwei) : a more efficient way to get cooldown tablets
         auto cur_time = time(nullptr);
         // we should skip all the tablets which are not running and those pending to do cooldown
@@ -1053,17 +1054,19 @@ void StorageEngine::_cooldown_tasks_producer_callback() {
             return _running_cooldown_tablets.find(tablet->tablet_id()) !=
                    _running_cooldown_tablets.end();
         };
-        _tablet_manager->get_cooldown_tablets(&tablets, std::move(skip_tablet));
+        _tablet_manager->get_cooldown_tablets(&tablets, &rowsets, std::move(skip_tablet));
         LOG(INFO) << "cooldown producer get tablet num: " << tablets.size();
         int max_priority = tablets.size();
+        int index = 0;
         for (const auto& tablet : tablets) {
             {
                 std::lock_guard<std::mutex> lock(_running_cooldown_mutex);
                 _running_cooldown_tablets.insert(tablet->tablet_id());
             }
             PriorityThreadPool::Task task;
-            task.work_function = [tablet, task_size = tablets.size(), this]() {
-                Status st = tablet->cooldown();
+            RowsetSharedPtr rowset = std::move(rowsets[index++]);
+            task.work_function = [tablet, rowset, task_size = tablets.size(), this]() {
+                Status st = tablet->cooldown(rowset);
                 {
                     std::lock_guard<std::mutex> lock(_running_cooldown_mutex);
                     _running_cooldown_tablets.erase(tablet->tablet_id());
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 8cfce2d35ec523..2d4b430334b04f 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2017,7 +2017,7 @@ Status Tablet::create_rowset(const RowsetMetaSharedPtr& rowset_meta, RowsetShare
             rowset);
 }
 
-Status Tablet::cooldown() {
+Status Tablet::cooldown(RowsetSharedPtr rowset) {
     std::unique_lock schema_change_lock(_schema_change_lock, std::try_to_lock);
     if (!schema_change_lock.owns_lock()) {
         return Status::Error<TRY_LOCK_FAILED>("try schema_change_lock failed");
@@ -2038,7 +2038,7 @@ Status Tablet::cooldown() {
 
     if (_cooldown_replica_id == replica_id()) {
         // this replica is cooldown replica
-        RETURN_IF_ERROR(_cooldown_data());
+        RETURN_IF_ERROR(_cooldown_data(std::move(rowset)));
     } else {
         Status st = _follow_cooldowned_data();
         if (UNLIKELY(!st.ok())) {
@@ -2051,12 +2051,27 @@ Status Tablet::cooldown() {
 }
 
 // hold SHARED `cooldown_conf_lock`
-Status Tablet::_cooldown_data() {
+Status Tablet::_cooldown_data(RowsetSharedPtr rowset) {
     DCHECK(_cooldown_replica_id == replica_id());
 
     std::shared_ptr<io::RemoteFileSystem> dest_fs;
     RETURN_IF_ERROR(get_remote_file_system(storage_policy_id(), &dest_fs));
-    auto old_rowset = pick_cooldown_rowset();
+    RowsetSharedPtr old_rowset = nullptr;
+
+    if (rowset) {
+        const auto& rowset_id = rowset->rowset_id();
+        const auto& rowset_version = rowset->version();
+        std::shared_lock meta_rlock(_meta_lock);
+        auto iter = _rs_version_map.find(rowset_version);
+        if (iter != _rs_version_map.end() && iter->second->rowset_id() == rowset_id) {
+            old_rowset = rowset;
+        }
+    }
+
+    if (!old_rowset) {
+        old_rowset = pick_cooldown_rowset();
+    }
+
     if (!old_rowset) {
         LOG(INFO) << "cannot pick cooldown rowset in tablet " << tablet_id();
         return Status::OK();
@@ -2353,23 +2368,23 @@ RowsetSharedPtr Tablet::pick_cooldown_rowset() {
     return rowset;
 }
 
-bool Tablet::need_cooldown(int64_t* cooldown_timestamp, size_t* file_size) {
+RowsetSharedPtr Tablet::need_cooldown(int64_t* cooldown_timestamp, size_t* file_size) {
     int64_t id = storage_policy_id();
     if (id <= 0) {
         VLOG_DEBUG << "tablet does not need cooldown, tablet id: " << tablet_id();
-        return false;
+        return nullptr;
     }
     auto storage_policy = get_storage_policy(id);
     if (!storage_policy) {
         LOG(WARNING) << "Cannot get storage policy: " << id;
-        return false;
+        return nullptr;
     }
     auto cooldown_ttl_sec = storage_policy->cooldown_ttl;
     auto cooldown_datetime = storage_policy->cooldown_datetime;
     RowsetSharedPtr rowset = pick_cooldown_rowset();
     if (!rowset) {
         VLOG_DEBUG << "pick cooldown rowset, get null, tablet id: " << tablet_id();
-        return false;
+        return nullptr;
     }
 
     int64_t newest_cooldown_time = std::numeric_limits<int64_t>::max();
@@ -2387,13 +2402,13 @@ bool Tablet::need_cooldown(int64_t* cooldown_timestamp, size_t* file_size) {
         *file_size = rowset->data_disk_size();
         VLOG_DEBUG << "tablet need cooldown, tablet id: " << tablet_id()
                    << " file_size: " << *file_size;
-        return true;
+        return rowset;
     }
 
     VLOG_DEBUG << "tablet does not need cooldown, tablet id: " << tablet_id()
                << " ttl sec: " << cooldown_ttl_sec << " cooldown datetime: " << cooldown_datetime
                << " newest write time: " << rowset->newest_write_timestamp();
-    return false;
+    return nullptr;
 }
 
 void Tablet::record_unused_remote_rowset(const RowsetId& rowset_id, const std::string& resource,
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 614288921ed3f8..9f7141b404f04e 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -355,11 +355,11 @@ class Tablet final : public BaseTablet {
     int64_t last_failed_follow_cooldown_time() const { return _last_failed_follow_cooldown_time; }
 
     // Cooldown to remote fs.
-    Status cooldown();
+    Status cooldown(RowsetSharedPtr rowset = nullptr);
 
     RowsetSharedPtr pick_cooldown_rowset();
 
-    bool need_cooldown(int64_t* cooldown_timestamp, size_t* file_size);
+    RowsetSharedPtr need_cooldown(int64_t* cooldown_timestamp, size_t* file_size);
 
     std::pair<int64_t, int64_t> cooldown_conf() const {
         std::shared_lock rlock(_cooldown_conf_lock);
@@ -591,7 +591,7 @@ class Tablet final : public BaseTablet {
     ////////////////////////////////////////////////////////////////////////////
     // begin cooldown functions
     ////////////////////////////////////////////////////////////////////////////
-    Status _cooldown_data();
+    Status _cooldown_data(RowsetSharedPtr rowset);
     Status _follow_cooldowned_data();
     Status _read_cooldown_meta(const std::shared_ptr<io::RemoteFileSystem>& fs,
                                TabletMetaPB* tablet_meta_pb);
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index ee060d0fed76ea..6010121424d5df 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1414,9 +1414,11 @@ void TabletManager::get_tablets_distribution_on_different_disks(
 }
 
 struct SortCtx {
-    SortCtx(TabletSharedPtr tablet, int64_t cooldown_timestamp, int64_t file_size)
+    SortCtx(TabletSharedPtr tablet, RowsetSharedPtr rowset, int64_t cooldown_timestamp,
+            int64_t file_size)
             : tablet(tablet), cooldown_timestamp(cooldown_timestamp), file_size(file_size) {}
     TabletSharedPtr tablet;
+    RowsetSharedPtr rowset;
     // to ensure the tablet with -1 would always be greater than other
     uint64_t cooldown_timestamp;
     int64_t file_size;
@@ -1429,6 +1431,7 @@ struct SortCtx {
 };
 
 void TabletManager::get_cooldown_tablets(std::vector<TabletSharedPtr>* tablets,
+                                         std::vector<RowsetSharedPtr>* rowsets,
                                          std::function<bool(const TabletSharedPtr&)> skip_tablet) {
     std::vector<SortCtx> sort_ctx_vec;
     std::vector<std::weak_ptr<Tablet>> candidates;
@@ -1436,14 +1439,16 @@ void TabletManager::get_cooldown_tablets(std::vector<TabletSharedPtr>* tablets,
                     filter_all_tablets);
     auto get_cooldown_tablet = [&sort_ctx_vec, &skip_tablet](std::weak_ptr<Tablet>& t) {
         const TabletSharedPtr& tablet = t.lock();
+        RowsetSharedPtr rowset = nullptr;
         if (UNLIKELY(nullptr == tablet)) {
             return;
         }
         std::shared_lock rdlock(tablet->get_header_lock());
         int64_t cooldown_timestamp = -1;
         size_t file_size = -1;
-        if (!skip_tablet(tablet) && tablet->need_cooldown(&cooldown_timestamp, &file_size)) {
-            sort_ctx_vec.emplace_back(tablet, cooldown_timestamp, file_size);
+        if (!skip_tablet(tablet) &&
+            (rowset = tablet->need_cooldown(&cooldown_timestamp, &file_size))) {
+            sort_ctx_vec.emplace_back(tablet, rowset, cooldown_timestamp, file_size);
         }
     };
     std::for_each(candidates.begin(), candidates.end(), get_cooldown_tablet);
@@ -1453,6 +1458,7 @@ void TabletManager::get_cooldown_tablets(std::vector<TabletSharedPtr>* tablets,
     for (SortCtx& ctx : sort_ctx_vec) {
         VLOG_DEBUG << "get cooldown tablet: " << ctx.tablet->tablet_id();
         tablets->push_back(std::move(ctx.tablet));
+        rowsets->push_back(std::move(ctx.rowset));
     }
 }
 
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 47d22930cbfe0c..04131b8d3bb216 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -156,6 +156,7 @@ class TabletManager {
             std::map<int64_t, std::map<DataDir*, int64_t>>& tablets_num_on_disk,
             std::map<int64_t, std::map<DataDir*, std::vector<TabletSize>>>& tablets_info_on_disk);
     void get_cooldown_tablets(std::vector<TabletSharedPtr>* tables,
+                              std::vector<RowsetSharedPtr>* rowsets,
                               std::function<bool(const TabletSharedPtr&)> skip_tablet);
 
     void get_all_tablets_storage_format(TCheckStorageFormatResult* result);
diff --git a/be/test/olap/tablet_test.cpp b/be/test/olap/tablet_test.cpp
index a5fc57a206e94b..65454dea5e0829 100644
--- a/be/test/olap/tablet_test.cpp
+++ b/be/test/olap/tablet_test.cpp
@@ -343,8 +343,8 @@ TEST_F(TestTablet, cooldown_policy) {
 
         int64_t cooldown_timestamp = -1;
         size_t file_size = -1;
-        bool ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
-        ASSERT_TRUE(ret);
+        auto ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
+        ASSERT_TRUE(ret != nullptr);
         ASSERT_EQ(cooldown_timestamp, 250);
         ASSERT_EQ(file_size, 84699);
     }
@@ -357,8 +357,8 @@ TEST_F(TestTablet, cooldown_policy) {
 
         int64_t cooldown_timestamp = -1;
         size_t file_size = -1;
-        bool ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
-        ASSERT_TRUE(ret);
+        auto ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
+        ASSERT_TRUE(ret != nullptr);
         ASSERT_EQ(cooldown_timestamp, 3800);
         ASSERT_EQ(file_size, 84699);
     }
@@ -371,8 +371,8 @@ TEST_F(TestTablet, cooldown_policy) {
 
         int64_t cooldown_timestamp = -1;
         size_t file_size = -1;
-        bool ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
-        ASSERT_FALSE(ret);
+        auto ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
+        ASSERT_FALSE(ret != nullptr);
         ASSERT_EQ(cooldown_timestamp, -1);
         ASSERT_EQ(file_size, -1);
     }
@@ -385,11 +385,11 @@ TEST_F(TestTablet, cooldown_policy) {
 
         int64_t cooldown_timestamp = -1;
         size_t file_size = -1;
-        bool ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
+        auto ret = _tablet->need_cooldown(&cooldown_timestamp, &file_size);
         // the rowset with earliest version woule be picked up to do cooldown of which the timestamp
         // is UnixSeconds() - 250
         int64_t expect_cooldown_timestamp = UnixSeconds() - 50;
-        ASSERT_TRUE(ret);
+        ASSERT_TRUE(ret != nullptr);
         ASSERT_EQ(cooldown_timestamp, expect_cooldown_timestamp);
         ASSERT_EQ(file_size, 84699);
     }

From bc08535285ac489559b0e50c4c20f84bb2b72ea4 Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Thu, 28 Dec 2023 13:26:09 +0800
Subject: [PATCH 105/299] [fix](Nereids) throw readable exception when meet
 unsupport sup-query (#29147)

---
 .../src/main/java/org/apache/doris/nereids/util/Utils.java  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
index 456dcafdb2fe46..baf37c1c6473f8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
@@ -163,8 +163,10 @@ public static List<Expression> getCorrelatedSlots(List<Expression> correlatedPre
             List<Expression> correlatedSlots) {
         List<Expression> slots = new ArrayList<>();
         correlatedPredicates.forEach(predicate -> {
-            if (!(predicate instanceof BinaryExpression) && !(predicate instanceof Not)) {
-                throw new AnalysisException("UnSupported expr type: " + correlatedPredicates);
+            if (!(predicate instanceof BinaryExpression)
+                    && (!(predicate instanceof Not) || !(predicate.child(0) instanceof BinaryExpression))) {
+                throw new AnalysisException("Unsupported correlated subquery with"
+                        + " non-equals correlated predicate " + predicate.toSql());
             }
 
             BinaryExpression binaryExpression;

From f816d13c56a3067a98994a4a74003c6eeb14da78 Mon Sep 17 00:00:00 2001
From: jakevin <jakevingoo@gmail.com>
Date: Thu, 28 Dec 2023 14:00:41 +0800
Subject: [PATCH 106/299] [feature](Nereids): eliminate groupby (#28615)

---
 .../doris/nereids/jobs/executor/Rewriter.java |   5 +
 .../properties/FunctionalDependencies.java    |   3 +-
 .../apache/doris/nereids/rules/RuleType.java  |   1 +
 .../rules/rewrite/EliminateGroupBy.java       |  83 +++++++++++++++
 .../rules/rewrite/EliminateGroupByTest.java   | 100 ++++++++++++++++++
 5 files changed, 191 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index fc3ef04fba8d07..4fe9df664c39b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -55,6 +55,7 @@
 import org.apache.doris.nereids.rules.rewrite.EliminateDedupJoinCondition;
 import org.apache.doris.nereids.rules.rewrite.EliminateEmptyRelation;
 import org.apache.doris.nereids.rules.rewrite.EliminateFilter;
+import org.apache.doris.nereids.rules.rewrite.EliminateGroupBy;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinByFK;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinCondition;
 import org.apache.doris.nereids.rules.rewrite.EliminateLimit;
@@ -276,6 +277,10 @@ public class Rewriter extends AbstractBatchJobExecutor {
                     topDown(new BuildAggForUnion())
             ),
 
+            topic("Eliminate GroupBy",
+                    topDown(new EliminateGroupBy())
+            ),
+
             topic("Eager aggregation",
                     topDown(
                             new PushDownSumThroughJoin(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
index c2ba22b5dc8da1..f013740c1729e3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FunctionalDependencies.java
@@ -220,7 +220,8 @@ public void replace(Map<Slot, Slot> replaceMap) {
                     .map(s -> replaceMap.getOrDefault(s, s))
                     .collect(Collectors.toSet());
             slotSets = slotSets.stream()
-                    .map(set -> set.stream().map(replaceMap::get).collect(ImmutableSet.toImmutableSet()))
+                    .map(set -> set.stream().map(s -> replaceMap.getOrDefault(s, s))
+                            .collect(ImmutableSet.toImmutableSet()))
                     .collect(Collectors.toSet());
         }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 8edf2d079a2157..32794946dda19b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -207,6 +207,7 @@ public enum RuleType {
     ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE),
     ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE),
     ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE),
+    ELIMINATE_GROUP_BY(RuleTypeClass.REWRITE),
     ELIMINATE_DEDUP_JOIN_CONDITION(RuleTypeClass.REWRITE),
     ELIMINATE_NULL_AWARE_LEFT_ANTI_JOIN(RuleTypeClass.REWRITE),
     ELIMINATE_ASSERT_NUM_ROWS(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java
new file mode 100644
index 00000000000000..3b95e9b44e06f0
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupBy.java
@@ -0,0 +1,83 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.If;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.util.PlanUtils;
+
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Eliminate GroupBy.
+ */
+public class EliminateGroupBy extends OneRewriteRuleFactory {
+
+    @Override
+    public Rule build() {
+        return logicalAggregate()
+                .when(agg -> agg.getGroupByExpressions().stream().allMatch(expr -> expr instanceof Slot))
+                .then(agg -> {
+                    Set<Slot> groupby = agg.getGroupByExpressions().stream().map(e -> (Slot) e)
+                            .collect(Collectors.toSet());
+                    Plan child = agg.child();
+                    boolean unique = child.getLogicalProperties().getFunctionalDependencies()
+                            .isUniqueAndNotNull(groupby);
+                    if (!unique) {
+                        return null;
+                    }
+                    Set<AggregateFunction> aggregateFunctions = agg.getAggregateFunctions();
+                    if (!aggregateFunctions.stream().allMatch(
+                            f -> (f instanceof Sum || f instanceof Count || f instanceof Min || f instanceof Max)
+                                    && (f.arity() == 1 && f.child(0) instanceof Slot))) {
+                        return null;
+                    }
+
+                    List<NamedExpression> newOutput = agg.getOutputExpressions().stream().map(ne -> {
+                        if (ne instanceof Alias && ne.child(0) instanceof AggregateFunction) {
+                            AggregateFunction f = (AggregateFunction) ne.child(0);
+                            if (f instanceof Sum || f instanceof Min || f instanceof Max) {
+                                return new Alias(ne.getExprId(), f.child(0), ne.getName());
+                            } else if (f instanceof Count) {
+                                return (NamedExpression) ne.withChildren(
+                                        new If(new IsNull(f.child(0)), Literal.of(0), Literal.of(1)));
+                            } else {
+                                throw new IllegalStateException("Unexpected aggregate function: " + f);
+                            }
+                        } else {
+                            return ne;
+                        }
+                    }).collect(Collectors.toList());
+                    return PlanUtils.projectOrSelf(newOutput, child);
+                }).toRule(RuleType.ELIMINATE_GROUP_BY);
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java
new file mode 100644
index 00000000000000..14f62d25f7fe54
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByTest.java
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Test;
+
+class EliminateGroupByTest extends TestWithFeService implements MemoPatternMatchSupported {
+
+    @Override
+    protected void runBeforeAll() throws Exception {
+        createDatabase("eliminate_group_by");
+        createTable(
+                "create table eliminate_group_by.t (\n"
+                        + "id int not null,\n"
+                        + "name varchar(128),\n"
+                        + "age int, sex int"
+                        + ")\n"
+                        + "UNIQUE KEY(id)\n"
+                        + "distributed by hash(id) buckets 10\n"
+                        + "properties('replication_num' = '1');"
+        );
+        connectContext.setDatabase("default_cluster:eliminate_group_by");
+    }
+
+    @Test
+    void eliminateMax() {
+        //  -> select id, age as max(age) from t;
+        String sql = "select id, max(age) from t group by id";
+
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(
+                        logicalProject().when(p -> p.getProjects().get(0).toSql().equals("id")
+                                && p.getProjects().get(1).toSql().equals("age AS `max(age)`"))
+                );
+    }
+
+    @Test
+    void eliminateMin() {
+        //  -> select id, age as min(age) from t;
+        String sql = "select id, min(age) from t group by id";
+
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(
+                        logicalProject().when(p -> p.getProjects().get(0).toSql().equals("id")
+                                && p.getProjects().get(1).toSql().equals("age AS `min(age)`"))
+                );
+    }
+
+    @Test
+    void eliminateSum() {
+        //  -> select id, age as sum(age) from t;
+        String sql = "select id, sum(age) from t group by id";
+
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(
+                        logicalProject().when(p -> p.getProjects().get(0).toSql().equals("id")
+                                && p.getProjects().get(1).toSql().equals("age AS `sum(age)`"))
+                );
+    }
+
+    @Test
+    void eliminateCount() {
+        //  -> select id, case when age is not null then 1 else 0 end from t;
+        String sql = "select id, count(age) from t group by id";
+
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(
+                        logicalProject().when(p -> p.getProjects().get(0).toSql().equals("id")
+                                && p.getProjects().get(1).toSql().equals("if(age IS NULL, 0, 1) AS `if(age IS NULL, 0, 1)`")
+                        )
+                );
+    }
+}

From 4f2d54d462a680268349e8b167bdab96ed6eef2c Mon Sep 17 00:00:00 2001
From: caiconghui <55968745+caiconghui@users.noreply.github.com>
Date: Thu, 28 Dec 2023 14:17:35 +0800
Subject: [PATCH 107/299] [fix](DatabaseTransactionMgr) Fix clean label bug
 which may cause inconsitent editlog operation (#29198)

---
 .../org/apache/doris/load/loadv2/LoadManager.java    | 12 +++---------
 .../doris/transaction/DatabaseTransactionMgr.java    | 10 +++++++---
 .../doris/transaction/GlobalTransactionMgr.java      |  4 ++--
 3 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
index 3909ff5d7849e2..35968139ec83df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadManager.java
@@ -786,22 +786,16 @@ private void cleanLabelInternal(long dbId, String label, boolean isReplay) {
         } finally {
             writeUnlock();
         }
-        LOG.info("clean {} labels on db {} with label '{}' in load mgr.", counter, dbId, label);
-
         // 2. Remove from DatabaseTransactionMgr
         try {
-            Env.getCurrentGlobalTransactionMgr().cleanLabel(dbId, label);
+            Env.getCurrentGlobalTransactionMgr().cleanLabel(dbId, label, isReplay);
         } catch (AnalysisException e) {
             // just ignore, because we don't want to throw any exception here.
             LOG.warn("Exception:", e);
         }
 
-        // 3. Log
-        if (!isReplay) {
-            CleanLabelOperationLog log = new CleanLabelOperationLog(dbId, label);
-            Env.getCurrentEnv().getEditLog().logCleanLabel(log);
-        }
-        LOG.info("finished to clean label on db {} with label {}. is replay: {}", dbId, label, isReplay);
+        LOG.info("finished to clean {} labels on db {} with label '{}' in load mgr. is replay: {}",
+                counter, dbId, label, isReplay);
     }
 
     private void readLock() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 29471255f29bfc..5bb954289cc514 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -50,6 +50,7 @@
 import org.apache.doris.metric.MetricRepo;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.persist.BatchRemoveTransactionsOperationV2;
+import org.apache.doris.persist.CleanLabelOperationLog;
 import org.apache.doris.persist.EditLog;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.statistics.AnalysisManager;
@@ -2072,7 +2073,7 @@ public void unprotectWriteAllTransactionStates(DataOutput out) throws IOExceptio
         }
     }
 
-    protected void cleanLabel(String label) {
+    protected void cleanLabel(String label, boolean isReplay) {
         Set<Long> removedTxnIds = Sets.newHashSet();
         writeLock();
         try {
@@ -2113,11 +2114,14 @@ protected void cleanLabel(String label) {
             // So that we can keep consistency in meta image
             finalStatusTransactionStateDequeShort.removeIf(txn -> removedTxnIds.contains(txn.getTransactionId()));
             finalStatusTransactionStateDequeLong.removeIf(txn -> removedTxnIds.contains(txn.getTransactionId()));
+
+            if (!isReplay) {
+                CleanLabelOperationLog log = new CleanLabelOperationLog(dbId, label);
+                Env.getCurrentEnv().getEditLog().logCleanLabel(log);
+            }
         } finally {
             writeUnlock();
         }
-        LOG.info("clean {} labels on db {} with label '{}' in database transaction mgr.", removedTxnIds.size(), dbId,
-                label);
     }
 
     public long getTxnNumByStatus(TransactionStatus status) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
index ee228a10fc867d..4832d66bedb030 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
@@ -754,8 +754,8 @@ public List<TransactionState> getPreCommittedTxnList(Long dbId) throws AnalysisE
         return getDatabaseTransactionMgr(dbId).getPreCommittedTxnList();
     }
 
-    public void cleanLabel(Long dbId, String label) throws AnalysisException {
-        getDatabaseTransactionMgr(dbId).cleanLabel(label);
+    public void cleanLabel(Long dbId, String label, boolean isReplay) throws AnalysisException {
+        getDatabaseTransactionMgr(dbId).cleanLabel(label, isReplay);
     }
 
     public Long getTransactionIdByLabel(Long dbId, String label, List<TransactionStatus> statusList)

From 2e910dac2ac5590ca2e8389e17675808e2462e71 Mon Sep 17 00:00:00 2001
From: zhengyu <freeman.zhang1992@gmail.com>
Date: Thu, 28 Dec 2023 14:32:29 +0800
Subject: [PATCH 108/299] [enhencement](segcompaction) cancel inflight
 segcompaction tasks faster when load finish (#28901)

[Goal]
When building the rowset writer, avoid waiting for inflight segcompaction
to elimite long tail latency for load.

[Current situation]
1. The segcompaction of a rowset is executed serially. During the build phase,
we need to wait for the completion of the inflight segcompaction task.

2. If the rowset writer finishes writing and starts building meta, then segments
that have not been compacted will not be submitted to segcompaction worker.
We simply ignore them to accelerate the build process.

3. But this is not enough. If a segcompaction task has already been submitted to
the worker thread pool, we will set a cancelled flag for the worker,
and nothing will be done during execution to complete the task ASAP.

4. But this is still not enough. Although the latency of the segcompaction task
has been shortened by aforemetioned method, tasks may still be queuing in the
thread pool.

[Solution]
We can increase the worker thread pool to avoid queuing congestion, but this is
not the best solution.
Segcompaction should be a best effort work, and should not use too many CPU and
memory resources. So we adopted the strategy of unbinding build and segcompaction,
specifically:

1. For the segcompaction task that is performing compaction operations, we should
not interrupt it, otherwise it may cause file corruption

2. For those tasks still queued, we no longer care about their results (because
these tasks will know they are cancelled and will not perform any actual operations),
so we just ignore them and continue with the subsequent rowset build process

Signed-off-by: freemandealer <freeman.zhang1992@gmail.com>
---
 be/src/olap/olap_server.cpp               | 15 +++++++++-----
 be/src/olap/rowset/beta_rowset_writer.cpp | 24 +++++++++++------------
 be/src/olap/rowset/beta_rowset_writer.h   |  4 ++--
 be/src/olap/rowset/segcompaction.cpp      | 15 +++++++++++---
 be/src/olap/rowset/segcompaction.h        |  6 ++++--
 be/src/olap/storage_engine.h              |  7 ++++---
 6 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 29aa4eb05fa099..9ee24dee19ab62 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -996,17 +996,22 @@ Status StorageEngine::submit_compaction_task(TabletSharedPtr tablet, CompactionT
     return _submit_compaction_task(tablet, compaction_type, force);
 }
 
-Status StorageEngine::_handle_seg_compaction(SegcompactionWorker* worker,
-                                             SegCompactionCandidatesSharedPtr segments) {
+Status StorageEngine::_handle_seg_compaction(std::shared_ptr<SegcompactionWorker> worker,
+                                             SegCompactionCandidatesSharedPtr segments,
+                                             uint64_t submission_time) {
+    // note: be aware that worker->_writer maybe released when the task is cancelled
+    uint64_t exec_queue_time = GetCurrentTimeMicros() - submission_time;
+    LOG(INFO) << "segcompaction thread pool queue time(ms): " << exec_queue_time / 1000;
     worker->compact_segments(segments);
     // return OK here. error will be reported via BetaRowsetWriter::_segcompaction_status
     return Status::OK();
 }
 
-Status StorageEngine::submit_seg_compaction_task(SegcompactionWorker* worker,
+Status StorageEngine::submit_seg_compaction_task(std::shared_ptr<SegcompactionWorker> worker,
                                                  SegCompactionCandidatesSharedPtr segments) {
-    return _seg_compaction_thread_pool->submit_func(
-            std::bind<void>(&StorageEngine::_handle_seg_compaction, this, worker, segments));
+    uint64_t submission_time = GetCurrentTimeMicros();
+    return _seg_compaction_thread_pool->submit_func(std::bind<void>(
+            &StorageEngine::_handle_seg_compaction, this, worker, segments, submission_time));
 }
 
 Status StorageEngine::process_index_change_task(const TAlterInvertedIndexReq& request) {
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index 6850ce0f43c6bd..ce30a03c800550 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -105,7 +105,7 @@ BaseBetaRowsetWriter::BaseBetaRowsetWriter()
           _total_index_size(0) {}
 
 BetaRowsetWriter::BetaRowsetWriter(StorageEngine& engine)
-        : _engine(engine), _segcompaction_worker(std::make_unique<SegcompactionWorker>(this)) {}
+        : _engine(engine), _segcompaction_worker(std::make_shared<SegcompactionWorker>(this)) {}
 
 BaseBetaRowsetWriter::~BaseBetaRowsetWriter() {
     // TODO(lingbin): Should wrapper exception logic, no need to know file ops directly.
@@ -381,14 +381,9 @@ Status BetaRowsetWriter::_rename_compacted_indices(int64_t begin, int64_t end, u
     return Status::OK();
 }
 
+// return true if there isn't any flying segcompaction, otherwise return false
 bool BetaRowsetWriter::_check_and_set_is_doing_segcompaction() {
-    std::lock_guard<std::mutex> l(_is_doing_segcompaction_lock);
-    if (!_is_doing_segcompaction) {
-        _is_doing_segcompaction = true;
-        return true;
-    } else {
-        return false;
-    }
+    return !_is_doing_segcompaction.exchange(true);
 }
 
 Status BetaRowsetWriter::_segcompaction_if_necessary() {
@@ -410,7 +405,7 @@ Status BetaRowsetWriter::_segcompaction_if_necessary() {
             LOG(INFO) << "submit segcompaction task, tablet_id:" << _context.tablet_id
                       << " rowset_id:" << _context.rowset_id << " segment num:" << _num_segment
                       << ", segcompacted_point:" << _segcompacted_point;
-            status = _engine.submit_seg_compaction_task(_segcompaction_worker.get(), segments);
+            status = _engine.submit_seg_compaction_task(_segcompaction_worker, segments);
             if (status.ok()) {
                 return status;
             }
@@ -546,9 +541,14 @@ Status BetaRowsetWriter::_close_file_writers() {
     // if _segment_start_id is not zero, that means it's a transient rowset writer for
     // MoW partial update, don't need to do segment compaction.
     if (_segment_start_id == 0) {
-        _segcompaction_worker->cancel();
-        RETURN_NOT_OK_STATUS_WITH_WARN(_wait_flying_segcompaction(),
-                                       "segcompaction failed when build new rowset");
+        if (_segcompaction_worker->cancel()) {
+            std::lock_guard lk(_is_doing_segcompaction_lock);
+            _is_doing_segcompaction = false;
+            _segcompacting_cond.notify_all();
+        } else {
+            RETURN_NOT_OK_STATUS_WITH_WARN(_wait_flying_segcompaction(),
+                                           "segcompaction failed when build new rowset");
+        }
         RETURN_NOT_OK_STATUS_WITH_WARN(_segcompaction_rename_last_segments(),
                                        "rename last segments failed when build new rowset");
         if (_segcompaction_worker->get_file_writer()) {
diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h
index 347994e243e7a4..0bd96f28291812 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -225,11 +225,11 @@ class BetaRowsetWriter : public BaseBetaRowsetWriter {
                                                   // already been segment compacted
     std::atomic<int32_t> _num_segcompacted {0};   // index for segment compaction
 
-    std::unique_ptr<SegcompactionWorker> _segcompaction_worker;
+    std::shared_ptr<SegcompactionWorker> _segcompaction_worker;
 
     // ensure only one inflight segcompaction task for each rowset
     std::atomic<bool> _is_doing_segcompaction {false};
-    // enforce compare-and-swap on _is_doing_segcompaction
+    // enforce condition variable on _is_doing_segcompaction
     std::mutex _is_doing_segcompaction_lock;
     std::condition_variable _segcompacting_cond;
 
diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp
index be92557da237d6..ad3e78a6081964 100644
--- a/be/src/olap/rowset/segcompaction.cpp
+++ b/be/src/olap/rowset/segcompaction.cpp
@@ -302,10 +302,12 @@ Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPt
 
 void SegcompactionWorker::compact_segments(SegCompactionCandidatesSharedPtr segments) {
     Status status = Status::OK();
-    if (_cancelled) {
-        LOG(INFO) << "segcompaction worker is cancelled, skipping segcompaction task";
-    } else {
+    if (_is_compacting_state_mutable.exchange(false)) {
         status = _do_compact_segments(segments);
+    } else {
+        // note: be aware that _writer maybe released when the task is cancelled
+        LOG(INFO) << "segcompaction worker is cancelled, skipping segcompaction task";
+        return;
     }
     if (!status.ok()) {
         int16_t errcode = status.code();
@@ -330,6 +332,13 @@ void SegcompactionWorker::compact_segments(SegCompactionCandidatesSharedPtr segm
         _writer->_is_doing_segcompaction = false;
         _writer->_segcompacting_cond.notify_all();
     }
+    _is_compacting_state_mutable = true;
+}
+
+bool SegcompactionWorker::cancel() {
+    // return true if the task is canncellable (actual compaction is not started)
+    // return false when the task is not cancellable (it is in the middle of segcompaction)
+    return _is_compacting_state_mutable.exchange(false);
 }
 
 } // namespace doris
diff --git a/be/src/olap/rowset/segcompaction.h b/be/src/olap/rowset/segcompaction.h
index e2b5812ad8c32e..5aef89992d30b8 100644
--- a/be/src/olap/rowset/segcompaction.h
+++ b/be/src/olap/rowset/segcompaction.h
@@ -54,7 +54,7 @@ class SegcompactionWorker {
     io::FileWriterPtr& get_file_writer() { return _file_writer; }
 
     // set the cancel flag, tasks already started will not be cancelled.
-    void cancel() { _cancelled = true; }
+    bool cancel();
 
 private:
     Status _create_segment_writer_for_segcompaction(
@@ -77,6 +77,8 @@ class SegcompactionWorker {
     // Currently cloud storage engine doesn't need segcompaction
     BetaRowsetWriter* _writer = nullptr;
     io::FileWriterPtr _file_writer;
-    std::atomic<bool> _cancelled = false;
+
+    // the state is not mutable when 1)actual compaction operation started or 2) cancelled
+    std::atomic<bool> _is_compacting_state_mutable = true;
 };
 } // namespace doris
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 77d3efeaaf8e69..750e7a4ca21cab 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -196,7 +196,7 @@ class StorageEngine {
 
     Status submit_compaction_task(TabletSharedPtr tablet, CompactionType compaction_type,
                                   bool force);
-    Status submit_seg_compaction_task(SegcompactionWorker* worker,
+    Status submit_seg_compaction_task(std::shared_ptr<SegcompactionWorker> worker,
                                       SegCompactionCandidatesSharedPtr segments);
 
     std::unique_ptr<ThreadPool>& tablet_publish_txn_thread_pool() {
@@ -313,8 +313,9 @@ class StorageEngine {
     void _remove_unused_remote_files_callback();
     void _cold_data_compaction_producer_callback();
 
-    Status _handle_seg_compaction(SegcompactionWorker* worker,
-                                  SegCompactionCandidatesSharedPtr segments);
+    Status _handle_seg_compaction(std::shared_ptr<SegcompactionWorker> worker,
+                                  SegCompactionCandidatesSharedPtr segments,
+                                  uint64_t submission_time);
 
     Status _handle_index_change(IndexBuilderSharedPtr index_builder);
 

From fe93a8f1d08b37926fd0c40ce338594674580c6b Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Thu, 28 Dec 2023 14:39:10 +0800
Subject: [PATCH 109/299] [cleanup](move-memtable) remove unused log in load
 stream stub (#29084)

---
 be/src/vec/sink/load_stream_stub.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 535e941b3b67cf..7835876596701e 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -44,9 +44,6 @@ int LoadStreamStub::LoadStreamReplyHandler::on_received_messages(brpc::StreamId
             for (auto tablet_id : response.success_tablet_ids()) {
                 ss << " " << tablet_id;
             }
-            if (response.success_tablet_ids_size() == 0) {
-                ss << " none";
-            }
             std::lock_guard<bthread::Mutex> lock(_success_tablets_mutex);
             for (auto tablet_id : response.success_tablet_ids()) {
                 _success_tablets.push_back(tablet_id);
@@ -57,9 +54,6 @@ int LoadStreamStub::LoadStreamReplyHandler::on_received_messages(brpc::StreamId
             for (auto tablet_id : response.failed_tablet_ids()) {
                 ss << " " << tablet_id;
             }
-            if (response.failed_tablet_ids_size() == 0) {
-                ss << " none";
-            }
             std::lock_guard<bthread::Mutex> lock(_failed_tablets_mutex);
             for (auto tablet_id : response.failed_tablet_ids()) {
                 _failed_tablets.push_back(tablet_id);

From 29a7c0d6775dd69fee91e93bb1f2cedced19f50f Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Thu, 28 Dec 2023 14:54:09 +0800
Subject: [PATCH 110/299] [pipelineX](scan) ignore storage data distribution by
 default (#29192)

---
 .../translator/PhysicalPlanTranslator.java    |    4 -
 .../apache/doris/planner/PlanFragment.java    |   10 -
 .../org/apache/doris/planner/ScanNode.java    |    1 -
 .../java/org/apache/doris/qe/Coordinator.java |   38 +-
 .../org/apache/doris/qe/SessionVariable.java  |    2 +-
 .../sql/q01.out                               |    7 +
 .../sql/q02.out                               |   47 +
 .../sql/q03.out                               |   13 +
 .../sql/q04.out                               |    8 +
 .../sql/q05.out                               |    8 +
 .../sql/q06.out                               |    4 +
 .../sql/q07.out                               |    7 +
 .../sql/q08.out                               |    5 +
 .../sql/q09.out                               |  178 ++
 .../sql/q10.out                               |   23 +
 .../sql/q11.out                               | 2544 +++++++++++++++
 .../sql/q12.out                               |    5 +
 .../sql/q13.out                               |   40 +
 .../sql/q14.out                               |    4 +
 .../sql/q15.out                               |    4 +
 .../sql/q16.out                               | 2765 +++++++++++++++++
 .../sql/q17.out                               |    4 +
 .../sql/q18.out                               |    8 +
 .../sql/q19.out                               |    4 +
 .../sql/q20.out                               |   12 +
 .../sql/q21.out                               |   50 +
 .../sql/q22.out                               |   10 +
 .../ddl/customer.sql                          |   17 +
 .../ddl/customer_delete.sql                   |    1 +
 .../ddl/lineitem.sql                          |   25 +
 .../ddl/lineitem_delete.sql                   |    1 +
 .../ddl/nation.sql                            |   13 +
 .../ddl/nation_delete.sql                     |    1 +
 .../ddl/orders.sql                            |   18 +
 .../ddl/orders_delete.sql                     |    1 +
 .../ddl/part.sql                              |   18 +
 .../ddl/part_delete.sql                       |    1 +
 .../ddl/partsupp.sql                          |   14 +
 .../ddl/partsupp_delete.sql                   |    1 +
 .../ddl/region.sql                            |   12 +
 .../ddl/region_delete.sql                     |    1 +
 .../ddl/revenue1.sql                          |   11 +
 .../ddl/revenue1_delete.sql                   |    1 +
 .../ddl/supplier.sql                          |   15 +
 .../ddl/supplier_delete.sql                   |    1 +
 .../load.groovy                               |   84 +
 .../sql/q01.sql                               |   22 +
 .../sql/q02.sql                               |   42 +
 .../sql/q03.sql                               |   24 +
 .../sql/q04.sql                               |   19 +
 .../sql/q05.sql                               |   25 +
 .../sql/q06.sql                               |   11 +
 .../sql/q07.sql                               |   39 +
 .../sql/q08.sql                               |   38 +
 .../sql/q09.sql                               |   32 +
 .../sql/q10.sql                               |   33 +
 .../sql/q11.sql                               |   28 +
 .../sql/q12.sql                               |   29 +
 .../sql/q13.sql                               |   21 +
 .../sql/q14.sql                               |   13 +
 .../sql/q15.sql                               |   20 +
 .../sql/q16.sql                               |   30 +
 .../sql/q17.sql                               |   16 +
 .../sql/q18.sql                               |   34 +
 .../sql/q19.sql                               |   35 +
 .../sql/q20.sql                               |   33 +
 .../sql/q21.sql                               |   39 +
 .../sql/q22.sql                               |   35 +
 68 files changed, 6630 insertions(+), 29 deletions(-)
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q01.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q02.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q03.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q04.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q05.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q06.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q07.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q08.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q09.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q10.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q11.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q12.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q13.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q14.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q15.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q16.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q17.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q18.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q19.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q20.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q21.out
 create mode 100644 regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q22.out
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier_delete.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/load.groovy
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q01.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q02.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q03.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q04.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q05.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q06.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q07.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q08.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q09.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q10.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q11.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q12.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q13.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q14.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q15.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q16.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q17.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q18.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q19.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q20.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q21.sql
 create mode 100644 regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q22.sql

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 06ab5e4fdbcedd..75d886a1dc7d6d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -865,10 +865,6 @@ public PlanFragment visitPhysicalHashAggregate(
             // Set colocate info in agg node. This is a hint for local shuffling to decide which type of
             // local exchanger will be used.
             aggregationNode.setColocate(true);
-
-            if (aggregate.getAggMode().isFinalPhase) {
-                inputPlanFragment.setHasColocateFinalizeAggNode(true);
-            }
         }
         setPlanRoot(inputPlanFragment, aggregationNode, aggregate);
         if (aggregate.getStats() != null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
index c128a2ecec7188..d95343d0e53627 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanFragment.java
@@ -149,8 +149,6 @@ public class PlanFragment extends TreeNode<PlanFragment> {
     // has colocate plan node
     private boolean hasColocatePlanNode = false;
 
-    private boolean hasColocateFinalizeAggNode = false;
-
     private boolean hasNullAwareLeftAntiJoin = false;
 
     private TResultSinkType resultSinkType = TResultSinkType.MYSQL_PROTOCAL;
@@ -475,14 +473,6 @@ public void setBucketNum(int bucketNum) {
         this.bucketNum = bucketNum;
     }
 
-    public boolean isHasColocateFinalizeAggNode() {
-        return hasColocateFinalizeAggNode;
-    }
-
-    public void setHasColocateFinalizeAggNode(boolean hasColocateFinalizeAggNode) {
-        this.hasColocateFinalizeAggNode = hasColocateFinalizeAggNode;
-    }
-
     public boolean isHasNullAwareLeftAntiJoin() {
         return hasNullAwareLeftAntiJoin;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
index ff744d52627877..d21fab7ac9a6d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java
@@ -724,7 +724,6 @@ public boolean ignoreStorageDataDistribution(ConnectContext context) {
         return !isKeySearch() && context != null
                 && context.getSessionVariable().isIgnoreStorageDataDistribution()
                 && context.getSessionVariable().getEnablePipelineXEngine()
-                && !fragment.isHasColocateFinalizeAggNode()
                 && !fragment.isHasNullAwareLeftAntiJoin();
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index bb7ea290cae8ed..2d4e5764aad93f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -2021,16 +2021,18 @@ private void computeFragmentHosts() throws Exception {
                             return scanNode.getId().asInt() == planNodeId;
                         }).findFirst();
 
-                        // disable shared scan optimization if one of conditions below is met:
-                        // 1. Use non-pipeline or pipelineX engine
-                        // 2. This fragment has a colocated scan node
-                        // 3. This fragment has a FileScanNode
-                        // 4. Disable shared scan optimization by session variable
+                        /**
+                         * Ignore storage data distribution iff:
+                         * 1. Current fragment is not forced to use data distribution.
+                         * 2. `parallelExecInstanceNum` is larger than scan ranges.
+                         * 3. Use Nereids planner.
+                         */
                         boolean sharedScan = true;
+                        int expectedInstanceNum = Math.min(parallelExecInstanceNum,
+                                leftMostNode.getNumInstances());
                         if (node.isPresent() && (!node.get().shouldDisableSharedScan(context)
-                                || (node.get().ignoreStorageDataDistribution(context) && useNereids))) {
-                            int expectedInstanceNum = Math.min(parallelExecInstanceNum,
-                                    leftMostNode.getNumInstances());
+                                || (node.get().ignoreStorageDataDistribution(context)
+                                && expectedInstanceNum > perNodeScanRanges.size() && useNereids))) {
                             expectedInstanceNum = Math.max(expectedInstanceNum, 1);
                             // if have limit and conjunts, only need 1 instance to save cpu and
                             // mem resource
@@ -2040,7 +2042,7 @@ private void computeFragmentHosts() throws Exception {
 
                             perInstanceScanRanges = Collections.nCopies(expectedInstanceNum, perNodeScanRanges);
                         } else {
-                            int expectedInstanceNum = 1;
+                            expectedInstanceNum = 1;
                             if (parallelExecInstanceNum > 1) {
                                 //the scan instance num should not larger than the tablets num
                                 expectedInstanceNum = Math.min(perNodeScanRanges.size(), parallelExecInstanceNum);
@@ -2859,10 +2861,6 @@ private void assignScanRanges(PlanFragmentId fragmentId, int parallelExecInstanc
         BucketSeqToScanRange bucketSeqToScanRange = fragmentIdBucketSeqToScanRangeMap.get(fragmentId);
         Set<Integer> scanNodeIds = fragmentIdToScanNodeIds.get(fragmentId);
 
-        boolean ignoreStorageDataDistribution = scanNodes.stream().filter(scanNode -> {
-            return scanNodeIds.contains(scanNode.getId().asInt());
-        }).allMatch(node -> node.ignoreStorageDataDistribution(context)) && useNereids;
-
         // 1. count each node in one fragment should scan how many tablet, gather them in one list
         Map<TNetworkAddress, List<Pair<Integer, Map<Integer, List<TScanRangeParams>>>>> addressToScanRanges
                 = Maps.newHashMap();
@@ -2885,6 +2883,20 @@ private void assignScanRanges(PlanFragmentId fragmentId, int parallelExecInstanc
             }
             addressToScanRanges.get(address).add(filteredScanRanges);
         }
+
+        /**
+         * Ignore storage data distribution iff:
+         * 1. Current fragment is not forced to use data distribution.
+         * 2. `parallelExecInstanceNum` is larger than scan ranges.
+         * 3. Use Nereids planner.
+         */
+        boolean ignoreStorageDataDistribution = scanNodes.stream().filter(scanNode -> {
+            return scanNodeIds.contains(scanNode.getId().asInt());
+        }).allMatch(node -> node.ignoreStorageDataDistribution(context))
+                && addressToScanRanges.entrySet().stream().allMatch(addressScanRange -> {
+                    return addressScanRange.getValue().size() < parallelExecInstanceNum;
+                }) && useNereids;
+
         FragmentScanRangeAssignment assignment = params.scanRangeAssignment;
         for (Map.Entry<TNetworkAddress, List<Pair<Integer, Map<Integer, List<TScanRangeParams>>>>> addressScanRange
                 : addressToScanRanges.entrySet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index c1ea2f29ff292f..02402e69c70150 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -808,7 +808,7 @@ public class SessionVariable implements Serializable, Writable {
 
     @VariableMgr.VarAttr(name = IGNORE_STORAGE_DATA_DISTRIBUTION, fuzzy = false,
             varType = VariableAnnotation.EXPERIMENTAL, needForward = true)
-    private boolean ignoreStorageDataDistribution = false;
+    private boolean ignoreStorageDataDistribution = true;
 
     @VariableMgr.VarAttr(
             name = ENABLE_LOCAL_SHUFFLE, fuzzy = false, varType = VariableAnnotation.EXPERIMENTAL,
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q01.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q01.out
new file mode 100644
index 00000000000000..dbfc153a72976d
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q01.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q01 --
+A	F	3774200.00	5320753880.69	5054096266.6828	5256751331.449234	25.5375	36002.1238	0.0501	147790
+N	F	95257.00	133737795.84	127132372.6512	132286291.229445	25.3006	35521.3269	0.0493	3765
+N	O	7459297.00	10512270008.90	9986238338.3847	10385578376.585467	25.5455	36000.9246	0.0500	292000
+R	F	3785523.00	5337950526.47	5071818532.9420	5274405503.049367	25.5259	35994.0292	0.0499	148301
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q02.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q02.out
new file mode 100644
index 00000000000000..29617e6969104a
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q02.out
@@ -0,0 +1,47 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q02 --
+9828.21	Supplier#000000647	UNITED KINGDOM	13120	Manufacturer#5	x5U7MBZmwfG9	33-258-202-4782	s the slyly even ideas poach fluffily 
+9508.37	Supplier#000000070	FRANCE	3563	Manufacturer#1	INWNH2w,OOWgNDq0BRCcBwOMQc6PdFDc4	16-821-608-1166	ests sleep quickly express ideas. ironic ideas haggle about the final T
+9508.37	Supplier#000000070	FRANCE	17268	Manufacturer#4	INWNH2w,OOWgNDq0BRCcBwOMQc6PdFDc4	16-821-608-1166	ests sleep quickly express ideas. ironic ideas haggle about the final T
+9453.01	Supplier#000000802	ROMANIA	10021	Manufacturer#5	,6HYXb4uaHITmtMBj4Ak57Pd	29-342-882-6463	gular frets. permanently special multipliers believe blithely alongs
+9453.01	Supplier#000000802	ROMANIA	13275	Manufacturer#4	,6HYXb4uaHITmtMBj4Ak57Pd	29-342-882-6463	gular frets. permanently special multipliers believe blithely alongs
+9192.10	Supplier#000000115	UNITED KINGDOM	13325	Manufacturer#1	nJ 2t0f7Ve,wL1,6WzGBJLNBUCKlsV	33-597-248-1220	es across the carefully express accounts boost caref
+9032.15	Supplier#000000959	GERMANY	4958	Manufacturer#4	8grA EHBnwOZhO	17-108-642-3106	nding dependencies nag furiou
+8702.02	Supplier#000000333	RUSSIA	11810	Manufacturer#3	MaVf XgwPdkiX4nfJGOis8Uu2zKiIZH	32-508-202-6136	oss the deposits cajole carefully even pinto beans. regular foxes detect alo
+8615.50	Supplier#000000812	FRANCE	10551	Manufacturer#2	8qh4tezyScl5bidLAysvutB,,ZI2dn6xP	16-585-724-6633	y quickly regular deposits? quickly pending packages after the caref
+8615.50	Supplier#000000812	FRANCE	13811	Manufacturer#4	8qh4tezyScl5bidLAysvutB,,ZI2dn6xP	16-585-724-6633	y quickly regular deposits? quickly pending packages after the caref
+8488.53	Supplier#000000367	RUSSIA	6854	Manufacturer#4	E Sv9brQVf43Mzz	32-458-198-9557	ages. carefully final excuses nag finally. carefully ironic deposits abov
+8430.52	Supplier#000000646	FRANCE	11384	Manufacturer#3	IUzsmT,2oBgjhWP2TlXTL6IkJH,4h,1SJRt	16-601-220-5489	ites among the always final ideas kindle according to the theodolites. notornis in
+8271.39	Supplier#000000146	RUSSIA	4637	Manufacturer#5	rBDNgCr04x0sfdzD5,gFOutCiG2	32-792-619-3155	s cajole quickly special requests. quickly enticing theodolites h
+8096.98	Supplier#000000574	RUSSIA	323	Manufacturer#4	2O8 sy9g2mlBOuEjzj0pA2pevk,	32-866-246-8752	ully after the regular requests. slyly final dependencies wake slyly along the busy deposit
+7392.78	Supplier#000000170	UNITED KINGDOM	7655	Manufacturer#2	RtsXQ,SunkA XHy9	33-803-340-5398	ake carefully across the quickly
+7205.20	Supplier#000000477	GERMANY	10956	Manufacturer#5	VtaNKN5Mqui5yh7j2ldd5waf	17-180-144-7991	 excuses wake express deposits. furiously careful asymptotes according to the carefull
+6820.35	Supplier#000000007	UNITED KINGDOM	13217	Manufacturer#5	s,4TicNGB4uO6PaSqNBUq	33-990-965-2201	s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit
+6721.70	Supplier#000000954	FRANCE	4191	Manufacturer#3	P3O5p UFz1QsLmZX	16-537-341-8517	ect blithely blithely final acco
+6329.90	Supplier#000000996	GERMANY	10735	Manufacturer#2	Wx4dQwOAwWjfSCGupfrM	17-447-811-3282	 ironic forges cajole blithely agai
+6173.87	Supplier#000000408	RUSSIA	18139	Manufacturer#1	qcor1u,vJXAokjnL5,dilyYNmh	32-858-724-2950	blithely pending packages cajole furiously slyly pending notornis. slyly final 
+5364.99	Supplier#000000785	RUSSIA	13784	Manufacturer#4	W VkHBpQyD3qjQjWGpWicOpmILFehmEdWy67kUGY	32-297-653-2203	 packages boost carefully. express ideas along
+5069.27	Supplier#000000328	GERMANY	16327	Manufacturer#1	SMm24d WG62	17-231-513-5721	he unusual ideas. slyly final packages a
+4941.88	Supplier#000000321	ROMANIA	7320	Manufacturer#5	pLngFl5yeMcHyov	29-573-279-1406	y final requests impress s
+4672.25	Supplier#000000239	RUSSIA	12238	Manufacturer#1	XO101kgHrJagK2FL1U6QCaTE ncCsMbeuTgK6o8	32-396-654-6826	arls wake furiously deposits. even, regular depen
+4586.49	Supplier#000000680	RUSSIA	5679	Manufacturer#3	UhvDfdEfJh,Qbe7VZb8uSGO2TU 0jEa6nXZXE	32-522-382-1620	 the regularly regular dependencies. carefully bold excuses under th
+4518.31	Supplier#000000149	FRANCE	18344	Manufacturer#5	pVyWsjOidpHKp4NfKU4yLeym	16-660-553-2456	ts detect along the foxes. final Tiresias are. idly pending deposits haggle; even, blithe pin
+4315.15	Supplier#000000509	FRANCE	18972	Manufacturer#2	SF7dR8V5pK	16-298-154-3365	ronic orbits are furiously across the requests. quickly express ideas across the special, bold
+3526.53	Supplier#000000553	FRANCE	8036	Manufacturer#4	a,liVofXbCJ	16-599-552-3755	lar dinos nag slyly brave
+3526.53	Supplier#000000553	FRANCE	17018	Manufacturer#3	a,liVofXbCJ	16-599-552-3755	lar dinos nag slyly brave
+3294.68	Supplier#000000350	GERMANY	4841	Manufacturer#4	KIFxV73eovmwhh	17-113-181-4017	e slyly special foxes. furiously unusual deposits detect carefully carefully ruthless foxes. quick
+2972.26	Supplier#000000016	RUSSIA	1015	Manufacturer#4	YjP5C55zHDXL7LalK27zfQnwejdpin4AMpvh	32-822-502-4215	ously express ideas haggle quickly dugouts? fu
+2963.09	Supplier#000000840	ROMANIA	3080	Manufacturer#2	iYzUIypKhC0Y	29-781-337-5584	eep blithely regular dependencies. blithely regular platelets sublate alongside o
+2221.25	Supplier#000000771	ROMANIA	13981	Manufacturer#2	lwZ I15rq9kmZXUNhl	29-986-304-9006	nal foxes eat slyly about the fluffily permanent id
+1381.97	Supplier#000000104	FRANCE	18103	Manufacturer#3	Dcl4yGrzqv3OPeRO49bKh78XmQEDR7PBXIs0m	16-434-972-6922	gular ideas. bravely bold deposits haggle through the carefully final deposits. slyly unusual idea
+906.07	Supplier#000000138	ROMANIA	8363	Manufacturer#4	utbplAm g7RmxVfYoNdhcrQGWuzRqPe0qHSwbKw	29-533-434-6776	ickly unusual requests cajole. accounts above the furiously special excuses 
+765.69	Supplier#000000799	RUSSIA	11276	Manufacturer#2	jwFN7ZB3T9sMF	32-579-339-1495	nusual requests. furiously unusual epitaphs integrate. slyly 
+727.89	Supplier#000000470	ROMANIA	6213	Manufacturer#3	XckbzsAgBLbUkdfjgJEPjmUMTM8ebSMEvI	29-165-289-1523	gular excuses. furiously regular excuses sleep slyly caref
+683.07	Supplier#000000651	RUSSIA	4888	Manufacturer#4	oWekiBV6s,1g	32-181-426-4490	ly regular requests cajole abou
+167.56	Supplier#000000290	FRANCE	2037	Manufacturer#1	6Bk06GVtwZaKqg01	16-675-286-5102	 the theodolites. ironic, ironic deposits above 
+91.39	Supplier#000000949	UNITED KINGDOM	9430	Manufacturer#2	a,UE,6nRVl2fCphkOoetR1ajIzAEJ1Aa1G1HV	33-332-697-2768	pinto beans. carefully express requests hagg
+-314.06	Supplier#000000510	ROMANIA	17242	Manufacturer#4	VmXQl ,vY8JiEseo8Mv4zscvNCfsY	29-207-852-3454	 bold deposits. carefully even d
+-820.89	Supplier#000000409	GERMANY	2156	Manufacturer#5	LyXUYFz7aXrvy65kKAbTatGzGS,NDBcdtD	17-719-517-9836	y final, slow theodolites. furiously regular req
+-845.44	Supplier#000000704	ROMANIA	9926	Manufacturer#5	hQvlBqbqqnA5Dgo1BffRBX78tkkRu	29-300-896-5991	ctions. carefully sly requ
+-942.73	Supplier#000000563	GERMANY	5797	Manufacturer#1	Rc7U1cRUhYs03JD	17-108-537-2691	slyly furiously final decoys; silent, special realms poach f
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q03.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q03.out
new file mode 100644
index 00000000000000..58f0f989593615
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q03.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q03 --
+223140	355369.0698	1995-03-14	0
+584291	354494.7318	1995-02-21	0
+405063	353125.4577	1995-03-03	0
+573861	351238.2770	1995-03-09	0
+554757	349181.7426	1995-03-14	0
+506021	321075.5810	1995-03-10	0
+121604	318576.4154	1995-03-07	0
+108514	314967.0754	1995-02-20	0
+462502	312604.5420	1995-03-08	0
+178727	309728.9306	1995-02-25	0
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q04.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q04.out
new file mode 100644
index 00000000000000..b93c53211d4594
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q04.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q04 --
+1-URGENT	999
+2-HIGH	997
+3-MEDIUM	1031
+4-NOT SPECIFIED	989
+5-LOW	1077
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q05.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q05.out
new file mode 100644
index 00000000000000..061f8e49294748
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q05.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q05 --
+CHINA	7822103.0000
+INDIA	6376121.5085
+JAPAN	6000077.2184
+INDONESIA	5580475.4027
+VIETNAM	4497840.5466
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q06.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q06.out
new file mode 100644
index 00000000000000..f4a341a7582f1c
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q06.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q06 --
+11803420.2534
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q07.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q07.out
new file mode 100644
index 00000000000000..25628ec4758850
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q07.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q07 --
+FRANCE	GERMANY	1995	4637235.1501
+FRANCE	GERMANY	1996	5224779.5736
+GERMANY	FRANCE	1995	6232818.7037
+GERMANY	FRANCE	1996	5557312.1121
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q08.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q08.out
new file mode 100644
index 00000000000000..e02fa19a52f56b
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q08.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q08 --
+1995	0.02864874
+1996	0.01825027
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q09.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q09.out
new file mode 100644
index 00000000000000..416ac4dfd3f914
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q09.out
@@ -0,0 +1,178 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q09 --
+ALGERIA	1998	2321785.3682
+ALGERIA	1997	3685016.8589
+ALGERIA	1996	4276597.4253
+ALGERIA	1995	4418370.4154
+ALGERIA	1994	3864849.9521
+ALGERIA	1993	3541051.3865
+ALGERIA	1992	4310013.3482
+ARGENTINA	1998	2685983.8005
+ARGENTINA	1997	4242147.8124
+ARGENTINA	1996	3907867.0103
+ARGENTINA	1995	4605921.5011
+ARGENTINA	1994	3542096.1564
+ARGENTINA	1993	3949965.9388
+ARGENTINA	1992	4521180.4695
+BRAZIL	1998	2778730.3931
+BRAZIL	1997	4642037.4687
+BRAZIL	1996	4530304.6034
+BRAZIL	1995	4502344.8657
+BRAZIL	1994	4875806.5015
+BRAZIL	1993	4687478.6531
+BRAZIL	1992	5035200.0464
+CANADA	1998	2194509.0465
+CANADA	1997	3482197.9521
+CANADA	1996	3712231.2814
+CANADA	1995	4014814.8476
+CANADA	1994	4145304.4855
+CANADA	1993	3787069.6045
+CANADA	1992	4168009.4201
+CHINA	1998	3398578.0001
+CHINA	1997	6358959.3338
+CHINA	1996	6435158.3229
+CHINA	1995	6174776.2113
+CHINA	1994	6385751.0812
+CHINA	1993	5765034.1194
+CHINA	1992	6324034.2379
+EGYPT	1998	2333148.3334
+EGYPT	1997	3661244.2731
+EGYPT	1996	3765371.2368
+EGYPT	1995	4094744.2925
+EGYPT	1994	3566508.0818
+EGYPT	1993	3725283.7747
+EGYPT	1992	3373762.3335
+ETHIOPIA	1998	1953927.2682
+ETHIOPIA	1997	3285786.3266
+ETHIOPIA	1996	3525028.7952
+ETHIOPIA	1995	3781674.8911
+ETHIOPIA	1994	3037409.4360
+ETHIOPIA	1993	3008978.2677
+ETHIOPIA	1992	2721203.2355
+FRANCE	1998	2604373.8805
+FRANCE	1997	3982872.0488
+FRANCE	1996	3622479.2413
+FRANCE	1995	4479939.7020
+FRANCE	1994	3531013.1981
+FRANCE	1993	4086437.3102
+FRANCE	1992	3637792.1333
+GERMANY	1998	3291023.2965
+GERMANY	1997	5139337.3443
+GERMANY	1996	4799810.4577
+GERMANY	1995	5405785.7978
+GERMANY	1994	4555556.4592
+GERMANY	1993	4428195.1019
+GERMANY	1992	4656148.4204
+INDIA	1998	2591288.1874
+INDIA	1997	5159562.7033
+INDIA	1996	5307258.3049
+INDIA	1995	5148208.7902
+INDIA	1994	5164001.9582
+INDIA	1993	4321398.4388
+INDIA	1992	5297703.6935
+INDONESIA	1998	3094900.1597
+INDONESIA	1997	5719773.0358
+INDONESIA	1996	6037238.5993
+INDONESIA	1995	5266783.4899
+INDONESIA	1994	5470762.8729
+INDONESIA	1993	6189826.6613
+INDONESIA	1992	4414623.1549
+IRAN	1998	3214864.1209
+IRAN	1997	3688049.0691
+IRAN	1996	3621649.2247
+IRAN	1995	4420783.4205
+IRAN	1994	4373984.6523
+IRAN	1993	3731301.7814
+IRAN	1992	4417133.3662
+IRAQ	1998	2338859.4099
+IRAQ	1997	3622681.5643
+IRAQ	1996	4762291.8722
+IRAQ	1995	4558092.7359
+IRAQ	1994	4951604.1699
+IRAQ	1993	3830077.9911
+IRAQ	1992	3938636.4874
+JAPAN	1998	1849535.0802
+JAPAN	1997	4068688.8537
+JAPAN	1996	4044774.7597
+JAPAN	1995	4793005.8027
+JAPAN	1994	4114717.0568
+JAPAN	1993	3614468.7485
+JAPAN	1992	4266694.4700
+JORDAN	1998	1811488.0719
+JORDAN	1997	2951297.8678
+JORDAN	1996	3302528.3067
+JORDAN	1995	3221813.9990
+JORDAN	1994	2417892.0921
+JORDAN	1993	3107641.7661
+JORDAN	1992	3316379.0585
+KENYA	1998	2579075.4190
+KENYA	1997	2929194.2317
+KENYA	1996	3569129.5619
+KENYA	1995	3542889.1087
+KENYA	1994	3983095.3994
+KENYA	1993	3713988.9708
+KENYA	1992	3304641.8340
+MOROCCO	1998	1815334.8180
+MOROCCO	1997	3693214.8447
+MOROCCO	1996	4116175.9230
+MOROCCO	1995	3515127.1402
+MOROCCO	1994	4003072.1120
+MOROCCO	1993	3599199.6679
+MOROCCO	1992	3958335.4224
+MOZAMBIQUE	1998	1620428.7346
+MOZAMBIQUE	1997	2802166.6473
+MOZAMBIQUE	1996	2409955.1755
+MOZAMBIQUE	1995	2771602.6274
+MOZAMBIQUE	1994	2548226.2158
+MOZAMBIQUE	1993	2843748.9053
+MOZAMBIQUE	1992	2556501.0943
+PERU	1998	2036430.3602
+PERU	1997	4064142.4091
+PERU	1996	4068678.5671
+PERU	1995	4657694.8412
+PERU	1994	4731959.4655
+PERU	1993	4144006.6610
+PERU	1992	3754635.0078
+ROMANIA	1998	1992773.6811
+ROMANIA	1997	2854639.8680
+ROMANIA	1996	3139337.3029
+ROMANIA	1995	3222153.3776
+ROMANIA	1994	3222844.3190
+ROMANIA	1993	3488994.0288
+ROMANIA	1992	3029274.4420
+RUSSIA	1998	2339865.6635
+RUSSIA	1997	4153619.5424
+RUSSIA	1996	3772067.4041
+RUSSIA	1995	4704988.8607
+RUSSIA	1994	4479082.8694
+RUSSIA	1993	4767719.9791
+RUSSIA	1992	4533465.5590
+SAUDI ARABIA	1998	3386948.9564
+SAUDI ARABIA	1997	5425980.3373
+SAUDI ARABIA	1996	5227607.1677
+SAUDI ARABIA	1995	4506731.6411
+SAUDI ARABIA	1994	4698658.7425
+SAUDI ARABIA	1993	5493626.5285
+SAUDI ARABIA	1992	4573560.0150
+UNITED KINGDOM	1998	2252021.5137
+UNITED KINGDOM	1997	4343926.8026
+UNITED KINGDOM	1996	4189476.3065
+UNITED KINGDOM	1995	4469569.8829
+UNITED KINGDOM	1994	4410094.6264
+UNITED KINGDOM	1993	4054677.1050
+UNITED KINGDOM	1992	3978688.8831
+UNITED STATES	1998	2238771.5581
+UNITED STATES	1997	4135581.5734
+UNITED STATES	1996	3624013.2660
+UNITED STATES	1995	3892244.5172
+UNITED STATES	1994	3289224.1138
+UNITED STATES	1993	3626170.2028
+UNITED STATES	1992	3993973.4997
+VIETNAM	1998	1924313.4862
+VIETNAM	1997	3436195.3709
+VIETNAM	1996	4017288.8927
+VIETNAM	1995	3644054.1372
+VIETNAM	1994	4141277.6665
+VIETNAM	1993	2556114.1693
+VIETNAM	1992	4090524.4905
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q10.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q10.out
new file mode 100644
index 00000000000000..7f9b9301115a3b
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q10.out
@@ -0,0 +1,23 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q10 --
+8242	Customer#000008242	622786.7297	6322.09	ETHIOPIA	P2n4nJhy,UqSo2s43YfSvYJDZ6lk	15-792-676-1184	slyly regular packages haggle carefully ironic ideas. courts are furiously. furiously unusual theodolites cajole. i
+7714	Customer#000007714	557400.3053	9799.98	IRAN	SnnIGB,SkmnWpX3	20-922-418-6024	arhorses according to the blithely express re
+11032	Customer#000011032	512500.9641	8496.93	UNITED KINGDOM	WIKHC7K3Cn7156iNOyfVG3cZ7YqkgsR,Ly	33-102-772-3533	posits-- furiously ironic accounts are again
+2455	Customer#000002455	481592.4053	2070.99	GERMANY	RVn1ZSRtLqPlJLIZxvpmsbgC02	17-946-225-9977	al asymptotes. finally ironic accounts cajole furiously. permanently unusual theodolites aro
+12106	Customer#000012106	479414.2133	5342.11	UNITED STATES	wth3twOmu6vy	34-905-346-4472	ly after the blithely regular foxes. accounts haggle carefully alongside of the blithely even ideas.
+8530	Customer#000008530	457855.9467	9734.95	MOROCCO	GMQyte94oDM7eD7exnkj 4hH9yq3	25-736-932-5850	slyly asymptotes. quickly final deposits in
+13984	Customer#000013984	446316.5104	3482.28	IRAN	qZXwuapCHvxbX	20-981-264-2952	y unusual courts could wake furiously
+1966	Customer#000001966	444059.0382	1937.72	ALGERIA	jPv1 UHra5JLALR5Isci5u0636RoAu7t vH	10-973-269-8886	the blithely even accounts. final deposits cajole around the blithely final packages. 
+11026	Customer#000011026	417913.4142	7738.76	ALGERIA	XorIktoJOAEJkpNNMx	10-184-163-4632	ly even dolphins eat along the blithely even instructions. express attainments cajole slyly. busy dolphins in
+8501	Customer#000008501	412797.5100	6906.70	ARGENTINA	776af4rOa mZ66hczs	11-317-552-5840	y final deposits after the fluffily even accounts are slyly final, regular
+1565	Customer#000001565	412506.0062	1820.03	BRAZIL	EWQO5Ck,nMuHVQimqL8dLrixRP6QKveXcz9QgorW	12-402-178-2007	ously regular accounts wake slyly ironic idea
+14398	Customer#000014398	408575.3600	-602.24	UNITED STATES	GWRCgIPHajtU21vICVvbJJerFu2cUk	34-814-111-5424	s. blithely even accounts cajole blithely. even foxes doubt-- 
+1465	Customer#000001465	405055.3457	9365.93	INDIA	tDRaTC7UgFbBX7VF6cVXYQA0	18-807-487-1074	s lose blithely ironic, regular packages. regular, final foxes haggle c
+12595	Customer#000012595	401402.2391	-6.92	INDIA	LmeaX5cR,w9NqKugl yRm98	18-186-132-3352	o the busy accounts. blithely special gifts maintain a
+961	Customer#000000961	401198.1737	6963.68	JAPAN	5,81YDLFuRR47KKzv8GXdmi3zyP37PlPn	22-989-463-6089	e final requests: busily final accounts believe a
+14299	Customer#000014299	400968.3751	6595.97	RUSSIA	7lFczTya0iM1bhEWT	32-156-618-1224	 carefully regular requests. quickly ironic accounts against the ru
+623	Customer#000000623	399883.4257	7887.60	INDONESIA	HXiFb9oWlgqZXrJPUCEJ6zZIPxAM4m6	19-113-202-7085	 requests. dolphins above the busily regular dependencies cajole after
+9151	Customer#000009151	396562.0295	5691.95	IRAQ	7gIdRdaxB91EVdyx8DyPjShpMD	21-834-147-4906	ajole fluffily. furiously regular accounts are special, silent account
+14819	Customer#000014819	396271.1036	7308.39	FRANCE	w8StIbymUXmLCcUag6sx6LUIp8E3pA,Ux	16-769-398-7926	ss, final asymptotes use furiously slyly ironic dependencies. special, express dugouts according to the dep
+13478	Customer#000013478	395513.1358	-778.11	KENYA	9VIsvIeZrJpC6OOdYheMC2vdtq8Ai0Rt	24-983-202-8240	r theodolites. slyly unusual pinto beans sleep fluffily against the asymptotes. quickly r
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q11.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q11.out
new file mode 100644
index 00000000000000..f4ddb2a26e1b0a
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q11.out
@@ -0,0 +1,2544 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q11 --
+12098	16227681.21
+5134	15709338.52
+13334	15023662.41
+17052	14351644.20
+3452	14070870.14
+12552	13332469.18
+1084	13170428.29
+5797	13038622.72
+12633	12892561.61
+403	12856217.34
+1833	12024581.72
+2084	11502875.36
+17349	11354213.05
+18427	11282385.24
+2860	11262529.95
+17852	10934711.93
+9871	10889253.68
+12231	10841131.39
+6366	10759786.81
+12146	10257362.66
+5043	10226395.88
+12969	10125777.93
+1504	10004397.08
+14327	9981697.08
+134	9965150.66
+6860	9805871.26
+10624	9776138.40
+15819	9775705.31
+3293	9674928.12
+19865	9653766.83
+8870	9648981.87
+15778	9636332.82
+12360	9635023.92
+14389	9475588.34
+3257	9451029.24
+9476	9435207.28
+19629	9391236.40
+7179	9386222.25
+15723	9383900.80
+4054	9313810.02
+2380	9307751.22
+19084	9302916.80
+4703	9280804.80
+18791	9267017.97
+19994	9235972.92
+9149	9121803.90
+15118	9120819.50
+6116	9079369.20
+7052	9077468.92
+14147	9069193.78
+7305	9035228.53
+9130	9024379.25
+16698	8991337.95
+1553	8977226.10
+16777	8961355.62
+1402	8953779.12
+18963	8934063.40
+8358	8930611.48
+17547	8860117.00
+5128	8844222.75
+17063	8840649.60
+15490	8833581.40
+14761	8817240.56
+19601	8791341.02
+16160	8740262.76
+13597	8702669.82
+13653	8693170.16
+16383	8691505.92
+325	8667741.28
+8879	8667584.38
+10564	8667098.22
+17429	8661827.90
+17403	8643350.30
+18294	8616583.43
+4181	8592684.66
+13008	8567480.64
+13211	8537000.01
+1884	8532644.34
+11101	8530945.32
+11562	8528028.57
+15878	8523591.84
+834	8522135.27
+2423	8517902.85
+15383	8513433.11
+18119	8507611.80
+7389	8506099.20
+5016	8489784.15
+17473	8444766.24
+6669	8428618.46
+384	8418472.27
+12052	8411519.28
+17562	8409022.83
+8128	8379149.47
+13813	8374830.84
+12800	8318626.78
+10887	8315019.36
+1644	8285453.08
+16638	8274568.00
+1394	8255140.60
+7219	8254985.30
+13358	8253829.80
+5562	8252365.16
+14861	8242296.15
+15416	8196621.53
+1963	8192206.61
+2841	8148678.45
+6635	8122918.28
+3609	8099812.05
+6372	8093695.06
+5231	8091704.18
+8755	8085017.03
+4071	8083755.18
+4977	8058501.72
+11873	8057540.30
+12127	8051142.94
+2401	8049525.94
+15964	8037547.75
+10129	8030855.04
+7605	8028622.42
+9327	8022291.30
+11814	7983589.59
+4646	7981660.60
+6032	7981511.28
+1076	7977074.75
+4043	7971929.90
+8932	7967222.19
+13705	7953344.52
+16855	7923819.00
+3234	7920022.00
+17752	7901362.77
+2097	7892993.17
+18599	7890774.40
+19596	7874564.59
+11392	7861172.48
+18121	7857581.60
+17452	7838465.36
+6076	7821316.80
+15134	7804760.25
+8641	7802917.15
+2134	7800185.43
+16352	7797112.00
+19312	7775952.24
+2895	7759327.23
+12860	7758135.21
+153	7755681.28
+15089	7735438.26
+14797	7725353.16
+15946	7722773.88
+10919	7722425.36
+9867	7721597.78
+11881	7713136.42
+16552	7708518.15
+6925	7703999.68
+12147	7703826.98
+8923	7702690.28
+16116	7697970.84
+10661	7673830.20
+17094	7665368.16
+18648	7650862.02
+12172	7641326.40
+15123	7633032.50
+4993	7600570.80
+17162	7592062.56
+13506	7580809.83
+3436	7575616.33
+6271	7559793.93
+13314	7555156.63
+17242	7550949.50
+2753	7549574.06
+7391	7543159.68
+7418	7541449.65
+116	7520874.24
+12436	7520234.54
+1181	7494798.28
+12963	7491248.10
+213	7479470.28
+7114	7477681.20
+18521	7476478.30
+8973	7458603.67
+4202	7454095.74
+12009	7442105.40
+10609	7429346.40
+5622	7424142.66
+5143	7422760.28
+898	7414133.80
+12257	7408190.63
+6740	7400350.35
+1146	7394394.48
+5485	7378181.94
+8437	7376353.59
+6531	7362366.78
+16463	7362106.80
+10412	7359552.42
+12857	7340801.58
+12354	7332343.20
+7616	7320032.37
+3426	7312340.41
+8622	7307266.33
+6818	7304782.10
+3705	7299380.10
+12733	7298398.17
+1574	7293446.04
+10554	7289933.72
+9363	7284647.70
+4611	7282115.00
+7857	7266948.92
+9616	7265005.30
+15860	7254466.66
+15554	7247592.51
+3422	7247586.45
+9134	7236404.67
+17408	7220083.60
+15853	7219984.90
+9266	7218517.11
+1881	7208346.45
+10148	7205335.83
+8860	7202401.44
+8373	7189039.50
+10859	7188990.69
+12670	7188177.22
+2553	7180006.02
+19346	7176797.25
+1681	7160169.84
+15225	7158861.32
+1052	7158586.00
+77	7155531.10
+7231	7155250.38
+12622	7137408.42
+9814	7105363.14
+8695	7103187.00
+13174	7099182.53
+6179	7095134.05
+6451	7082495.36
+19860	7073206.83
+9307	7059973.68
+7819	7055963.04
+10556	7053491.07
+9366	7048690.74
+12124	7040021.31
+11476	7037906.76
+19245	7034045.24
+7562	7030275.79
+12290	7020372.06
+18118	7003396.80
+1253	7001569.62
+8662	6999834.27
+1779	6997385.73
+15386	6996871.79
+147	6989079.88
+9562	6983076.90
+3958	6969833.22
+7211	6966606.67
+12842	6923277.66
+4368	6918783.95
+11131	6918656.78
+4628	6894893.82
+5879	6881367.14
+16586	6865931.80
+32	6852925.59
+12119	6842773.70
+1371	6831137.52
+6136	6827917.01
+13857	6824240.60
+1074	6821747.88
+1863	6821522.19
+14597	6817385.66
+9271	6783068.88
+6389	6781075.68
+11703	6776538.36
+13701	6768880.56
+14880	6763788.24
+18428	6763670.54
+1006	6762065.94
+4927	6756765.21
+11659	6755246.68
+6815	6738928.35
+14367	6733857.20
+10703	6730936.46
+3150	6727920.40
+9963	6725919.35
+10438	6710153.62
+4745	6682153.67
+15297	6681711.28
+10848	6678666.25
+11749	6677895.73
+16739	6675549.12
+14915	6672248.66
+19841	6669191.20
+628	6666892.90
+1650	6657178.32
+7069	6648672.24
+7108	6646445.96
+8887	6641655.90
+18735	6636457.56
+3711	6632665.38
+2325	6630475.92
+6484	6622965.20
+2643	6617304.98
+7895	6615061.26
+12840	6604848.54
+4314	6600905.71
+19077	6591091.97
+17476	6576029.00
+7039	6559838.90
+8106	6558223.02
+2391	6557120.08
+7717	6547706.88
+12476	6546988.15
+9155	6540793.60
+360	6530297.41
+6383	6529336.02
+4830	6518998.92
+12600	6511549.46
+13740	6508057.92
+4678	6507847.98
+17815	6502284.76
+7329	6490811.95
+13884	6490063.10
+5147	6487069.00
+16548	6482024.50
+14144	6476413.40
+10181	6474984.88
+11031	6463308.02
+19958	6461506.38
+6043	6458177.64
+10060	6455476.89
+9144	6454042.05
+7043	6448019.98
+17346	6444307.52
+13963	6442014.48
+7111	6441947.07
+14140	6439955.54
+2327	6438977.28
+14812	6437152.54
+16755	6430895.36
+14840	6430549.14
+14134	6422079.26
+12655	6405496.79
+1518	6390148.22
+9888	6385033.02
+7387	6384005.18
+5393	6381083.04
+11057	6375974.22
+16818	6368828.80
+4576	6364925.71
+19644	6350000.33
+139	6336065.55
+11735	6334305.90
+10108	6332055.99
+15628	6329801.34
+4349	6314949.68
+7048	6313868.55
+17119	6298935.38
+18597	6291416.31
+2488	6286372.05
+2296	6275519.25
+4343	6272834.20
+9958	6267714.61
+2147	6267475.32
+9368	6254578.92
+13134	6252432.76
+10119	6251456.25
+2093	6249342.36
+2392	6237476.56
+17231	6233509.18
+3773	6226654.68
+9839	6214044.10
+19349	6213969.30
+7869	6212751.00
+8158	6210279.24
+13179	6205773.65
+2349	6205589.26
+9052	6200729.70
+1744	6189967.36
+597	6183103.47
+16721	6176606.60
+7498	6176277.25
+15157	6176222.50
+17524	6171107.36
+7922	6170906.07
+6615	6170738.42
+2706	6165991.65
+16432	6162740.68
+16473	6162427.96
+12825	6161595.60
+18813	6154678.55
+1030	6146500.28
+2571	6145772.43
+7707	6144754.71
+10327	6137612.00
+4710	6132346.56
+3649	6130602.63
+4893	6128461.24
+12844	6128191.24
+8794	6122690.28
+1157	6117749.22
+10895	6112017.68
+16166	6108250.98
+11920	6107122.56
+5621	6102123.56
+6141	6093826.56
+14076	6086671.08
+15884	6080485.59
+4814	6080337.96
+5814	6079842.96
+1134	6078685.20
+19048	6071813.28
+303	6070601.35
+15211	6065678.06
+1020	6054168.00
+11292	6052522.83
+7134	6049865.25
+14119	6049619.55
+2389	6042429.12
+5926	6034269.52
+8553	6030922.95
+18814	6023255.17
+12702	6023190.99
+2644	6020931.33
+19628	6010704.00
+18112	6008707.52
+13860	6008170.29
+1456	6005092.14
+1633	6002068.85
+2301	6000302.29
+10740	5999809.05
+2630	5997008.43
+8818	5992296.90
+10043	5990594.02
+653	5987942.83
+6829	5985990.66
+15179	5977727.52
+9663	5973523.92
+5863	5973328.92
+3628	5966340.09
+7618	5960155.86
+2588	5952648.56
+4865	5949383.40
+4233	5944699.60
+13390	5944104.69
+16321	5942714.70
+9653	5941308.50
+18884	5925548.24
+4394	5920927.14
+19774	5916723.12
+1257	5914052.36
+2963	5911917.77
+17157	5899573.02
+383	5884693.36
+11709	5884134.48
+18135	5871431.61
+13169	5869736.88
+2932	5868995.80
+2888	5863229.30
+6820	5853106.80
+18622	5850951.60
+9066	5846052.72
+19840	5832251.40
+6325	5827298.43
+14092	5823711.72
+11532	5823277.12
+18043	5815248.70
+3932	5809134.60
+10364	5808371.40
+1179	5808303.24
+11441	5799378.33
+15767	5798263.26
+14063	5797204.00
+11383	5793893.12
+10065	5781764.45
+17785	5766838.76
+18349	5761765.77
+14824	5760339.78
+14699	5759490.18
+11628	5755137.20
+4403	5752376.64
+13090	5751949.25
+15332	5744606.28
+17573	5744384.15
+12895	5741007.02
+13577	5739209.94
+16594	5732523.64
+8497	5727509.84
+2872	5724068.72
+16181	5721788.16
+6639	5712041.22
+13095	5708798.25
+4535	5693341.20
+10956	5692585.50
+19755	5686913.37
+12995	5682200.41
+13157	5681454.94
+1887	5681341.24
+18472	5680356.00
+19814	5679360.00
+18583	5669360.52
+3894	5664385.49
+1740	5659552.04
+62	5659470.16
+16532	5653779.46
+995	5648973.45
+7486	5646851.87
+19007	5642710.93
+13060	5642024.64
+12371	5635710.45
+2280	5634077.40
+3115	5631352.32
+11107	5631252.46
+5873	5629125.75
+14743	5628732.80
+2224	5624746.62
+2653	5623859.99
+17622	5623057.44
+14700	5615344.00
+14740	5613929.39
+6403	5611924.51
+6896	5609169.88
+10384	5607337.52
+16433	5605707.03
+5898	5604937.76
+4789	5600488.36
+8132	5593107.50
+3838	5592313.20
+13631	5586424.80
+11233	5585247.01
+849	5583516.45
+14653	5581550.45
+14788	5580433.00
+18181	5578562.88
+19815	5577102.62
+5584	5576692.20
+5385	5576420.19
+13780	5569028.61
+9342	5566783.71
+19056	5566524.12
+8189	5565694.74
+13808	5560721.96
+10635	5560058.55
+8304	5550784.41
+14257	5549164.06
+8999	5542100.10
+19134	5539312.56
+8360	5538031.10
+6397	5536651.92
+2597	5525317.76
+8631	5515909.38
+16729	5512663.65
+11861	5511785.93
+16853	5511689.91
+6341	5502790.08
+1312	5496649.40
+5566	5495885.87
+12519	5490649.97
+19032	5488105.02
+8231	5479312.50
+3026	5466732.32
+6388	5466168.80
+15349	5464571.52
+18985	5463897.13
+19848	5454266.40
+17378	5453284.74
+4000	5448690.39
+3710	5445822.53
+13181	5439774.06
+6420	5438325.32
+3644	5437772.14
+1117	5437024.97
+13027	5436968.46
+12884	5432632.34
+12781	5429161.08
+10084	5428231.62
+5640	5423318.58
+16208	5422901.40
+753	5416899.74
+4120	5413822.46
+12348	5412061.71
+1586	5411198.55
+2503	5411125.80
+1381	5397790.10
+19673	5397746.77
+19389	5394426.40
+15532	5386361.38
+3562	5380335.08
+19114	5375133.00
+3204	5372215.46
+6653	5365178.00
+3553	5363690.29
+12687	5361682.12
+3279	5357505.65
+9840	5350112.18
+8258	5347064.67
+11387	5345284.00
+15210	5341117.92
+15092	5340896.80
+6052	5339762.18
+14043	5339587.68
+6284	5336153.00
+6297	5332357.74
+16254	5326607.83
+18436	5326198.50
+14436	5325517.14
+10368	5319983.08
+6349	5317943.40
+19984	5317620.00
+19080	5310678.60
+1403	5306771.32
+5995	5305056.12
+13214	5299420.45
+19293	5297054.40
+7130	5289761.48
+9809	5286784.80
+9273	5277924.77
+16786	5260035.00
+11032	5256354.35
+17719	5246322.20
+3400	5244203.68
+8278	5243718.48
+7873	5241168.80
+9357	5239695.77
+2239	5235767.83
+18695	5223960.35
+19271	5216498.29
+4563	5214673.70
+1896	5210699.79
+15755	5210481.24
+17628	5209882.37
+5747	5206892.24
+18057	5204907.40
+5075	5204412.81
+5325	5190162.75
+17675	5183135.36
+18818	5181295.30
+11064	5180667.57
+12578	5177222.33
+3725	5169183.75
+11112	5161546.00
+4749	5161499.44
+7632	5159180.50
+13119	5151665.51
+5653	5140324.64
+16134	5137482.51
+5671	5136815.55
+18348	5132981.07
+562	5132487.25
+2562	5126763.83
+10304	5125180.00
+7622	5120719.98
+12755	5118213.92
+1814	5100705.58
+11269	5096671.33
+2964	5095253.72
+1616	5091834.00
+9294	5090753.53
+16793	5085330.54
+4999	5081651.75
+17209	5079029.28
+7151	5078937.60
+15522	5072469.60
+3056	5072329.55
+7612	5068322.87
+18453	5063892.92
+18324	5058901.22
+12266	5058186.75
+19394	5056235.73
+1713	5054968.05
+15681	5051569.63
+8274	5043328.00
+18160	5043074.83
+18253	5041572.00
+11840	5040590.04
+1532	5033171.00
+584	5031602.64
+12382	5028901.00
+14814	5022200.07
+19058	5019001.92
+4487	5016640.86
+8482	5015444.25
+18476	5011136.36
+12335	5003581.40
+4455	4997933.31
+14355	4992822.92
+15253	4992642.20
+14069	4983244.20
+17843	4977294.37
+9389	4975885.83
+14435	4971442.19
+13254	4959481.45
+9773	4955887.80
+7615	4952421.54
+6476	4947250.05
+9253	4945159.70
+14932	4934395.48
+13253	4932867.45
+19322	4931525.78
+16945	4931440.61
+731	4930191.93
+6540	4925114.51
+5148	4923048.00
+1934	4921196.90
+15402	4920840.72
+17914	4919607.04
+5416	4916041.92
+16734	4914205.27
+14967	4900262.08
+8706	4894595.58
+136	4891960.92
+19494	4886028.30
+8737	4880640.44
+7653	4879423.64
+4149	4875782.40
+7890	4872424.40
+11142	4871415.42
+10386	4863623.50
+8603	4861814.61
+2680	4861223.74
+4891	4858103.88
+19236	4855097.69
+14251	4854739.86
+18895	4853365.10
+17134	4852041.04
+4932	4843701.45
+10033	4841647.72
+1383	4839144.31
+18721	4837983.36
+8618	4833125.42
+17386	4831545.84
+3790	4830282.36
+1043	4825921.31
+12434	4822597.78
+18385	4819643.40
+6046	4817460.06
+5821	4814423.45
+10836	4814303.24
+6848	4813954.08
+6880	4804600.35
+11249	4800116.82
+11970	4799739.66
+14253	4796521.29
+7782	4793227.13
+75	4790042.88
+7076	4789347.34
+9566	4782531.80
+2137	4767931.74
+2336	4763870.79
+15362	4759043.38
+3284	4755048.76
+12964	4753627.48
+1781	4752835.20
+5454	4748342.98
+12597	4742077.84
+19120	4733459.96
+14884	4731499.44
+825	4730720.28
+14683	4730482.32
+5361	4726113.00
+12179	4725018.10
+1461	4710954.69
+9890	4709658.40
+13369	4705085.39
+11612	4701627.99
+3096	4699414.40
+10383	4697866.47
+11293	4697336.06
+3383	4695825.20
+6231	4694381.72
+7396	4691319.06
+17827	4688797.44
+15856	4683368.21
+8253	4678560.86
+12327	4677984.74
+4395	4676829.82
+4232	4676646.40
+14260	4670522.80
+15288	4669273.99
+17526	4668545.64
+9884	4662693.84
+2118	4660352.78
+4524	4653956.60
+19090	4650872.94
+3928	4649359.44
+14325	4647762.17
+15476	4643469.04
+4179	4639931.76
+14408	4639631.28
+19424	4634817.44
+3334	4633102.50
+9477	4628073.51
+11803	4625396.80
+14805	4618452.18
+463	4616307.28
+16628	4607490.96
+3116	4604463.10
+19962	4602949.47
+12859	4602870.55
+12063	4600708.45
+5648	4592273.25
+8556	4590726.86
+15281	4589425.41
+9414	4587426.90
+13951	4586281.25
+19328	4582624.82
+15963	4579705.50
+10773	4573276.20
+14179	4568816.00
+1895	4563988.16
+6408	4561496.39
+5958	4554000.00
+3653	4548134.40
+11218	4546237.92
+19327	4543987.77
+9572	4535941.16
+14556	4531464.75
+2475	4529761.50
+9631	4529261.56
+1901	4528592.55
+86	4528475.38
+9586	4527146.22
+17361	4519098.87
+8112	4514949.45
+13468	4499728.20
+18239	4497633.64
+10215	4494553.60
+6211	4492264.96
+836	4490945.10
+895	4489141.50
+19542	4488393.75
+4322	4487884.23
+2116	4486944.65
+553	4486075.48
+2515	4485188.26
+16286	4481470.47
+12271	4478224.95
+16570	4465818.00
+7995	4457574.66
+18396	4457229.60
+16331	4455735.48
+18157	4452196.63
+5271	4452040.01
+11622	4451244.84
+4052	4446397.34
+2864	4446008.38
+490	4442892.30
+19837	4434172.39
+4114	4433657.85
+11436	4433070.15
+6085	4431306.57
+9735	4430445.60
+17834	4416286.33
+8157	4416116.65
+18840	4414925.32
+13553	4412261.70
+12562	4411183.04
+14025	4403442.16
+17964	4400360.09
+636	4399863.84
+8390	4389024.33
+231	4387397.30
+9699	4385891.02
+10622	4384005.32
+14364	4383236.90
+10580	4381533.23
+10124	4369800.96
+10451	4368867.50
+4673	4367113.44
+11351	4362616.50
+4770	4362397.32
+12932	4362042.60
+10603	4357216.50
+19733	4348931.75
+4222	4348871.91
+17319	4347687.69
+3375	4346529.48
+14995	4338295.65
+7675	4337499.60
+15043	4333921.20
+4835	4332648.00
+4408	4332588.90
+5559	4330577.09
+7376	4328936.54
+18061	4328793.98
+2749	4328671.53
+6628	4328501.88
+5888	4323049.72
+18872	4322595.62
+5476	4319642.58
+1755	4318935.63
+10623	4315822.56
+18775	4314677.64
+3570	4312697.87
+11147	4310740.57
+6071	4307612.40
+10807	4306006.00
+9550	4299478.56
+657	4296794.19
+19669	4294640.90
+8532	4290651.60
+13469	4281715.62
+8809	4280778.80
+11301	4276847.95
+6147	4266879.92
+2612	4265962.35
+15699	4256118.72
+12300	4254409.11
+3494	4250810.60
+11040	4250030.20
+6190	4244046.80
+17616	4239937.50
+7271	4234407.00
+14048	4226977.44
+4456	4224684.98
+10012	4223841.21
+11175	4223704.14
+18675	4215406.86
+10792	4214898.57
+10806	4209678.32
+18749	4204787.00
+17410	4198025.28
+8032	4195430.00
+11094	4192304.94
+17582	4187341.44
+12246	4183230.95
+6640	4182968.80
+7346	4174707.60
+12747	4169865.81
+3869	4164957.44
+13106	4161902.08
+10547	4159541.36
+15289	4156205.76
+1679	4156156.64
+1126	4155593.08
+19106	4147439.52
+9705	4144024.20
+15324	4142518.56
+16544	4140375.72
+8812	4139322.81
+10772	4134101.64
+2800	4127150.08
+15549	4124704.64
+3607	4118697.57
+1980	4117633.72
+214	4113117.36
+19217	4104217.60
+2460	4098577.46
+19156	4093864.46
+18359	4092727.29
+12865	4092526.84
+14616	4092434.54
+908	4088856.20
+11791	4083804.97
+4157	4078345.60
+3857	4070872.87
+15114	4056112.50
+395	4052997.76
+17456	4051457.28
+10562	4050894.19
+10884	4050330.76
+12177	4049842.68
+15595	4040577.56
+15916	4036044.50
+7084	4035102.72
+4424	4034761.56
+10874	4031015.85
+4740	4030403.76
+16585	4030010.26
+18824	4028984.10
+14875	4028452.08
+13855	4024828.34
+10932	4024002.40
+9084	4021362.45
+14352	4018089.74
+18086	4015180.68
+9514	4013666.67
+15787	4013154.56
+714	4010249.44
+8811	4009588.90
+14386	4007210.88
+616	4004057.26
+7460	4003412.48
+866	4003182.54
+782	4001299.94
+8562	3999441.62
+1366	3994060.86
+2879	3993056.55
+16679	3992434.99
+17306	3990723.30
+13140	3982817.39
+17942	3980857.04
+6572	3977676.28
+3578	3977523.94
+15802	3969946.90
+336	3967938.38
+9807	3964469.60
+12104	3964273.40
+4271	3962359.28
+6702	3961657.44
+19763	3955582.75
+369	3953702.88
+4089	3953455.68
+2593	3946153.80
+590	3943841.16
+8325	3942118.75
+158	3941881.65
+12054	3938362.69
+18330	3938303.88
+5354	3936239.58
+8150	3925793.46
+8344	3921293.60
+6069	3921130.55
+4032	3920008.59
+17939	3917750.27
+7014	3914471.20
+2840	3913131.58
+1868	3912987.54
+10975	3911920.48
+5374	3910802.74
+11128	3908156.46
+18449	3907589.40
+11740	3907459.84
+2356	3907189.00
+5721	3901585.97
+4231	3900779.05
+4352	3899933.44
+432	3899836.44
+15321	3899516.58
+10296	3897015.14
+5647	3895088.16
+7386	3891916.51
+507	3891487.68
+3995	3887387.07
+4278	3882294.02
+18407	3880267.86
+6127	3879166.71
+145	3875277.24
+19269	3874685.76
+18257	3874454.89
+9068	3869767.74
+576	3860007.79
+4860	3852862.02
+18793	3849838.16
+15988	3847257.05
+6891	3846386.75
+3231	3846344.30
+15237	3845421.00
+9035	3844166.85
+7597	3838643.35
+16349	3837121.65
+2497	3827850.20
+3616	3827390.95
+11566	3826122.47
+18403	3822033.02
+2972	3821903.55
+812	3821523.72
+2043	3820561.36
+505	3818922.03
+8257	3815071.92
+6084	3814194.95
+11253	3813917.24
+366	3812257.88
+13632	3811601.32
+14298	3801412.42
+7092	3798729.48
+2058	3796109.04
+14820	3791195.86
+7157	3788690.82
+17211	3786030.17
+16644	3786019.25
+15693	3783662.19
+2627	3782394.60
+11231	3782077.60
+12696	3781761.66
+8705	3778077.00
+16052	3771577.04
+99	3760269.31
+2082	3757517.50
+872	3750005.34
+7126	3749138.92
+10302	3744475.25
+17122	3741012.98
+10080	3740107.10
+16021	3739611.20
+3074	3739224.96
+3142	3738811.02
+13213	3735116.25
+13442	3733132.14
+11542	3731000.12
+13732	3730444.90
+2608	3729372.40
+5	3725511.50
+19157	3723844.72
+18231	3721707.99
+8179	3714155.04
+12740	3708646.91
+11597	3706528.59
+13968	3702376.08
+6436	3687346.44
+9181	3687134.08
+564	3680200.80
+13464	3678406.20
+14084	3673790.38
+2755	3670593.69
+14284	3668640.80
+12178	3653392.48
+15730	3650258.30
+5560	3649569.59
+8594	3647140.56
+7032	3646439.54
+16846	3644843.10
+1530	3642838.08
+3978	3639712.05
+2897	3639442.32
+16625	3636527.54
+12029	3636339.72
+16830	3633448.57
+9597	3632662.11
+5533	3630338.67
+5181	3625965.93
+8131	3625738.62
+8560	3620761.26
+11860	3618746.25
+12008	3614604.40
+10737	3611990.64
+18208	3611596.10
+5119	3611038.20
+11958	3601654.65
+15124	3598278.20
+14058	3597490.02
+12270	3593912.10
+17793	3593318.95
+9385	3587327.84
+12814	3587083.84
+5304	3586230.61
+3631	3582841.65
+610	3581917.30
+19317	3580412.43
+128	3567004.56
+11616	3566154.80
+10176	3565392.15
+7349	3564110.64
+1712	3560408.43
+18860	3559340.60
+17617	3557516.00
+6443	3556296.96
+15408	3554814.56
+16350	3554388.63
+17436	3554105.13
+5740	3551324.68
+12181	3550218.54
+16895	3550119.30
+19995	3548839.70
+4968	3548306.87
+2257	3546692.29
+1825	3543198.78
+18989	3539038.08
+18727	3536081.40
+16165	3533789.84
+3249	3533709.87
+11731	3532875.00
+13032	3532415.79
+9377	3531582.08
+5883	3531479.00
+1211	3528833.40
+12065	3526948.10
+10866	3526146.66
+2073	3520131.30
+2378	3512186.20
+16860	3509693.07
+389	3507814.64
+15604	3505653.27
+11257	3502831.80
+1327	3502022.60
+16602	3501074.88
+1493	3498808.95
+8224	3498179.52
+622	3497158.36
+3072	3495958.72
+1478	3494880.48
+3125	3494169.90
+2052	3488438.08
+8476	3487191.28
+10735	3477740.76
+14860	3476235.84
+6586	3475745.10
+5130	3472024.50
+7181	3471306.30
+618	3467906.52
+15698	3464859.47
+17585	3462450.46
+2548	3456856.96
+2632	3456230.74
+2882	3453986.86
+12216	3452907.15
+4925	3452904.63
+9012	3442581.36
+6667	3430076.40
+17958	3424962.56
+6093	3424241.92
+10648	3417414.00
+1462	3413248.61
+2569	3412388.82
+18616	3409880.91
+7368	3408036.45
+3110	3407374.60
+10824	3406819.29
+11510	3404701.96
+4840	3397236.40
+4449	3396993.60
+1358	3396616.32
+3885	3395817.60
+13381	3391953.52
+1655	3383051.51
+282	3381785.42
+4928	3374270.48
+3199	3372488.80
+16086	3370710.65
+8612	3362922.50
+19597	3360764.00
+8867	3354400.11
+4098	3353574.28
+12617	3351499.05
+14365	3347296.00
+10443	3345493.10
+76	3342081.82
+11585	3341941.22
+4383	3338960.27
+13910	3335964.16
+8076	3332449.89
+16005	3332190.40
+2622	3329364.45
+12822	3321183.52
+17076	3320398.06
+5392	3320357.15
+18628	3319615.84
+13695	3318525.99
+10326	3318274.16
+9109	3317833.90
+1489	3317620.80
+3378	3315948.00
+7738	3312979.20
+1844	3312277.36
+19963	3307500.00
+2436	3306419.05
+886	3302180.70
+15475	3301693.50
+6327	3300680.78
+6050	3299460.20
+9876	3298410.05
+19586	3291131.25
+14349	3289862.52
+10993	3287980.57
+18784	3286752.12
+1800	3285466.24
+990	3284595.50
+3823	3281992.94
+15737	3279305.96
+19518	3276759.63
+9032	3272440.32
+7786	3271217.28
+8648	3271162.44
+5532	3270187.97
+15914	3268520.98
+16065	3265068.84
+11212	3264657.03
+13229	3262022.28
+15827	3260862.72
+1582	3260340.00
+3827	3260093.76
+3546	3259244.07
+15849	3258918.00
+14856	3258379.40
+2028	3255013.96
+6618	3254581.95
+17461	3252926.88
+13551	3241602.20
+19561	3239795.32
+2276	3236172.30
+14203	3234649.39
+7757	3231351.84
+122	3226213.88
+12954	3225943.00
+647	3224783.76
+12383	3223989.44
+3831	3223126.60
+16836	3222260.73
+4565	3221597.44
+19426	3218106.54
+17855	3217813.02
+5624	3207777.36
+8368	3203376.45
+9480	3200904.00
+11181	3199500.53
+8981	3197864.00
+16426	3195995.97
+1648	3195558.90
+14404	3192729.60
+17867	3188571.00
+18117	3183229.04
+14289	3182261.60
+53	3182256.00
+15546	3180180.04
+16245	3178277.46
+1597	3176247.48
+1653	3173456.64
+2845	3171619.61
+15906	3171187.54
+18304	3168571.50
+14068	3167367.60
+6837	3165012.48
+9446	3164446.52
+18889	3156140.96
+16587	3154210.20
+7705	3152977.38
+1120	3151591.17
+17665	3148848.00
+5311	3146721.86
+14157	3144707.32
+7996	3131351.04
+8663	3130526.32
+18271	3127800.96
+6446	3125685.96
+6972	3125007.06
+2572	3123186.83
+13536	3122527.54
+6196	3122172.48
+9338	3121262.40
+11992	3118647.55
+2580	3118284.37
+9098	3117494.10
+5118	3112661.96
+10184	3109293.40
+9932	3105818.24
+18545	3102273.32
+10963	3099314.50
+8405	3097121.12
+9037	3095195.00
+179	3091107.28
+1930	3090915.80
+17723	3090624.66
+4308	3089472.75
+8702	3080129.92
+18621	3079984.80
+4501	3079781.10
+3590	3079049.42
+18264	3078858.44
+15648	3078564.06
+5998	3073264.00
+16904	3072610.80
+3794	3071333.09
+3147	3068485.32
+17221	3068337.22
+4709	3067523.31
+18017	3066743.41
+15613	3063987.86
+16271	3057051.34
+13621	3054774.59
+12919	3054518.50
+12493	3050836.30
+15838	3050645.95
+3273	3048955.15
+8324	3046011.25
+13628	3045324.50
+5522	3044408.50
+2202	3043132.05
+19052	3042566.55
+5767	3041871.70
+17895	3036452.22
+12586	3036386.30
+12425	3035041.52
+13517	3034351.47
+2363	3033336.60
+15060	3032598.51
+6764	3032591.10
+340	3030522.00
+4723	3028910.25
+3566	3027858.61
+17796	3026838.96
+15384	3023792.64
+16336	3010813.56
+679	3010713.30
+7554	3010667.80
+14553	3009756.96
+8379	3009745.17
+15436	3007499.77
+12471	3003991.86
+18059	3003037.53
+8536	3000746.00
+19033	2999373.28
+18179	2996151.20
+10711	2996143.17
+17271	2994264.79
+13932	2989023.58
+3101	2987788.16
+14550	2977853.65
+3080	2977232.58
+14533	2976490.49
+14439	2975313.24
+9237	2973124.78
+1205	2971470.28
+12361	2963419.47
+429	2962631.88
+3970	2960418.45
+8403	2957698.45
+1098	2957514.00
+7932	2955046.14
+16266	2952298.38
+19386	2948854.48
+13147	2947037.91
+2720	2947011.08
+3840	2944219.35
+13482	2942474.88
+9436	2940396.21
+19779	2937105.96
+18032	2933224.38
+7743	2932733.77
+14620	2930766.89
+4606	2927832.59
+18076	2924134.83
+19276	2918176.20
+7483	2915918.95
+8575	2915132.64
+11052	2913140.88
+17251	2908345.80
+8788	2907935.93
+10960	2906511.14
+18782	2903643.78
+19988	2897461.53
+726	2896009.27
+19942	2894251.36
+10864	2892252.48
+17840	2891563.22
+18717	2888939.96
+12391	2886051.30
+18219	2885921.06
+15100	2883342.33
+2491	2880385.74
+12389	2879696.96
+3880	2877770.24
+18579	2874542.48
+13647	2873838.34
+15758	2873804.92
+12917	2873659.60
+18866	2873616.26
+13894	2872986.12
+15200	2872571.93
+9628	2872404.56
+8568	2871598.08
+8389	2870237.88
+5788	2867210.18
+19450	2863310.66
+9440	2863162.92
+16795	2860135.41
+19643	2858987.80
+1974	2856825.84
+14622	2852089.12
+6885	2851437.62
+12532	2848992.64
+1087	2847858.80
+5777	2846407.41
+5629	2846076.12
+6316	2840544.65
+12767	2840514.12
+12134	2840036.91
+14476	2839853.01
+803	2838388.16
+18964	2836942.44
+6020	2833459.20
+10401	2832688.74
+1323	2829964.50
+1151	2829662.44
+1458	2824034.43
+2271	2820756.53
+18740	2814140.80
+7348	2811730.95
+4281	2807190.52
+8043	2804706.24
+3843	2804217.96
+7813	2802350.88
+347	2802245.52
+745	2801725.10
+10388	2799170.58
+18100	2793358.50
+19043	2789013.80
+10644	2787797.01
+16170	2787402.80
+398	2782729.05
+9370	2780078.13
+14504	2780036.04
+1210	2778485.76
+13385	2777445.62
+3799	2775223.60
+11325	2769766.02
+3489	2769554.52
+17181	2769028.50
+6964	2766653.78
+7381	2764898.80
+6253	2764394.64
+5975	2760819.72
+11996	2760687.86
+7570	2758977.12
+4387	2757672.00
+9014	2755367.42
+9403	2748021.66
+11653	2739731.07
+17697	2739312.29
+958	2738032.00
+18816	2737140.00
+14104	2735008.64
+15966	2732250.20
+17912	2724160.95
+7089	2720170.04
+16032	2718976.18
+16891	2717293.32
+19579	2716909.86
+17470	2715048.84
+12408	2712556.52
+4763	2711800.90
+1138	2709709.81
+7363	2708414.40
+7877	2705439.45
+17532	2703698.68
+10512	2701235.92
+11957	2700133.22
+2455	2699593.88
+15119	2696860.80
+9868	2696801.52
+14172	2695307.48
+16120	2689337.82
+13958	2679025.28
+15169	2676686.04
+2648	2672232.00
+6164	2671317.32
+12701	2669216.40
+16382	2669034.54
+15588	2667212.10
+14830	2666758.15
+9119	2665812.24
+1622	2665206.50
+878	2664045.79
+13269	2662784.12
+619	2655417.63
+18386	2653795.02
+2501	2652260.40
+2310	2651631.09
+19420	2649395.61
+4895	2645152.27
+7553	2643682.07
+17814	2642781.44
+16097	2642500.00
+10995	2640811.16
+14895	2637733.72
+18546	2637026.71
+9875	2631358.80
+9591	2626899.54
+6001	2625893.76
+7739	2624573.28
+10431	2624379.54
+4544	2615313.75
+16387	2603195.76
+18375	2601407.83
+8395	2598728.44
+18853	2593356.36
+4900	2592813.15
+1302	2592197.76
+17032	2589806.40
+14292	2589749.56
+43	2587359.58
+5221	2587024.04
+397	2579751.46
+17890	2579674.24
+12157	2575510.48
+7340	2574645.83
+19368	2572618.95
+8848	2570819.15
+13789	2570243.26
+14596	2568234.24
+8408	2567434.41
+19726	2565750.42
+13964	2565579.12
+7740	2563027.50
+14768	2560392.60
+11734	2559062.22
+10294	2558257.97
+15032	2557926.22
+9127	2556379.80
+2181	2553175.00
+16653	2552229.68
+3866	2549994.79
+16814	2548710.76
+1866	2545838.40
+3512	2532626.80
+4145	2529786.15
+12120	2528298.72
+644	2528123.05
+15379	2525181.01
+6392	2524063.08
+2652	2521456.80
+3363	2519202.23
+19167	2517993.18
+16042	2516599.92
+2892	2511854.40
+5711	2509401.72
+14591	2506344.69
+6564	2506277.34
+1231	2505421.24
+5049	2502603.00
+14576	2501606.69
+10211	2500852.20
+293	2493168.48
+7371	2491134.65
+18154	2491047.20
+9494	2489825.52
+14836	2480432.40
+19471	2480403.75
+802	2478998.33
+12541	2477242.60
+15065	2473563.94
+15995	2472803.20
+9408	2471953.56
+9776	2470447.90
+17325	2468989.05
+3391	2468317.72
+16123	2467022.22
+18758	2463798.06
+407	2460304.47
+6840	2456170.78
+9995	2455155.36
+3877	2453696.65
+5817	2452493.13
+14122	2452226.22
+16699	2450273.98
+8921	2450116.48
+15103	2449861.20
+7637	2449628.72
+3076	2443927.38
+6648	2443248.95
+17116	2442263.72
+1645	2440838.40
+3181	2440017.60
+5966	2431558.08
+15882	2428947.30
+7529	2428381.28
+12836	2427897.33
+18052	2427637.76
+13616	2426638.50
+16615	2424775.08
+18147	2424412.68
+4586	2424123.90
+14403	2423141.96
+11606	2422794.31
+13526	2422212.80
+3677	2421404.46
+5553	2418506.21
+12109	2416514.17
+13118	2415931.80
+1563	2408855.40
+16591	2408045.39
+6411	2404918.53
+10272	2402834.48
+10597	2400247.68
+13700	2398035.86
+9548	2397147.90
+14963	2395781.09
+13325	2390637.58
+13864	2388067.88
+7450	2383447.71
+9275	2382868.40
+5829	2378037.92
+13437	2377806.54
+13594	2375046.30
+11442	2374591.08
+15619	2374052.38
+9063	2374035.84
+5990	2368686.50
+7811	2363829.26
+9525	2362974.53
+5597	2361031.84
+8963	2360774.00
+1709	2359839.29
+15814	2358656.64
+17613	2357519.04
+5022	2354550.45
+17740	2354242.83
+3388	2351042.26
+13773	2348739.12
+14467	2348665.04
+11544	2345324.45
+349	2344664.13
+10356	2340862.72
+18272	2338754.60
+4627	2337430.84
+327	2335298.46
+19846	2332224.73
+10814	2330319.60
+13102	2326122.75
+18867	2323972.00
+2824	2323315.08
+19117	2319911.10
+1906	2319757.60
+245	2319450.90
+17318	2317860.39
+3862	2316453.72
+8100	2313874.12
+2958	2312239.47
+10263	2308514.06
+13814	2304940.40
+9394	2303161.74
+18080	2299416.78
+1271	2289526.98
+3327	2278474.48
+8740	2278405.92
+8119	2276428.17
+3368	2274373.62
+7963	2272300.80
+2151	2270932.72
+16995	2270264.68
+9918	2269733.07
+503	2268535.25
+16692	2256484.50
+793	2254198.72
+16455	2252361.86
+6644	2249521.82
+17280	2249437.50
+6813	2248982.00
+4674	2246915.32
+16325	2244369.80
+182	2243290.00
+4626	2242474.35
+10860	2241291.60
+14034	2241220.80
+2476	2240855.20
+4253	2239985.64
+3211	2239871.02
+1290	2233313.00
+8479	2232189.04
+11895	2231607.00
+3487	2230171.62
+14870	2229915.37
+16328	2229483.96
+18585	2228215.50
+7638	2228208.08
+5436	2225672.28
+14594	2223005.07
+4532	2215711.02
+7586	2210562.51
+11870	2205182.82
+18487	2203653.60
+9179	2202720.52
+16500	2201185.31
+3679	2200592.70
+12803	2198295.00
+18056	2196741.90
+11396	2195645.64
+5087	2194120.72
+8067	2192048.64
+15357	2191646.58
+4491	2189713.50
+208	2189046.80
+10958	2188766.82
+9126	2188410.50
+15084	2184327.02
+18850	2183309.52
+3398	2180250.00
+16137	2177318.76
+211	2174808.96
+18422	2174381.00
+15840	2173510.40
+19553	2173079.77
+8221	2169992.16
+17000	2169611.16
+6755	2168505.15
+10817	2167710.68
+8327	2167650.60
+543	2167368.00
+4553	2163371.52
+15019	2162288.00
+334	2162178.48
+8516	2161479.04
+11349	2158941.88
+3902	2157027.86
+14731	2155302.24
+326	2153380.08
+11403	2151242.30
+11657	2150446.08
+9496	2149219.01
+8110	2149120.13
+5153	2148527.25
+884	2148324.98
+8637	2146185.10
+2364	2145790.72
+12386	2145001.47
+10133	2144903.96
+9895	2143324.80
+13755	2142539.40
+4327	2138501.40
+3369	2137408.76
+5815	2136985.00
+19357	2132657.28
+2675	2124158.72
+17869	2123991.72
+11702	2122132.99
+17257	2117850.64
+9952	2116686.32
+3881	2111457.15
+10951	2111185.58
+2128	2109702.30
+6699	2106578.40
+3155	2103636.64
+16649	2101956.20
+15257	2100297.75
+9978	2099566.56
+16810	2098301.44
+10653	2093388.70
+10476	2092766.48
+10883	2087495.28
+9704	2086967.61
+1119	2085182.84
+19139	2079788.34
+2144	2078391.14
+9135	2076377.80
+18548	2075584.32
+10545	2075230.35
+6220	2074341.72
+8616	2072887.65
+5230	2072161.74
+13916	2070504.72
+4299	2069922.96
+894	2069688.16
+17847	2063367.04
+18879	2061902.25
+13036	2061600.17
+10606	2060492.40
+9454	2060016.48
+118	2059808.86
+9601	2059715.76
+13769	2057668.08
+1987	2057289.27
+13863	2055368.00
+13562	2054754.24
+1840	2054183.92
+17995	2053221.90
+17389	2051128.20
+15168	2045987.49
+2139	2045365.40
+4024	2044243.10
+8964	2041648.85
+181	2040167.04
+7628	2039548.92
+3	2038846.09
+15553	2036958.91
+11355	2035405.60
+13006	2034991.20
+3091	2031393.51
+1281	2030628.48
+1408	2028621.66
+18211	2024538.67
+2287	2020754.32
+6228	2019198.82
+4362	2018495.25
+10873	2013280.32
+7383	2009581.92
+1386	2006544.26
+9820	2005815.76
+18134	2003409.73
+15727	2000654.50
+157	2000148.16
+19571	1999891.11
+17728	1997944.40
+5278	1996644.21
+17737	1994653.76
+10220	1989890.98
+1397	1984509.22
+6195	1983928.26
+4270	1983726.95
+16965	1983286.25
+1683	1980638.64
+13086	1978609.40
+7124	1974039.38
+5211	1973843.76
+6794	1973149.47
+257	1973035.44
+6995	1968281.55
+8447	1967292.70
+15873	1967257.89
+12862	1964014.13
+8295	1961467.08
+931	1958825.22
+6876	1957359.48
+1932	1954592.40
+1061	1952688.06
+18108	1951143.67
+5138	1950861.00
+12598	1950211.61
+10829	1943924.62
+11950	1941211.00
+12076	1939323.96
+2176	1938691.37
+6616	1937401.88
+5893	1934358.58
+976	1933066.80
+13173	1932557.52
+14947	1929229.98
+16857	1928814.80
+13403	1928702.88
+4819	1926969.68
+13127	1926929.83
+6871	1926787.68
+15465	1925145.09
+1131	1920005.50
+11845	1913576.40
+8364	1909122.20
+16588	1904272.37
+6759	1903906.29
+11586	1901895.65
+8145	1901787.66
+17333	1897297.20
+13290	1890633.75
+6499	1887621.00
+4881	1887535.92
+7147	1886710.20
+3883	1886567.78
+18911	1885597.12
+11336	1883573.60
+8653	1883275.76
+19476	1881492.48
+14799	1880543.40
+14491	1879219.92
+11815	1877434.34
+3173	1874302.10
+7161	1873023.45
+14631	1873015.30
+4247	1869912.96
+3568	1865824.40
+1500	1865451.03
+11833	1863665.23
+495	1860771.30
+6776	1855589.17
+11374	1855221.12
+5637	1853782.17
+3597	1852826.80
+981	1852083.60
+16076	1850349.69
+17597	1845420.95
+19609	1843185.48
+10997	1843072.02
+3403	1842975.00
+897	1842845.10
+16697	1840630.68
+17644	1840597.80
+6485	1838812.02
+5492	1836202.88
+12038	1835075.06
+9325	1832634.84
+10637	1832347.44
+11318	1830158.39
+4357	1828730.00
+18553	1826335.20
+12623	1825950.85
+961	1825869.60
+1677	1821816.90
+8211	1820432.52
+19719	1819333.55
+19663	1819074.35
+16296	1818353.77
+16527	1817834.42
+4964	1815400.02
+1769	1812929.20
+13126	1808799.96
+7854	1807608.06
+18380	1803641.22
+6584	1802346.98
+7665	1801765.35
+16553	1796146.78
+17761	1795095.72
+11179	1794890.30
+15171	1794148.72
+3018	1793183.88
+15741	1788612.00
+5331	1783901.35
+9860	1775071.26
+7984	1774302.75
+15354	1774270.77
+17884	1774212.44
+16257	1771869.71
+10696	1768645.20
+2104	1767902.64
+14465	1764946.40
+10089	1764692.32
+6719	1762699.54
+3648	1760594.42
+7241	1759913.59
+11122	1757430.04
+17019	1752560.65
+13877	1744271.10
+15325	1743826.26
+17860	1739870.44
+2236	1739795.80
+4436	1738760.32
+7701	1738670.40
+8147	1736855.16
+6676	1736341.44
+19505	1735413.43
+9885	1731366.26
+2112	1725934.08
+5330	1722196.98
+3561	1720377.96
+10104	1714419.16
+16362	1712457.38
+15573	1712365.44
+15006	1711381.35
+14629	1709942.05
+9612	1709528.38
+19910	1709211.15
+13145	1708907.46
+11494	1707973.68
+15895	1706999.45
+8239	1705479.10
+2403	1705331.10
+19436	1702706.00
+3476	1702335.80
+6828	1702292.08
+771	1701589.50
+8448	1700312.44
+3755	1699047.03
+13895	1698679.03
+9785	1698056.37
+6180	1695571.53
+532	1694356.15
+6741	1692552.42
+19964	1692367.64
+3747	1691244.60
+3253	1690719.42
+16119	1688339.25
+7113	1681911.00
+12368	1681219.80
+16378	1679705.60
+1393	1675545.35
+11119	1675453.44
+4469	1674023.49
+6955	1672618.90
+11579	1672345.32
+19898	1671781.70
+15351	1659204.30
+6133	1658215.46
+9110	1658054.68
+2979	1656016.74
+18764	1653708.48
+8995	1653627.58
+13096	1651408.67
+15062	1650548.02
+7924	1650202.40
+10076	1647970.24
+15859	1646036.28
+17932	1642640.66
+19694	1642089.50
+13827	1642001.31
+17963	1639689.00
+10698	1635848.26
+18003	1633530.78
+8416	1633366.77
+476	1631154.06
+2806	1630782.80
+12129	1628615.47
+11215	1626624.70
+14061	1624933.44
+5956	1623586.10
+9043	1622670.40
+13287	1621980.36
+11410	1621420.90
+13990	1621268.20
+12952	1619215.18
+15181	1619088.68
+9784	1618120.53
+10733	1616168.88
+16054	1614531.23
+5864	1614397.83
+1875	1611927.00
+17381	1611664.80
+14562	1607467.92
+575	1605941.73
+2005	1605591.72
+4332	1605448.83
+4653	1602596.30
+15403	1601830.44
+17430	1599681.42
+4798	1593630.50
+12991	1593321.52
+15653	1593138.66
+10066	1593049.06
+8892	1592100.90
+6708	1590159.12
+9825	1589403.92
+8271	1588475.41
+17084	1584280.88
+4003	1583631.00
+869	1582643.16
+16400	1582313.20
+19088	1581708.56
+6581	1581346.80
+9481	1581048.60
+6092	1580846.49
+3624	1578777.30
+6503	1578507.78
+14557	1578280.96
+2428	1577543.92
+15513	1573560.21
+4641	1573363.54
+10152	1570213.60
+5932	1566902.52
+7482	1561323.50
+13745	1558358.34
+2251	1558274.70
+9845	1558068.12
+7603	1557388.20
+1809	1553837.20
+18128	1547643.36
+8086	1543199.04
+14948	1541721.57
+16725	1540948.50
+2999	1540317.66
+8861	1540008.47
+1964	1538815.25
+19374	1537884.78
+15428	1535994.36
+7449	1534782.48
+16884	1534509.16
+10271	1534397.34
+11782	1529963.22
+8184	1529750.70
+4560	1527433.24
+4616	1525374.46
+3814	1524077.04
+17265	1523932.08
+16520	1522906.28
+10475	1518705.06
+5094	1517317.83
+8626	1515142.07
+19895	1512286.68
+19933	1506235.36
+6854	1505626.00
+13995	1505562.18
+7102	1504945.67
+9079	1501237.20
+18329	1500146.90
+3742	1496990.77
+12395	1496904.43
+12214	1496489.40
+12298	1495554.30
+4978	1495389.50
+2927	1494280.10
+2119	1494151.14
+15143	1492039.75
+14548	1487406.60
+840	1486128.98
+5902	1486097.28
+10614	1482144.72
+5895	1481356.80
+15958	1480951.60
+11408	1479948.96
+8407	1474236.00
+6243	1471007.85
+10389	1469004.46
+13871	1468938.64
+19811	1464597.09
+10495	1464290.49
+4389	1463010.83
+1311	1461703.36
+17874	1459408.88
+6597	1458761.87
+19211	1456741.63
+12879	1456178.24
+8840	1455731.46
+14755	1454890.60
+16957	1454465.96
+9257	1454388.76
+5193	1454011.32
+6884	1452474.60
+19948	1452024.00
+15076	1448395.00
+16016	1447557.45
+11693	1445839.68
+6975	1440516.96
+4290	1439768.46
+18900	1438722.10
+14383	1438477.92
+15098	1435941.78
+9322	1435282.80
+458	1433040.45
+10042	1432906.35
+5052	1431900.90
+6600	1431116.55
+3630	1428665.04
+9636	1428193.84
+16511	1427308.74
+4045	1427248.35
+19562	1426348.82
+8814	1425690.09
+2616	1425178.04
+4587	1425109.40
+148	1424237.37
+2712	1423780.26
+10863	1423386.16
+16096	1421942.09
+18936	1421938.65
+18327	1419872.92
+11620	1419050.10
+3740	1418609.85
+3457	1418603.50
+1185	1417637.47
+8178	1417357.26
+17791	1413293.13
+13608	1411323.12
+17849	1409613.50
+6814	1406228.40
+14022	1406138.04
+14231	1403771.52
+19546	1402854.60
+19619	1402389.16
+5609	1402302.54
+5342	1401567.59
+3084	1401096.10
+5708	1400334.90
+17998	1399862.45
+19850	1397630.33
+14004	1395443.10
+13071	1394653.24
+2797	1393747.58
+2866	1392947.25
+19809	1389067.68
+13600	1380865.80
+13614	1380654.36
+5884	1380319.74
+9404	1378623.66
+10656	1376954.32
+12324	1376502.40
+7325	1375030.43
+13295	1373987.34
+11864	1373555.68
+6987	1373481.51
+8386	1371854.41
+10916	1370374.32
+12867	1369058.11
+14668	1369040.34
+13383	1367342.30
+18572	1366953.96
+1152	1366861.38
+6015	1366452.18
+3344	1366185.15
+7889	1365521.92
+13345	1364088.88
+6276	1363421.62
+8069	1361824.20
+17509	1360892.49
+15137	1358678.07
+17163	1357391.52
+4704	1356692.40
+8609	1356578.19
+12644	1356088.14
+17141	1356022.38
+11805	1354826.78
+6386	1354187.22
+3004	1352173.44
+8634	1350211.80
+4399	1349881.20
+10362	1349411.34
+1572	1348835.20
+7359	1348224.10
+11884	1346696.82
+11671	1346424.15
+5350	1346359.28
+3119	1345996.48
+5307	1345356.00
+16117	1345045.12
+8715	1342665.72
+5398	1341179.28
+7627	1338820.56
+8457	1337714.68
+4958	1334732.71
+84	1334146.71
+6932	1333235.36
+757	1332921.07
+4076	1332441.00
+1751	1329112.32
+15701	1327052.16
+4119	1326549.90
+1562	1325604.28
+8741	1325517.60
+1135	1325422.71
+1002	1323418.65
+5832	1323085.71
+5368	1322793.96
+5382	1322628.84
+5616	1319082.26
+2832	1318691.95
+3895	1317858.44
+8629	1317756.51
+5709	1317058.68
+18383	1316451.05
+15797	1314806.64
+1900	1313660.40
+13882	1310455.86
+6785	1309877.80
+14855	1309280.76
+7761	1308602.24
+14268	1306810.40
+6257	1306056.96
+19002	1305509.52
+5095	1303729.02
+10320	1301657.58
+7826	1299561.68
+13359	1298717.14
+7436	1298127.36
+5644	1295055.77
+11327	1290526.41
+5277	1289329.65
+15932	1286235.84
+14322	1284809.36
+144	1284270.12
+3043	1281162.79
+16788	1280955.34
+17136	1280443.12
+12560	1279117.95
+13833	1278834.75
+5414	1277893.26
+12582	1277592.32
+4644	1277535.00
+14032	1277077.88
+18325	1271719.68
+7072	1271228.48
+16868	1267469.42
+8137	1267425.81
+5976	1266206.85
+14125	1265569.05
+13299	1265287.55
+18376	1264249.30
+6157	1261759.92
+5002	1261669.64
+13368	1260918.60
+15589	1260059.76
+2149	1258981.44
+9639	1256283.38
+11689	1256027.92
+9083	1245924.24
+16231	1242625.65
+5084	1242385.28
+11634	1240760.18
+15617	1239731.25
+9865	1237181.62
+14212	1236365.52
+10325	1235223.36
+19582	1235105.76
+740	1234746.81
+19231	1233623.10
+16840	1233063.85
+5703	1231744.33
+5761	1229435.20
+15630	1226611.62
+10408	1224698.40
+9177	1221942.51
+13389	1221666.75
+6104	1221577.92
+9673	1218826.64
+2707	1217124.48
+18672	1214208.80
+5112	1209590.20
+6264	1208318.50
+18496	1207881.75
+10971	1207183.52
+19059	1206729.90
+431	1205938.44
+3821	1201192.75
+826	1200454.62
+3317	1200440.90
+19689	1198899.52
+19641	1198797.99
+6379	1197195.50
+814	1194417.40
+18643	1194000.78
+11865	1193965.76
+12393	1193896.80
+9218	1193660.58
+8674	1191881.32
+8582	1191804.02
+13084	1191508.00
+18844	1190239.96
+16061	1189935.00
+6134	1185550.80
+8628	1183245.60
+8884	1181547.48
+7697	1181032.50
+9044	1180922.60
+13257	1180158.57
+8066	1178808.12
+5876	1177376.80
+14694	1177059.31
+16062	1175391.00
+9104	1175178.90
+11600	1175091.06
+10337	1172684.92
+19188	1172349.78
+8833	1171372.93
+6895	1170602.07
+14100	1168878.40
+13538	1168554.28
+3408	1166645.16
+1860	1165673.68
+13436	1164278.70
+19325	1162733.70
+7403	1161982.00
+4882	1161404.81
+13105	1161320.58
+17880	1161256.02
+19284	1160927.60
+13476	1159035.15
+18913	1158208.30
+18523	1158135.00
+12508	1157538.45
+9090	1156362.64
+17653	1154338.08
+3926	1152652.52
+10183	1148324.57
+7556	1146268.14
+16436	1142656.47
+4741	1141614.00
+15651	1141497.93
+3183	1140081.36
+9532	1139902.50
+16403	1139306.37
+2368	1137421.16
+3889	1136395.50
+2885	1135838.14
+7851	1135110.76
+16234	1135017.24
+12746	1134531.04
+2647	1132941.12
+5373	1132158.01
+10340	1132004.24
+8873	1131949.28
+1132	1131338.88
+15594	1131328.62
+4376	1130282.20
+240	1126682.48
+2231	1124447.15
+929	1121383.92
+11599	1119307.27
+3765	1119093.50
+17635	1118420.16
+7119	1118285.08
+15121	1117715.34
+11858	1116963.54
+16963	1116929.45
+16356	1113648.98
+6924	1112198.40
+16223	1111257.00
+18091	1110043.02
+12628	1108954.80
+16043	1108831.05
+9402	1108290.48
+708	1107084.00
+4078	1105993.96
+17593	1104713.40
+12776	1104362.59
+7583	1102813.53
+14619	1102675.80
+8842	1100110.26
+4196	1099726.55
+2019	1098178.64
+6863	1097246.36
+6489	1096503.07
+2459	1094813.04
+11964	1094485.02
+3236	1093969.80
+17647	1093809.15
+17648	1093114.62
+119	1092687.48
+9626	1092080.00
+9124	1091569.68
+13175	1089851.76
+2532	1088706.35
+16083	1088295.39
+8874	1086011.34
+12872	1082970.30
+19821	1082520.84
+4800	1080389.70
+18696	1079685.36
+19545	1079184.33
+13120	1077742.28
+10588	1076203.83
+17696	1075092.72
+14651	1073222.23
+903	1071146.76
+5858	1070259.48
+8302	1069504.80
+18728	1069225.51
+18026	1068569.00
+19383	1066907.58
+18690	1065930.90
+5924	1065143.12
+4880	1065011.75
+12439	1064381.19
+16529	1062371.70
+19653	1057683.56
+3136	1056810.44
+18932	1056193.65
+2124	1054160.52
+16851	1052646.84
+10123	1051624.00
+5618	1048447.93
+19851	1045187.85
+16278	1044808.38
+11479	1044276.22
+13263	1042046.20
+6041	1041123.38
+7193	1040455.32
+19408	1039430.01
+11260	1036828.52
+5179	1035633.44
+1331	1034398.00
+7706	1034249.40
+8436	1033549.35
+1801	1031886.00
+4170	1031642.90
+11827	1031139.39
+17114	1027985.88
+18278	1026583.11
+1995	1025165.68
+7667	1022980.15
+6559	1021635.45
+17488	1021612.13
+16059	1019781.19
+7633	1018782.57
+10032	1016809.50
+2899	1016438.76
+14628	1016033.20
+10126	1015846.78
+3884	1014413.50
+16913	1013604.40
+18644	1010288.10
+19870	1007919.36
+18564	1007416.20
+10179	1004920.00
+883	1004650.68
+3627	1004461.04
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q12.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q12.out
new file mode 100644
index 00000000000000..7aba8164dd02ad
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q12.out
@@ -0,0 +1,5 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q12 --
+MAIL	647	945
+SHIP	620	943
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q13.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q13.out
new file mode 100644
index 00000000000000..783b2817443eff
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q13.out
@@ -0,0 +1,40 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q13 --
+0	5000
+10	665
+9	657
+11	621
+12	567
+8	564
+13	492
+18	482
+7	480
+20	456
+14	456
+16	449
+19	447
+15	432
+17	423
+21	412
+22	371
+6	337
+23	323
+24	256
+25	204
+5	204
+26	155
+27	141
+28	97
+4	94
+29	64
+3	48
+30	27
+31	26
+32	14
+33	11
+2	11
+34	6
+35	5
+1	2
+36	1
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q14.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q14.out
new file mode 100644
index 00000000000000..4741d52618705e
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q14.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q14 --
+16.2838556890
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q15.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q15.out
new file mode 100644
index 00000000000000..916c705a213a67
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q15.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q15 --
+677	Supplier#000000677	8mhrffG7D2WJBSQbOGstQ	23-290-639-3315	1614410.2928
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q16.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q16.out
new file mode 100644
index 00000000000000..2598eee6e87731
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q16.out
@@ -0,0 +1,2765 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q16 --
+Brand#14	SMALL ANODIZED NICKEL	45	12
+Brand#22	SMALL BURNISHED BRASS	19	12
+Brand#25	PROMO POLISHED COPPER	14	12
+Brand#35	LARGE ANODIZED STEEL	45	12
+Brand#35	PROMO BRUSHED COPPER	9	12
+Brand#51	ECONOMY ANODIZED STEEL	9	12
+Brand#53	LARGE BRUSHED NICKEL	45	12
+Brand#11	ECONOMY POLISHED COPPER	14	8
+Brand#11	LARGE PLATED STEEL	23	8
+Brand#11	PROMO POLISHED STEEL	23	8
+Brand#11	STANDARD ANODIZED COPPER	9	8
+Brand#12	ECONOMY BURNISHED BRASS	9	8
+Brand#12	LARGE ANODIZED BRASS	14	8
+Brand#12	SMALL ANODIZED TIN	23	8
+Brand#12	SMALL BRUSHED NICKEL	23	8
+Brand#12	STANDARD ANODIZED BRASS	3	8
+Brand#12	STANDARD BURNISHED TIN	23	8
+Brand#13	ECONOMY POLISHED BRASS	9	8
+Brand#13	LARGE BURNISHED COPPER	45	8
+Brand#13	MEDIUM ANODIZED STEEL	23	8
+Brand#13	MEDIUM PLATED NICKEL	3	8
+Brand#13	PROMO BURNISHED BRASS	9	8
+Brand#13	PROMO POLISHED BRASS	3	8
+Brand#13	PROMO POLISHED TIN	36	8
+Brand#13	SMALL BURNISHED STEEL	23	8
+Brand#13	STANDARD BRUSHED STEEL	9	8
+Brand#14	ECONOMY BRUSHED TIN	3	8
+Brand#14	ECONOMY BURNISHED TIN	23	8
+Brand#14	PROMO BRUSHED STEEL	9	8
+Brand#14	PROMO PLATED TIN	45	8
+Brand#15	ECONOMY PLATED TIN	9	8
+Brand#15	STANDARD BRUSHED COPPER	14	8
+Brand#15	STANDARD PLATED TIN	3	8
+Brand#21	ECONOMY POLISHED TIN	3	8
+Brand#21	PROMO POLISHED COPPER	9	8
+Brand#21	PROMO POLISHED TIN	49	8
+Brand#21	SMALL POLISHED STEEL	3	8
+Brand#21	STANDARD PLATED BRASS	49	8
+Brand#21	STANDARD PLATED NICKEL	49	8
+Brand#22	ECONOMY ANODIZED TIN	49	8
+Brand#22	ECONOMY BRUSHED BRASS	14	8
+Brand#22	LARGE BURNISHED TIN	36	8
+Brand#22	MEDIUM ANODIZED STEEL	36	8
+Brand#22	MEDIUM PLATED STEEL	9	8
+Brand#22	PROMO POLISHED NICKEL	9	8
+Brand#22	SMALL ANODIZED STEEL	19	8
+Brand#22	STANDARD ANODIZED COPPER	23	8
+Brand#23	ECONOMY BRUSHED NICKEL	23	8
+Brand#23	LARGE ANODIZED BRASS	9	8
+Brand#23	LARGE ANODIZED STEEL	23	8
+Brand#23	SMALL BRUSHED COPPER	23	8
+Brand#23	STANDARD BRUSHED TIN	3	8
+Brand#23	STANDARD BURNISHED NICKEL	49	8
+Brand#23	STANDARD PLATED NICKEL	36	8
+Brand#24	ECONOMY ANODIZED BRASS	19	8
+Brand#24	ECONOMY POLISHED BRASS	36	8
+Brand#24	LARGE BURNISHED STEEL	14	8
+Brand#24	MEDIUM PLATED NICKEL	36	8
+Brand#25	ECONOMY BRUSHED STEEL	49	8
+Brand#25	MEDIUM BURNISHED TIN	3	8
+Brand#25	PROMO ANODIZED TIN	36	8
+Brand#25	PROMO PLATED NICKEL	3	8
+Brand#25	SMALL BURNISHED BRASS	3	8
+Brand#31	LARGE ANODIZED BRASS	3	8
+Brand#31	SMALL ANODIZED COPPER	3	8
+Brand#31	SMALL ANODIZED NICKEL	9	8
+Brand#31	SMALL ANODIZED STEEL	14	8
+Brand#32	MEDIUM ANODIZED STEEL	49	8
+Brand#32	MEDIUM BURNISHED COPPER	19	8
+Brand#32	SMALL BURNISHED STEEL	23	8
+Brand#32	STANDARD BURNISHED STEEL	45	8
+Brand#34	ECONOMY ANODIZED NICKEL	49	8
+Brand#34	LARGE BURNISHED TIN	49	8
+Brand#34	MEDIUM BURNISHED NICKEL	3	8
+Brand#34	PROMO ANODIZED TIN	3	8
+Brand#34	SMALL BRUSHED TIN	3	8
+Brand#34	STANDARD BURNISHED TIN	23	8
+Brand#35	MEDIUM BRUSHED STEEL	45	8
+Brand#35	PROMO BURNISHED STEEL	14	8
+Brand#35	SMALL BURNISHED STEEL	23	8
+Brand#35	SMALL POLISHED COPPER	14	8
+Brand#35	STANDARD PLATED COPPER	9	8
+Brand#41	ECONOMY BRUSHED BRASS	23	8
+Brand#41	LARGE BURNISHED STEEL	23	8
+Brand#41	PROMO BURNISHED TIN	14	8
+Brand#41	PROMO PLATED STEEL	36	8
+Brand#41	PROMO POLISHED TIN	19	8
+Brand#41	SMALL BURNISHED COPPER	23	8
+Brand#42	LARGE POLISHED TIN	14	8
+Brand#42	MEDIUM ANODIZED TIN	49	8
+Brand#42	MEDIUM BRUSHED TIN	14	8
+Brand#42	MEDIUM BURNISHED NICKEL	23	8
+Brand#42	MEDIUM PLATED COPPER	45	8
+Brand#42	MEDIUM PLATED TIN	45	8
+Brand#42	SMALL PLATED COPPER	36	8
+Brand#43	ECONOMY BRUSHED STEEL	45	8
+Brand#43	LARGE BRUSHED COPPER	19	8
+Brand#43	PROMO BRUSHED BRASS	36	8
+Brand#43	SMALL BURNISHED TIN	45	8
+Brand#43	SMALL PLATED COPPER	45	8
+Brand#44	PROMO POLISHED TIN	23	8
+Brand#44	SMALL POLISHED NICKEL	14	8
+Brand#44	SMALL POLISHED TIN	45	8
+Brand#44	STANDARD BURNISHED COPPER	3	8
+Brand#51	LARGE ANODIZED BRASS	19	8
+Brand#51	LARGE POLISHED COPPER	23	8
+Brand#51	MEDIUM ANODIZED TIN	9	8
+Brand#51	MEDIUM ANODIZED TIN	14	8
+Brand#51	MEDIUM BURNISHED NICKEL	23	8
+Brand#51	SMALL ANODIZED COPPER	45	8
+Brand#51	SMALL ANODIZED COPPER	49	8
+Brand#51	SMALL BRUSHED COPPER	45	8
+Brand#51	SMALL BRUSHED TIN	36	8
+Brand#51	STANDARD POLISHED TIN	3	8
+Brand#52	ECONOMY ANODIZED STEEL	3	8
+Brand#52	ECONOMY PLATED TIN	19	8
+Brand#52	LARGE PLATED TIN	3	8
+Brand#52	MEDIUM ANODIZED TIN	19	8
+Brand#52	MEDIUM BURNISHED COPPER	3	8
+Brand#52	PROMO POLISHED BRASS	23	8
+Brand#52	SMALL PLATED COPPER	36	8
+Brand#52	SMALL POLISHED NICKEL	9	8
+Brand#52	STANDARD POLISHED NICKEL	45	8
+Brand#53	ECONOMY POLISHED STEEL	45	8
+Brand#53	LARGE POLISHED NICKEL	3	8
+Brand#53	SMALL BRUSHED COPPER	14	8
+Brand#53	STANDARD PLATED STEEL	45	8
+Brand#54	ECONOMY POLISHED BRASS	49	8
+Brand#54	ECONOMY POLISHED TIN	23	8
+Brand#54	LARGE ANODIZED NICKEL	49	8
+Brand#54	MEDIUM BRUSHED STEEL	9	8
+Brand#54	SMALL BURNISHED NICKEL	14	8
+Brand#54	SMALL PLATED TIN	14	8
+Brand#54	STANDARD BURNISHED STEEL	14	8
+Brand#54	STANDARD PLATED BRASS	23	8
+Brand#55	MEDIUM BURNISHED TIN	36	8
+Brand#55	PROMO ANODIZED BRASS	14	8
+Brand#55	STANDARD BURNISHED COPPER	45	8
+Brand#15	STANDARD PLATED TIN	36	7
+Brand#23	SMALL POLISHED BRASS	49	7
+Brand#42	STANDARD PLATED COPPER	19	7
+Brand#51	LARGE POLISHED NICKEL	14	7
+Brand#11	ECONOMY ANODIZED BRASS	19	4
+Brand#11	ECONOMY ANODIZED BRASS	45	4
+Brand#11	ECONOMY ANODIZED NICKEL	36	4
+Brand#11	ECONOMY BRUSHED COPPER	3	4
+Brand#11	ECONOMY BRUSHED COPPER	9	4
+Brand#11	ECONOMY BRUSHED STEEL	9	4
+Brand#11	ECONOMY BRUSHED STEEL	36	4
+Brand#11	ECONOMY BURNISHED BRASS	36	4
+Brand#11	ECONOMY BURNISHED COPPER	9	4
+Brand#11	ECONOMY BURNISHED COPPER	49	4
+Brand#11	ECONOMY BURNISHED NICKEL	14	4
+Brand#11	ECONOMY BURNISHED NICKEL	49	4
+Brand#11	ECONOMY PLATED COPPER	19	4
+Brand#11	ECONOMY PLATED NICKEL	45	4
+Brand#11	ECONOMY PLATED TIN	9	4
+Brand#11	ECONOMY POLISHED BRASS	3	4
+Brand#11	ECONOMY POLISHED COPPER	3	4
+Brand#11	ECONOMY POLISHED COPPER	45	4
+Brand#11	ECONOMY POLISHED NICKEL	36	4
+Brand#11	ECONOMY POLISHED STEEL	23	4
+Brand#11	ECONOMY POLISHED TIN	14	4
+Brand#11	LARGE ANODIZED COPPER	23	4
+Brand#11	LARGE ANODIZED NICKEL	9	4
+Brand#11	LARGE ANODIZED STEEL	9	4
+Brand#11	LARGE ANODIZED TIN	45	4
+Brand#11	LARGE BRUSHED STEEL	19	4
+Brand#11	LARGE BRUSHED TIN	3	4
+Brand#11	LARGE BRUSHED TIN	14	4
+Brand#11	LARGE BURNISHED COPPER	9	4
+Brand#11	LARGE BURNISHED COPPER	19	4
+Brand#11	LARGE BURNISHED STEEL	23	4
+Brand#11	LARGE BURNISHED TIN	9	4
+Brand#11	LARGE PLATED COPPER	23	4
+Brand#11	LARGE PLATED TIN	9	4
+Brand#11	LARGE PLATED TIN	14	4
+Brand#11	LARGE PLATED TIN	23	4
+Brand#11	LARGE POLISHED NICKEL	49	4
+Brand#11	MEDIUM ANODIZED BRASS	45	4
+Brand#11	MEDIUM ANODIZED TIN	14	4
+Brand#11	MEDIUM BRUSHED BRASS	14	4
+Brand#11	MEDIUM BRUSHED BRASS	45	4
+Brand#11	MEDIUM BRUSHED NICKEL	14	4
+Brand#11	MEDIUM BRUSHED NICKEL	36	4
+Brand#11	MEDIUM BRUSHED STEEL	19	4
+Brand#11	MEDIUM BURNISHED COPPER	9	4
+Brand#11	MEDIUM BURNISHED TIN	36	4
+Brand#11	MEDIUM PLATED BRASS	3	4
+Brand#11	MEDIUM PLATED TIN	19	4
+Brand#11	PROMO ANODIZED BRASS	3	4
+Brand#11	PROMO ANODIZED BRASS	19	4
+Brand#11	PROMO ANODIZED BRASS	45	4
+Brand#11	PROMO ANODIZED BRASS	49	4
+Brand#11	PROMO ANODIZED STEEL	23	4
+Brand#11	PROMO ANODIZED TIN	45	4
+Brand#11	PROMO BRUSHED BRASS	23	4
+Brand#11	PROMO BRUSHED STEEL	3	4
+Brand#11	PROMO BURNISHED BRASS	23	4
+Brand#11	PROMO BURNISHED BRASS	36	4
+Brand#11	PROMO BURNISHED BRASS	49	4
+Brand#11	PROMO BURNISHED TIN	9	4
+Brand#11	PROMO PLATED BRASS	9	4
+Brand#11	PROMO PLATED BRASS	45	4
+Brand#11	PROMO PLATED NICKEL	19	4
+Brand#11	PROMO POLISHED BRASS	3	4
+Brand#11	PROMO POLISHED BRASS	9	4
+Brand#11	PROMO POLISHED BRASS	19	4
+Brand#11	PROMO POLISHED COPPER	14	4
+Brand#11	PROMO POLISHED COPPER	45	4
+Brand#11	PROMO POLISHED TIN	49	4
+Brand#11	SMALL ANODIZED COPPER	36	4
+Brand#11	SMALL ANODIZED NICKEL	3	4
+Brand#11	SMALL ANODIZED NICKEL	14	4
+Brand#11	SMALL ANODIZED TIN	14	4
+Brand#11	SMALL ANODIZED TIN	19	4
+Brand#11	SMALL ANODIZED TIN	45	4
+Brand#11	SMALL BRUSHED TIN	14	4
+Brand#11	SMALL BRUSHED TIN	23	4
+Brand#11	SMALL BRUSHED TIN	45	4
+Brand#11	SMALL BURNISHED BRASS	49	4
+Brand#11	SMALL BURNISHED COPPER	23	4
+Brand#11	SMALL PLATED COPPER	45	4
+Brand#11	SMALL PLATED NICKEL	3	4
+Brand#11	SMALL PLATED STEEL	36	4
+Brand#11	SMALL PLATED TIN	19	4
+Brand#11	SMALL POLISHED BRASS	14	4
+Brand#11	SMALL POLISHED BRASS	23	4
+Brand#11	SMALL POLISHED COPPER	14	4
+Brand#11	SMALL POLISHED COPPER	36	4
+Brand#11	SMALL POLISHED STEEL	9	4
+Brand#11	STANDARD BRUSHED COPPER	23	4
+Brand#11	STANDARD BRUSHED NICKEL	14	4
+Brand#11	STANDARD BRUSHED TIN	14	4
+Brand#11	STANDARD BURNISHED BRASS	3	4
+Brand#11	STANDARD BURNISHED STEEL	23	4
+Brand#11	STANDARD PLATED BRASS	19	4
+Brand#11	STANDARD PLATED TIN	19	4
+Brand#11	STANDARD POLISHED NICKEL	45	4
+Brand#11	STANDARD POLISHED TIN	14	4
+Brand#11	STANDARD POLISHED TIN	45	4
+Brand#12	ECONOMY ANODIZED BRASS	23	4
+Brand#12	ECONOMY ANODIZED COPPER	14	4
+Brand#12	ECONOMY ANODIZED NICKEL	19	4
+Brand#12	ECONOMY ANODIZED NICKEL	45	4
+Brand#12	ECONOMY ANODIZED STEEL	9	4
+Brand#12	ECONOMY BRUSHED COPPER	36	4
+Brand#12	ECONOMY BRUSHED NICKEL	49	4
+Brand#12	ECONOMY BRUSHED STEEL	49	4
+Brand#12	ECONOMY BURNISHED COPPER	45	4
+Brand#12	ECONOMY PLATED COPPER	23	4
+Brand#12	ECONOMY PLATED STEEL	23	4
+Brand#12	ECONOMY PLATED TIN	36	4
+Brand#12	ECONOMY POLISHED BRASS	14	4
+Brand#12	ECONOMY POLISHED COPPER	45	4
+Brand#12	ECONOMY POLISHED NICKEL	9	4
+Brand#12	LARGE ANODIZED NICKEL	9	4
+Brand#12	LARGE ANODIZED NICKEL	49	4
+Brand#12	LARGE ANODIZED STEEL	49	4
+Brand#12	LARGE ANODIZED TIN	36	4
+Brand#12	LARGE ANODIZED TIN	45	4
+Brand#12	LARGE BURNISHED BRASS	14	4
+Brand#12	LARGE BURNISHED BRASS	19	4
+Brand#12	LARGE BURNISHED COPPER	9	4
+Brand#12	LARGE BURNISHED NICKEL	45	4
+Brand#12	LARGE BURNISHED TIN	36	4
+Brand#12	LARGE PLATED BRASS	3	4
+Brand#12	LARGE PLATED STEEL	36	4
+Brand#12	LARGE PLATED STEEL	45	4
+Brand#12	LARGE PLATED TIN	23	4
+Brand#12	LARGE POLISHED COPPER	14	4
+Brand#12	LARGE POLISHED COPPER	19	4
+Brand#12	LARGE POLISHED COPPER	49	4
+Brand#12	LARGE POLISHED STEEL	3	4
+Brand#12	MEDIUM ANODIZED COPPER	9	4
+Brand#12	MEDIUM ANODIZED COPPER	45	4
+Brand#12	MEDIUM ANODIZED NICKEL	45	4
+Brand#12	MEDIUM BRUSHED BRASS	19	4
+Brand#12	MEDIUM BRUSHED COPPER	9	4
+Brand#12	MEDIUM BRUSHED COPPER	36	4
+Brand#12	MEDIUM BRUSHED COPPER	49	4
+Brand#12	MEDIUM BRUSHED NICKEL	3	4
+Brand#12	MEDIUM BRUSHED NICKEL	14	4
+Brand#12	MEDIUM BRUSHED NICKEL	23	4
+Brand#12	MEDIUM BURNISHED BRASS	3	4
+Brand#12	MEDIUM BURNISHED COPPER	36	4
+Brand#12	MEDIUM BURNISHED NICKEL	19	4
+Brand#12	MEDIUM BURNISHED TIN	14	4
+Brand#12	MEDIUM PLATED BRASS	23	4
+Brand#12	MEDIUM PLATED TIN	19	4
+Brand#12	MEDIUM PLATED TIN	23	4
+Brand#12	MEDIUM PLATED TIN	49	4
+Brand#12	PROMO ANODIZED BRASS	9	4
+Brand#12	PROMO ANODIZED BRASS	45	4
+Brand#12	PROMO ANODIZED NICKEL	14	4
+Brand#12	PROMO ANODIZED STEEL	49	4
+Brand#12	PROMO ANODIZED TIN	3	4
+Brand#12	PROMO ANODIZED TIN	19	4
+Brand#12	PROMO BRUSHED COPPER	14	4
+Brand#12	PROMO BRUSHED COPPER	19	4
+Brand#12	PROMO BRUSHED NICKEL	23	4
+Brand#12	PROMO BRUSHED STEEL	23	4
+Brand#12	PROMO BRUSHED STEEL	36	4
+Brand#12	PROMO BURNISHED BRASS	49	4
+Brand#12	PROMO BURNISHED TIN	9	4
+Brand#12	PROMO BURNISHED TIN	14	4
+Brand#12	PROMO PLATED BRASS	36	4
+Brand#12	PROMO POLISHED COPPER	23	4
+Brand#12	PROMO POLISHED NICKEL	3	4
+Brand#12	PROMO POLISHED NICKEL	9	4
+Brand#12	PROMO POLISHED STEEL	14	4
+Brand#12	PROMO POLISHED TIN	23	4
+Brand#12	PROMO POLISHED TIN	36	4
+Brand#12	SMALL ANODIZED BRASS	36	4
+Brand#12	SMALL ANODIZED COPPER	23	4
+Brand#12	SMALL ANODIZED STEEL	36	4
+Brand#12	SMALL ANODIZED TIN	14	4
+Brand#12	SMALL BRUSHED COPPER	19	4
+Brand#12	SMALL BRUSHED COPPER	36	4
+Brand#12	SMALL BRUSHED TIN	36	4
+Brand#12	SMALL BURNISHED BRASS	14	4
+Brand#12	SMALL BURNISHED COPPER	9	4
+Brand#12	SMALL BURNISHED COPPER	36	4
+Brand#12	SMALL PLATED BRASS	9	4
+Brand#12	SMALL POLISHED BRASS	49	4
+Brand#12	SMALL POLISHED NICKEL	19	4
+Brand#12	SMALL POLISHED TIN	3	4
+Brand#12	STANDARD ANODIZED BRASS	19	4
+Brand#12	STANDARD ANODIZED NICKEL	19	4
+Brand#12	STANDARD ANODIZED STEEL	19	4
+Brand#12	STANDARD BRUSHED COPPER	36	4
+Brand#12	STANDARD BRUSHED NICKEL	23	4
+Brand#12	STANDARD BRUSHED STEEL	49	4
+Brand#12	STANDARD BURNISHED BRASS	23	4
+Brand#12	STANDARD BURNISHED COPPER	14	4
+Brand#12	STANDARD BURNISHED NICKEL	45	4
+Brand#12	STANDARD BURNISHED NICKEL	49	4
+Brand#12	STANDARD BURNISHED TIN	3	4
+Brand#12	STANDARD BURNISHED TIN	14	4
+Brand#12	STANDARD PLATED BRASS	19	4
+Brand#12	STANDARD PLATED NICKEL	45	4
+Brand#12	STANDARD PLATED STEEL	36	4
+Brand#12	STANDARD PLATED STEEL	45	4
+Brand#12	STANDARD PLATED TIN	9	4
+Brand#12	STANDARD POLISHED BRASS	49	4
+Brand#12	STANDARD POLISHED COPPER	3	4
+Brand#12	STANDARD POLISHED NICKEL	23	4
+Brand#12	STANDARD POLISHED TIN	14	4
+Brand#13	ECONOMY ANODIZED NICKEL	14	4
+Brand#13	ECONOMY ANODIZED NICKEL	19	4
+Brand#13	ECONOMY ANODIZED STEEL	45	4
+Brand#13	ECONOMY ANODIZED STEEL	49	4
+Brand#13	ECONOMY BRUSHED BRASS	3	4
+Brand#13	ECONOMY BURNISHED STEEL	14	4
+Brand#13	ECONOMY BURNISHED TIN	19	4
+Brand#13	ECONOMY BURNISHED TIN	45	4
+Brand#13	ECONOMY PLATED COPPER	19	4
+Brand#13	ECONOMY PLATED NICKEL	3	4
+Brand#13	ECONOMY PLATED STEEL	23	4
+Brand#13	ECONOMY PLATED TIN	3	4
+Brand#13	ECONOMY POLISHED BRASS	3	4
+Brand#13	ECONOMY POLISHED COPPER	9	4
+Brand#13	ECONOMY POLISHED COPPER	49	4
+Brand#13	ECONOMY POLISHED STEEL	23	4
+Brand#13	ECONOMY POLISHED STEEL	49	4
+Brand#13	LARGE ANODIZED BRASS	23	4
+Brand#13	LARGE ANODIZED COPPER	19	4
+Brand#13	LARGE ANODIZED NICKEL	9	4
+Brand#13	LARGE ANODIZED STEEL	45	4
+Brand#13	LARGE ANODIZED TIN	19	4
+Brand#13	LARGE BRUSHED BRASS	3	4
+Brand#13	LARGE BRUSHED BRASS	9	4
+Brand#13	LARGE BRUSHED BRASS	19	4
+Brand#13	LARGE BRUSHED COPPER	9	4
+Brand#13	LARGE BRUSHED COPPER	36	4
+Brand#13	LARGE BRUSHED NICKEL	3	4
+Brand#13	LARGE BRUSHED NICKEL	9	4
+Brand#13	LARGE BRUSHED NICKEL	14	4
+Brand#13	LARGE BRUSHED NICKEL	23	4
+Brand#13	LARGE BRUSHED STEEL	19	4
+Brand#13	LARGE BRUSHED TIN	49	4
+Brand#13	LARGE BURNISHED BRASS	49	4
+Brand#13	LARGE BURNISHED TIN	49	4
+Brand#13	LARGE PLATED COPPER	23	4
+Brand#13	LARGE PLATED STEEL	14	4
+Brand#13	LARGE PLATED STEEL	19	4
+Brand#13	LARGE PLATED STEEL	36	4
+Brand#13	LARGE PLATED TIN	14	4
+Brand#13	LARGE PLATED TIN	45	4
+Brand#13	LARGE POLISHED BRASS	3	4
+Brand#13	LARGE POLISHED BRASS	23	4
+Brand#13	LARGE POLISHED BRASS	49	4
+Brand#13	MEDIUM ANODIZED BRASS	3	4
+Brand#13	MEDIUM ANODIZED BRASS	36	4
+Brand#13	MEDIUM ANODIZED COPPER	14	4
+Brand#13	MEDIUM ANODIZED NICKEL	3	4
+Brand#13	MEDIUM ANODIZED STEEL	14	4
+Brand#13	MEDIUM ANODIZED STEEL	19	4
+Brand#13	MEDIUM ANODIZED STEEL	36	4
+Brand#13	MEDIUM BRUSHED BRASS	49	4
+Brand#13	MEDIUM BRUSHED COPPER	23	4
+Brand#13	MEDIUM BRUSHED NICKEL	45	4
+Brand#13	MEDIUM BURNISHED BRASS	9	4
+Brand#13	MEDIUM BURNISHED STEEL	19	4
+Brand#13	MEDIUM BURNISHED STEEL	49	4
+Brand#13	MEDIUM PLATED BRASS	3	4
+Brand#13	MEDIUM PLATED BRASS	23	4
+Brand#13	MEDIUM PLATED BRASS	36	4
+Brand#13	MEDIUM PLATED COPPER	19	4
+Brand#13	MEDIUM PLATED COPPER	23	4
+Brand#13	MEDIUM PLATED STEEL	3	4
+Brand#13	PROMO ANODIZED BRASS	14	4
+Brand#13	PROMO ANODIZED COPPER	9	4
+Brand#13	PROMO ANODIZED COPPER	45	4
+Brand#13	PROMO ANODIZED STEEL	23	4
+Brand#13	PROMO BRUSHED COPPER	49	4
+Brand#13	PROMO BURNISHED COPPER	19	4
+Brand#13	PROMO BURNISHED NICKEL	9	4
+Brand#13	PROMO BURNISHED STEEL	23	4
+Brand#13	PROMO BURNISHED STEEL	45	4
+Brand#13	PROMO BURNISHED TIN	19	4
+Brand#13	PROMO PLATED BRASS	14	4
+Brand#13	PROMO PLATED BRASS	19	4
+Brand#13	PROMO PLATED COPPER	3	4
+Brand#13	PROMO PLATED COPPER	19	4
+Brand#13	PROMO PLATED TIN	19	4
+Brand#13	PROMO POLISHED BRASS	49	4
+Brand#13	PROMO POLISHED STEEL	45	4
+Brand#13	PROMO POLISHED TIN	14	4
+Brand#13	SMALL ANODIZED STEEL	23	4
+Brand#13	SMALL ANODIZED TIN	3	4
+Brand#13	SMALL ANODIZED TIN	45	4
+Brand#13	SMALL BRUSHED COPPER	3	4
+Brand#13	SMALL BRUSHED NICKEL	19	4
+Brand#13	SMALL BRUSHED TIN	9	4
+Brand#13	SMALL BRUSHED TIN	45	4
+Brand#13	SMALL BURNISHED BRASS	19	4
+Brand#13	SMALL BURNISHED BRASS	45	4
+Brand#13	SMALL PLATED BRASS	9	4
+Brand#13	SMALL PLATED TIN	45	4
+Brand#13	SMALL POLISHED NICKEL	19	4
+Brand#13	SMALL POLISHED STEEL	49	4
+Brand#13	STANDARD ANODIZED COPPER	45	4
+Brand#13	STANDARD ANODIZED NICKEL	9	4
+Brand#13	STANDARD ANODIZED NICKEL	19	4
+Brand#13	STANDARD ANODIZED STEEL	14	4
+Brand#13	STANDARD ANODIZED TIN	9	4
+Brand#13	STANDARD ANODIZED TIN	36	4
+Brand#13	STANDARD BRUSHED BRASS	19	4
+Brand#13	STANDARD BRUSHED TIN	9	4
+Brand#13	STANDARD BURNISHED BRASS	9	4
+Brand#13	STANDARD BURNISHED BRASS	14	4
+Brand#13	STANDARD BURNISHED COPPER	45	4
+Brand#13	STANDARD PLATED BRASS	49	4
+Brand#13	STANDARD PLATED COPPER	19	4
+Brand#13	STANDARD PLATED NICKEL	23	4
+Brand#13	STANDARD PLATED TIN	9	4
+Brand#13	STANDARD POLISHED BRASS	49	4
+Brand#13	STANDARD POLISHED COPPER	9	4
+Brand#13	STANDARD POLISHED COPPER	49	4
+Brand#13	STANDARD POLISHED NICKEL	14	4
+Brand#13	STANDARD POLISHED NICKEL	19	4
+Brand#13	STANDARD POLISHED STEEL	23	4
+Brand#14	ECONOMY ANODIZED BRASS	19	4
+Brand#14	ECONOMY ANODIZED COPPER	9	4
+Brand#14	ECONOMY ANODIZED STEEL	19	4
+Brand#14	ECONOMY ANODIZED STEEL	45	4
+Brand#14	ECONOMY BRUSHED BRASS	19	4
+Brand#14	ECONOMY BRUSHED COPPER	45	4
+Brand#14	ECONOMY BRUSHED NICKEL	14	4
+Brand#14	ECONOMY BRUSHED TIN	14	4
+Brand#14	ECONOMY BURNISHED COPPER	9	4
+Brand#14	ECONOMY BURNISHED COPPER	19	4
+Brand#14	ECONOMY BURNISHED STEEL	36	4
+Brand#14	ECONOMY BURNISHED TIN	3	4
+Brand#14	ECONOMY PLATED BRASS	36	4
+Brand#14	ECONOMY PLATED COPPER	49	4
+Brand#14	ECONOMY PLATED STEEL	45	4
+Brand#14	ECONOMY PLATED TIN	9	4
+Brand#14	ECONOMY POLISHED COPPER	3	4
+Brand#14	ECONOMY POLISHED TIN	19	4
+Brand#14	LARGE ANODIZED COPPER	9	4
+Brand#14	LARGE ANODIZED COPPER	23	4
+Brand#14	LARGE ANODIZED NICKEL	3	4
+Brand#14	LARGE ANODIZED NICKEL	9	4
+Brand#14	LARGE ANODIZED NICKEL	19	4
+Brand#14	LARGE ANODIZED TIN	9	4
+Brand#14	LARGE BRUSHED COPPER	14	4
+Brand#14	LARGE BRUSHED NICKEL	45	4
+Brand#14	LARGE PLATED BRASS	3	4
+Brand#14	LARGE PLATED NICKEL	3	4
+Brand#14	LARGE PLATED NICKEL	14	4
+Brand#14	LARGE PLATED NICKEL	49	4
+Brand#14	LARGE PLATED TIN	49	4
+Brand#14	LARGE POLISHED BRASS	9	4
+Brand#14	LARGE POLISHED BRASS	14	4
+Brand#14	LARGE POLISHED BRASS	36	4
+Brand#14	LARGE POLISHED NICKEL	3	4
+Brand#14	LARGE POLISHED NICKEL	14	4
+Brand#14	LARGE POLISHED STEEL	9	4
+Brand#14	LARGE POLISHED STEEL	23	4
+Brand#14	LARGE POLISHED STEEL	36	4
+Brand#14	MEDIUM ANODIZED NICKEL	3	4
+Brand#14	MEDIUM ANODIZED NICKEL	49	4
+Brand#14	MEDIUM ANODIZED STEEL	23	4
+Brand#14	MEDIUM ANODIZED STEEL	36	4
+Brand#14	MEDIUM BRUSHED BRASS	9	4
+Brand#14	MEDIUM BRUSHED COPPER	23	4
+Brand#14	MEDIUM BRUSHED STEEL	14	4
+Brand#14	MEDIUM BURNISHED COPPER	14	4
+Brand#14	MEDIUM BURNISHED STEEL	3	4
+Brand#14	MEDIUM BURNISHED STEEL	49	4
+Brand#14	MEDIUM PLATED BRASS	36	4
+Brand#14	MEDIUM PLATED STEEL	49	4
+Brand#14	MEDIUM PLATED TIN	14	4
+Brand#14	PROMO ANODIZED BRASS	49	4
+Brand#14	PROMO ANODIZED STEEL	36	4
+Brand#14	PROMO BRUSHED STEEL	19	4
+Brand#14	PROMO BURNISHED BRASS	23	4
+Brand#14	PROMO BURNISHED STEEL	36	4
+Brand#14	PROMO PLATED BRASS	9	4
+Brand#14	PROMO PLATED BRASS	45	4
+Brand#14	PROMO PLATED COPPER	45	4
+Brand#14	PROMO PLATED STEEL	3	4
+Brand#14	PROMO POLISHED BRASS	9	4
+Brand#14	PROMO POLISHED COPPER	49	4
+Brand#14	PROMO POLISHED STEEL	19	4
+Brand#14	SMALL ANODIZED STEEL	23	4
+Brand#14	SMALL ANODIZED TIN	23	4
+Brand#14	SMALL BRUSHED BRASS	19	4
+Brand#14	SMALL BRUSHED BRASS	36	4
+Brand#14	SMALL BRUSHED COPPER	9	4
+Brand#14	SMALL BRUSHED TIN	36	4
+Brand#14	SMALL BURNISHED BRASS	45	4
+Brand#14	SMALL BURNISHED COPPER	9	4
+Brand#14	SMALL BURNISHED COPPER	14	4
+Brand#14	SMALL BURNISHED COPPER	45	4
+Brand#14	SMALL BURNISHED NICKEL	36	4
+Brand#14	SMALL BURNISHED STEEL	36	4
+Brand#14	SMALL BURNISHED TIN	23	4
+Brand#14	SMALL PLATED NICKEL	3	4
+Brand#14	SMALL PLATED NICKEL	9	4
+Brand#14	SMALL PLATED STEEL	14	4
+Brand#14	SMALL POLISHED BRASS	36	4
+Brand#14	SMALL POLISHED COPPER	36	4
+Brand#14	SMALL POLISHED NICKEL	9	4
+Brand#14	SMALL POLISHED STEEL	14	4
+Brand#14	SMALL POLISHED TIN	14	4
+Brand#14	STANDARD ANODIZED BRASS	19	4
+Brand#14	STANDARD ANODIZED NICKEL	14	4
+Brand#14	STANDARD ANODIZED STEEL	9	4
+Brand#14	STANDARD BRUSHED COPPER	45	4
+Brand#14	STANDARD BRUSHED NICKEL	45	4
+Brand#14	STANDARD BRUSHED TIN	45	4
+Brand#14	STANDARD BURNISHED BRASS	9	4
+Brand#14	STANDARD BURNISHED BRASS	23	4
+Brand#14	STANDARD BURNISHED BRASS	49	4
+Brand#14	STANDARD BURNISHED NICKEL	9	4
+Brand#14	STANDARD PLATED BRASS	36	4
+Brand#14	STANDARD PLATED COPPER	45	4
+Brand#14	STANDARD POLISHED NICKEL	3	4
+Brand#14	STANDARD POLISHED NICKEL	9	4
+Brand#14	STANDARD POLISHED TIN	19	4
+Brand#15	ECONOMY ANODIZED COPPER	14	4
+Brand#15	ECONOMY ANODIZED STEEL	19	4
+Brand#15	ECONOMY ANODIZED STEEL	36	4
+Brand#15	ECONOMY BRUSHED BRASS	36	4
+Brand#15	ECONOMY BRUSHED COPPER	14	4
+Brand#15	ECONOMY BRUSHED NICKEL	14	4
+Brand#15	ECONOMY BRUSHED STEEL	3	4
+Brand#15	ECONOMY BRUSHED TIN	3	4
+Brand#15	ECONOMY BURNISHED BRASS	14	4
+Brand#15	ECONOMY BURNISHED COPPER	3	4
+Brand#15	ECONOMY BURNISHED COPPER	23	4
+Brand#15	ECONOMY PLATED NICKEL	49	4
+Brand#15	ECONOMY PLATED STEEL	3	4
+Brand#15	ECONOMY PLATED STEEL	19	4
+Brand#15	ECONOMY PLATED STEEL	45	4
+Brand#15	LARGE ANODIZED BRASS	19	4
+Brand#15	LARGE ANODIZED BRASS	36	4
+Brand#15	LARGE ANODIZED BRASS	45	4
+Brand#15	LARGE ANODIZED COPPER	3	4
+Brand#15	LARGE ANODIZED NICKEL	9	4
+Brand#15	LARGE ANODIZED TIN	19	4
+Brand#15	LARGE BRUSHED BRASS	9	4
+Brand#15	LARGE BRUSHED BRASS	19	4
+Brand#15	LARGE BRUSHED COPPER	14	4
+Brand#15	LARGE BRUSHED STEEL	9	4
+Brand#15	LARGE BRUSHED STEEL	14	4
+Brand#15	LARGE BRUSHED STEEL	19	4
+Brand#15	LARGE BRUSHED STEEL	36	4
+Brand#15	LARGE BURNISHED BRASS	14	4
+Brand#15	LARGE BURNISHED BRASS	19	4
+Brand#15	LARGE BURNISHED COPPER	9	4
+Brand#15	LARGE BURNISHED COPPER	45	4
+Brand#15	LARGE BURNISHED TIN	49	4
+Brand#15	LARGE PLATED BRASS	19	4
+Brand#15	LARGE PLATED COPPER	3	4
+Brand#15	LARGE PLATED COPPER	23	4
+Brand#15	LARGE PLATED NICKEL	36	4
+Brand#15	MEDIUM ANODIZED BRASS	23	4
+Brand#15	MEDIUM ANODIZED COPPER	9	4
+Brand#15	MEDIUM ANODIZED NICKEL	3	4
+Brand#15	MEDIUM ANODIZED TIN	19	4
+Brand#15	MEDIUM BRUSHED BRASS	9	4
+Brand#15	MEDIUM BRUSHED TIN	23	4
+Brand#15	MEDIUM BURNISHED COPPER	36	4
+Brand#15	MEDIUM BURNISHED TIN	45	4
+Brand#15	MEDIUM PLATED COPPER	9	4
+Brand#15	MEDIUM PLATED NICKEL	9	4
+Brand#15	MEDIUM PLATED NICKEL	19	4
+Brand#15	MEDIUM PLATED STEEL	36	4
+Brand#15	MEDIUM PLATED STEEL	49	4
+Brand#15	MEDIUM PLATED TIN	9	4
+Brand#15	MEDIUM PLATED TIN	14	4
+Brand#15	MEDIUM PLATED TIN	23	4
+Brand#15	PROMO ANODIZED COPPER	23	4
+Brand#15	PROMO ANODIZED STEEL	14	4
+Brand#15	PROMO ANODIZED TIN	45	4
+Brand#15	PROMO BRUSHED COPPER	14	4
+Brand#15	PROMO BRUSHED COPPER	19	4
+Brand#15	PROMO BRUSHED NICKEL	19	4
+Brand#15	PROMO BRUSHED NICKEL	23	4
+Brand#15	PROMO BRUSHED STEEL	14	4
+Brand#15	PROMO BRUSHED TIN	36	4
+Brand#15	PROMO BURNISHED NICKEL	9	4
+Brand#15	PROMO BURNISHED STEEL	45	4
+Brand#15	PROMO PLATED COPPER	3	4
+Brand#15	PROMO PLATED COPPER	36	4
+Brand#15	PROMO PLATED STEEL	3	4
+Brand#15	PROMO PLATED TIN	49	4
+Brand#15	PROMO POLISHED COPPER	3	4
+Brand#15	PROMO POLISHED NICKEL	36	4
+Brand#15	PROMO POLISHED STEEL	36	4
+Brand#15	PROMO POLISHED TIN	49	4
+Brand#15	SMALL ANODIZED BRASS	14	4
+Brand#15	SMALL ANODIZED BRASS	19	4
+Brand#15	SMALL ANODIZED COPPER	9	4
+Brand#15	SMALL ANODIZED TIN	45	4
+Brand#15	SMALL BRUSHED BRASS	3	4
+Brand#15	SMALL BRUSHED COPPER	19	4
+Brand#15	SMALL BRUSHED STEEL	23	4
+Brand#15	SMALL BRUSHED TIN	45	4
+Brand#15	SMALL BURNISHED BRASS	19	4
+Brand#15	SMALL BURNISHED COPPER	14	4
+Brand#15	SMALL BURNISHED NICKEL	19	4
+Brand#15	SMALL BURNISHED NICKEL	49	4
+Brand#15	SMALL BURNISHED STEEL	9	4
+Brand#15	SMALL BURNISHED TIN	19	4
+Brand#15	SMALL BURNISHED TIN	23	4
+Brand#15	SMALL BURNISHED TIN	36	4
+Brand#15	SMALL PLATED BRASS	3	4
+Brand#15	SMALL PLATED COPPER	23	4
+Brand#15	SMALL PLATED COPPER	49	4
+Brand#15	SMALL PLATED NICKEL	36	4
+Brand#15	SMALL PLATED NICKEL	45	4
+Brand#15	SMALL PLATED STEEL	3	4
+Brand#15	SMALL PLATED TIN	9	4
+Brand#15	SMALL POLISHED COPPER	9	4
+Brand#15	SMALL POLISHED NICKEL	3	4
+Brand#15	SMALL POLISHED STEEL	19	4
+Brand#15	SMALL POLISHED STEEL	36	4
+Brand#15	SMALL POLISHED TIN	19	4
+Brand#15	SMALL POLISHED TIN	49	4
+Brand#15	STANDARD ANODIZED NICKEL	19	4
+Brand#15	STANDARD ANODIZED NICKEL	49	4
+Brand#15	STANDARD ANODIZED TIN	36	4
+Brand#15	STANDARD BRUSHED NICKEL	3	4
+Brand#15	STANDARD BURNISHED BRASS	23	4
+Brand#15	STANDARD BURNISHED STEEL	3	4
+Brand#15	STANDARD BURNISHED STEEL	45	4
+Brand#15	STANDARD PLATED BRASS	36	4
+Brand#15	STANDARD PLATED COPPER	14	4
+Brand#15	STANDARD PLATED COPPER	23	4
+Brand#15	STANDARD PLATED NICKEL	19	4
+Brand#15	STANDARD PLATED TIN	45	4
+Brand#15	STANDARD POLISHED BRASS	14	4
+Brand#15	STANDARD POLISHED COPPER	23	4
+Brand#15	STANDARD POLISHED NICKEL	45	4
+Brand#21	ECONOMY ANODIZED BRASS	3	4
+Brand#21	ECONOMY ANODIZED NICKEL	14	4
+Brand#21	ECONOMY ANODIZED STEEL	19	4
+Brand#21	ECONOMY ANODIZED STEEL	23	4
+Brand#21	ECONOMY ANODIZED STEEL	49	4
+Brand#21	ECONOMY ANODIZED TIN	19	4
+Brand#21	ECONOMY BRUSHED BRASS	9	4
+Brand#21	ECONOMY BRUSHED BRASS	14	4
+Brand#21	ECONOMY BRUSHED BRASS	36	4
+Brand#21	ECONOMY BRUSHED COPPER	49	4
+Brand#21	ECONOMY BRUSHED STEEL	45	4
+Brand#21	ECONOMY BRUSHED TIN	49	4
+Brand#21	ECONOMY BURNISHED BRASS	3	4
+Brand#21	ECONOMY BURNISHED COPPER	45	4
+Brand#21	ECONOMY BURNISHED STEEL	19	4
+Brand#21	ECONOMY BURNISHED STEEL	36	4
+Brand#21	ECONOMY PLATED BRASS	36	4
+Brand#21	ECONOMY PLATED COPPER	3	4
+Brand#21	ECONOMY PLATED COPPER	14	4
+Brand#21	ECONOMY PLATED NICKEL	49	4
+Brand#21	ECONOMY POLISHED NICKEL	3	4
+Brand#21	ECONOMY POLISHED NICKEL	9	4
+Brand#21	LARGE ANODIZED COPPER	3	4
+Brand#21	LARGE ANODIZED COPPER	9	4
+Brand#21	LARGE ANODIZED STEEL	36	4
+Brand#21	LARGE ANODIZED TIN	45	4
+Brand#21	LARGE BRUSHED COPPER	45	4
+Brand#21	LARGE BRUSHED STEEL	23	4
+Brand#21	LARGE BURNISHED BRASS	49	4
+Brand#21	LARGE BURNISHED COPPER	19	4
+Brand#21	LARGE BURNISHED STEEL	49	4
+Brand#21	LARGE BURNISHED TIN	49	4
+Brand#21	LARGE PLATED BRASS	19	4
+Brand#21	LARGE PLATED NICKEL	23	4
+Brand#21	LARGE PLATED NICKEL	49	4
+Brand#21	LARGE PLATED TIN	19	4
+Brand#21	LARGE POLISHED BRASS	49	4
+Brand#21	LARGE POLISHED COPPER	14	4
+Brand#21	LARGE POLISHED NICKEL	3	4
+Brand#21	LARGE POLISHED NICKEL	14	4
+Brand#21	LARGE POLISHED STEEL	14	4
+Brand#21	LARGE POLISHED TIN	49	4
+Brand#21	MEDIUM ANODIZED COPPER	14	4
+Brand#21	MEDIUM ANODIZED NICKEL	49	4
+Brand#21	MEDIUM BRUSHED COPPER	3	4
+Brand#21	MEDIUM BRUSHED COPPER	49	4
+Brand#21	MEDIUM BRUSHED STEEL	23	4
+Brand#21	MEDIUM BRUSHED TIN	3	4
+Brand#21	MEDIUM BRUSHED TIN	14	4
+Brand#21	MEDIUM BURNISHED NICKEL	14	4
+Brand#21	MEDIUM BURNISHED STEEL	23	4
+Brand#21	MEDIUM BURNISHED TIN	3	4
+Brand#21	MEDIUM PLATED BRASS	3	4
+Brand#21	MEDIUM PLATED BRASS	19	4
+Brand#21	MEDIUM PLATED STEEL	36	4
+Brand#21	PROMO ANODIZED BRASS	9	4
+Brand#21	PROMO ANODIZED COPPER	14	4
+Brand#21	PROMO ANODIZED NICKEL	23	4
+Brand#21	PROMO ANODIZED STEEL	3	4
+Brand#21	PROMO ANODIZED STEEL	14	4
+Brand#21	PROMO ANODIZED STEEL	36	4
+Brand#21	PROMO BRUSHED NICKEL	45	4
+Brand#21	PROMO BRUSHED STEEL	14	4
+Brand#21	PROMO BRUSHED STEEL	23	4
+Brand#21	PROMO BRUSHED STEEL	45	4
+Brand#21	PROMO BURNISHED BRASS	19	4
+Brand#21	PROMO BURNISHED COPPER	19	4
+Brand#21	PROMO BURNISHED NICKEL	9	4
+Brand#21	PROMO BURNISHED TIN	19	4
+Brand#21	PROMO PLATED NICKEL	9	4
+Brand#21	PROMO PLATED NICKEL	36	4
+Brand#21	PROMO PLATED STEEL	49	4
+Brand#21	PROMO PLATED TIN	3	4
+Brand#21	PROMO POLISHED NICKEL	23	4
+Brand#21	PROMO POLISHED TIN	14	4
+Brand#21	PROMO POLISHED TIN	19	4
+Brand#21	PROMO POLISHED TIN	23	4
+Brand#21	SMALL BRUSHED BRASS	23	4
+Brand#21	SMALL BRUSHED COPPER	49	4
+Brand#21	SMALL BURNISHED BRASS	23	4
+Brand#21	SMALL BURNISHED BRASS	36	4
+Brand#21	SMALL BURNISHED STEEL	19	4
+Brand#21	SMALL BURNISHED TIN	19	4
+Brand#21	SMALL PLATED BRASS	45	4
+Brand#21	SMALL PLATED COPPER	45	4
+Brand#21	SMALL PLATED STEEL	45	4
+Brand#21	SMALL PLATED TIN	14	4
+Brand#21	SMALL PLATED TIN	45	4
+Brand#21	SMALL POLISHED COPPER	9	4
+Brand#21	SMALL POLISHED NICKEL	23	4
+Brand#21	SMALL POLISHED TIN	3	4
+Brand#21	STANDARD ANODIZED BRASS	9	4
+Brand#21	STANDARD ANODIZED NICKEL	19	4
+Brand#21	STANDARD ANODIZED TIN	45	4
+Brand#21	STANDARD BURNISHED COPPER	36	4
+Brand#21	STANDARD BURNISHED NICKEL	23	4
+Brand#21	STANDARD BURNISHED TIN	9	4
+Brand#21	STANDARD PLATED BRASS	14	4
+Brand#21	STANDARD PLATED COPPER	19	4
+Brand#21	STANDARD PLATED NICKEL	3	4
+Brand#21	STANDARD PLATED STEEL	9	4
+Brand#21	STANDARD PLATED TIN	9	4
+Brand#21	STANDARD POLISHED BRASS	9	4
+Brand#21	STANDARD POLISHED COPPER	49	4
+Brand#21	STANDARD POLISHED STEEL	36	4
+Brand#21	STANDARD POLISHED TIN	36	4
+Brand#22	ECONOMY ANODIZED STEEL	9	4
+Brand#22	ECONOMY ANODIZED STEEL	14	4
+Brand#22	ECONOMY ANODIZED STEEL	23	4
+Brand#22	ECONOMY ANODIZED TIN	9	4
+Brand#22	ECONOMY ANODIZED TIN	36	4
+Brand#22	ECONOMY BRUSHED NICKEL	36	4
+Brand#22	ECONOMY BRUSHED NICKEL	45	4
+Brand#22	ECONOMY BURNISHED BRASS	9	4
+Brand#22	ECONOMY BURNISHED BRASS	23	4
+Brand#22	ECONOMY BURNISHED BRASS	45	4
+Brand#22	ECONOMY BURNISHED NICKEL	19	4
+Brand#22	ECONOMY BURNISHED NICKEL	49	4
+Brand#22	ECONOMY BURNISHED STEEL	9	4
+Brand#22	ECONOMY BURNISHED STEEL	14	4
+Brand#22	ECONOMY BURNISHED STEEL	23	4
+Brand#22	ECONOMY PLATED BRASS	36	4
+Brand#22	ECONOMY PLATED COPPER	23	4
+Brand#22	ECONOMY PLATED TIN	3	4
+Brand#22	ECONOMY POLISHED TIN	49	4
+Brand#22	LARGE ANODIZED BRASS	19	4
+Brand#22	LARGE ANODIZED COPPER	36	4
+Brand#22	LARGE ANODIZED STEEL	3	4
+Brand#22	LARGE BRUSHED BRASS	23	4
+Brand#22	LARGE BRUSHED BRASS	49	4
+Brand#22	LARGE BRUSHED STEEL	49	4
+Brand#22	LARGE BURNISHED COPPER	19	4
+Brand#22	LARGE BURNISHED STEEL	23	4
+Brand#22	LARGE BURNISHED STEEL	45	4
+Brand#22	LARGE BURNISHED TIN	45	4
+Brand#22	LARGE PLATED COPPER	14	4
+Brand#22	LARGE PLATED STEEL	49	4
+Brand#22	LARGE POLISHED BRASS	19	4
+Brand#22	LARGE POLISHED COPPER	19	4
+Brand#22	LARGE POLISHED COPPER	23	4
+Brand#22	LARGE POLISHED NICKEL	19	4
+Brand#22	LARGE POLISHED TIN	49	4
+Brand#22	MEDIUM ANODIZED BRASS	45	4
+Brand#22	MEDIUM ANODIZED COPPER	19	4
+Brand#22	MEDIUM ANODIZED COPPER	49	4
+Brand#22	MEDIUM ANODIZED NICKEL	9	4
+Brand#22	MEDIUM ANODIZED NICKEL	14	4
+Brand#22	MEDIUM ANODIZED NICKEL	36	4
+Brand#22	MEDIUM ANODIZED TIN	3	4
+Brand#22	MEDIUM ANODIZED TIN	9	4
+Brand#22	MEDIUM BRUSHED BRASS	3	4
+Brand#22	MEDIUM BRUSHED BRASS	14	4
+Brand#22	MEDIUM BRUSHED COPPER	3	4
+Brand#22	MEDIUM BRUSHED COPPER	45	4
+Brand#22	MEDIUM BRUSHED NICKEL	14	4
+Brand#22	MEDIUM BRUSHED TIN	45	4
+Brand#22	MEDIUM BURNISHED COPPER	36	4
+Brand#22	MEDIUM BURNISHED TIN	19	4
+Brand#22	MEDIUM BURNISHED TIN	23	4
+Brand#22	MEDIUM BURNISHED TIN	49	4
+Brand#22	MEDIUM PLATED BRASS	49	4
+Brand#22	MEDIUM PLATED COPPER	9	4
+Brand#22	MEDIUM PLATED STEEL	3	4
+Brand#22	PROMO ANODIZED BRASS	9	4
+Brand#22	PROMO ANODIZED STEEL	36	4
+Brand#22	PROMO ANODIZED TIN	45	4
+Brand#22	PROMO BRUSHED BRASS	3	4
+Brand#22	PROMO BRUSHED BRASS	9	4
+Brand#22	PROMO BRUSHED BRASS	36	4
+Brand#22	PROMO BRUSHED STEEL	36	4
+Brand#22	PROMO BURNISHED BRASS	23	4
+Brand#22	PROMO BURNISHED COPPER	9	4
+Brand#22	PROMO PLATED BRASS	14	4
+Brand#22	PROMO PLATED BRASS	45	4
+Brand#22	PROMO PLATED NICKEL	3	4
+Brand#22	PROMO PLATED STEEL	19	4
+Brand#22	PROMO POLISHED BRASS	3	4
+Brand#22	PROMO POLISHED STEEL	14	4
+Brand#22	PROMO POLISHED STEEL	23	4
+Brand#22	SMALL ANODIZED TIN	36	4
+Brand#22	SMALL ANODIZED TIN	49	4
+Brand#22	SMALL BRUSHED NICKEL	3	4
+Brand#22	SMALL BRUSHED NICKEL	36	4
+Brand#22	SMALL BRUSHED NICKEL	45	4
+Brand#22	SMALL BRUSHED TIN	45	4
+Brand#22	SMALL BURNISHED STEEL	23	4
+Brand#22	SMALL BURNISHED TIN	14	4
+Brand#22	SMALL PLATED STEEL	3	4
+Brand#22	SMALL PLATED TIN	9	4
+Brand#22	SMALL PLATED TIN	36	4
+Brand#22	SMALL POLISHED BRASS	23	4
+Brand#22	SMALL POLISHED NICKEL	19	4
+Brand#22	STANDARD ANODIZED BRASS	14	4
+Brand#22	STANDARD ANODIZED BRASS	23	4
+Brand#22	STANDARD BRUSHED COPPER	49	4
+Brand#22	STANDARD BRUSHED NICKEL	3	4
+Brand#22	STANDARD BRUSHED NICKEL	23	4
+Brand#22	STANDARD BRUSHED STEEL	9	4
+Brand#22	STANDARD BRUSHED TIN	19	4
+Brand#22	STANDARD BURNISHED COPPER	45	4
+Brand#22	STANDARD BURNISHED NICKEL	3	4
+Brand#22	STANDARD BURNISHED NICKEL	14	4
+Brand#22	STANDARD BURNISHED NICKEL	45	4
+Brand#22	STANDARD BURNISHED STEEL	3	4
+Brand#22	STANDARD BURNISHED STEEL	36	4
+Brand#22	STANDARD BURNISHED STEEL	45	4
+Brand#22	STANDARD BURNISHED STEEL	49	4
+Brand#22	STANDARD PLATED BRASS	45	4
+Brand#22	STANDARD PLATED NICKEL	3	4
+Brand#22	STANDARD PLATED NICKEL	45	4
+Brand#22	STANDARD PLATED STEEL	14	4
+Brand#22	STANDARD PLATED TIN	19	4
+Brand#22	STANDARD PLATED TIN	49	4
+Brand#22	STANDARD POLISHED COPPER	9	4
+Brand#22	STANDARD POLISHED STEEL	49	4
+Brand#22	STANDARD POLISHED TIN	45	4
+Brand#23	ECONOMY ANODIZED NICKEL	49	4
+Brand#23	ECONOMY ANODIZED STEEL	14	4
+Brand#23	ECONOMY ANODIZED STEEL	49	4
+Brand#23	ECONOMY ANODIZED TIN	49	4
+Brand#23	ECONOMY BRUSHED BRASS	3	4
+Brand#23	ECONOMY BRUSHED COPPER	9	4
+Brand#23	ECONOMY BRUSHED TIN	9	4
+Brand#23	ECONOMY BURNISHED STEEL	49	4
+Brand#23	ECONOMY PLATED COPPER	14	4
+Brand#23	ECONOMY PLATED NICKEL	23	4
+Brand#23	ECONOMY PLATED STEEL	14	4
+Brand#23	ECONOMY POLISHED NICKEL	9	4
+Brand#23	LARGE ANODIZED BRASS	14	4
+Brand#23	LARGE ANODIZED COPPER	9	4
+Brand#23	LARGE ANODIZED COPPER	14	4
+Brand#23	LARGE ANODIZED COPPER	45	4
+Brand#23	LARGE ANODIZED STEEL	19	4
+Brand#23	LARGE ANODIZED STEEL	36	4
+Brand#23	LARGE ANODIZED STEEL	49	4
+Brand#23	LARGE ANODIZED TIN	9	4
+Brand#23	LARGE PLATED BRASS	9	4
+Brand#23	LARGE PLATED BRASS	49	4
+Brand#23	LARGE PLATED COPPER	3	4
+Brand#23	LARGE POLISHED BRASS	45	4
+Brand#23	LARGE POLISHED STEEL	9	4
+Brand#23	MEDIUM ANODIZED BRASS	19	4
+Brand#23	MEDIUM ANODIZED NICKEL	3	4
+Brand#23	MEDIUM ANODIZED NICKEL	14	4
+Brand#23	MEDIUM ANODIZED STEEL	45	4
+Brand#23	MEDIUM ANODIZED TIN	36	4
+Brand#23	MEDIUM ANODIZED TIN	45	4
+Brand#23	MEDIUM BRUSHED COPPER	3	4
+Brand#23	MEDIUM BRUSHED COPPER	23	4
+Brand#23	MEDIUM BRUSHED NICKEL	3	4
+Brand#23	MEDIUM BRUSHED TIN	14	4
+Brand#23	MEDIUM BURNISHED BRASS	9	4
+Brand#23	MEDIUM BURNISHED BRASS	45	4
+Brand#23	MEDIUM BURNISHED COPPER	19	4
+Brand#23	MEDIUM PLATED COPPER	19	4
+Brand#23	MEDIUM PLATED COPPER	36	4
+Brand#23	MEDIUM PLATED COPPER	45	4
+Brand#23	MEDIUM PLATED NICKEL	9	4
+Brand#23	MEDIUM PLATED NICKEL	14	4
+Brand#23	PROMO ANODIZED COPPER	9	4
+Brand#23	PROMO ANODIZED COPPER	19	4
+Brand#23	PROMO ANODIZED STEEL	36	4
+Brand#23	PROMO ANODIZED TIN	14	4
+Brand#23	PROMO BRUSHED BRASS	3	4
+Brand#23	PROMO BRUSHED BRASS	19	4
+Brand#23	PROMO BRUSHED BRASS	36	4
+Brand#23	PROMO BRUSHED COPPER	3	4
+Brand#23	PROMO BRUSHED TIN	49	4
+Brand#23	PROMO BURNISHED BRASS	14	4
+Brand#23	PROMO BURNISHED BRASS	45	4
+Brand#23	PROMO BURNISHED COPPER	14	4
+Brand#23	PROMO PLATED BRASS	23	4
+Brand#23	PROMO POLISHED BRASS	14	4
+Brand#23	PROMO POLISHED BRASS	23	4
+Brand#23	PROMO POLISHED COPPER	36	4
+Brand#23	PROMO POLISHED STEEL	36	4
+Brand#23	SMALL ANODIZED BRASS	23	4
+Brand#23	SMALL ANODIZED STEEL	23	4
+Brand#23	SMALL BRUSHED BRASS	49	4
+Brand#23	SMALL BRUSHED COPPER	45	4
+Brand#23	SMALL BRUSHED STEEL	3	4
+Brand#23	SMALL BRUSHED STEEL	19	4
+Brand#23	SMALL BURNISHED BRASS	36	4
+Brand#23	SMALL BURNISHED COPPER	45	4
+Brand#23	SMALL BURNISHED COPPER	49	4
+Brand#23	SMALL BURNISHED STEEL	45	4
+Brand#23	SMALL PLATED BRASS	36	4
+Brand#23	SMALL PLATED BRASS	49	4
+Brand#23	SMALL PLATED COPPER	14	4
+Brand#23	SMALL PLATED TIN	14	4
+Brand#23	SMALL POLISHED BRASS	9	4
+Brand#23	SMALL POLISHED BRASS	14	4
+Brand#23	SMALL POLISHED NICKEL	3	4
+Brand#23	SMALL POLISHED STEEL	14	4
+Brand#23	SMALL POLISHED TIN	9	4
+Brand#23	STANDARD ANODIZED BRASS	19	4
+Brand#23	STANDARD ANODIZED BRASS	45	4
+Brand#23	STANDARD ANODIZED COPPER	19	4
+Brand#23	STANDARD ANODIZED TIN	3	4
+Brand#23	STANDARD BRUSHED COPPER	36	4
+Brand#23	STANDARD BRUSHED NICKEL	19	4
+Brand#23	STANDARD BRUSHED STEEL	49	4
+Brand#23	STANDARD BURNISHED COPPER	19	4
+Brand#23	STANDARD PLATED BRASS	3	4
+Brand#23	STANDARD PLATED BRASS	9	4
+Brand#23	STANDARD PLATED STEEL	36	4
+Brand#23	STANDARD PLATED TIN	19	4
+Brand#23	STANDARD POLISHED BRASS	9	4
+Brand#23	STANDARD POLISHED BRASS	49	4
+Brand#23	STANDARD POLISHED STEEL	19	4
+Brand#23	STANDARD POLISHED STEEL	49	4
+Brand#23	STANDARD POLISHED TIN	23	4
+Brand#24	ECONOMY ANODIZED BRASS	3	4
+Brand#24	ECONOMY ANODIZED BRASS	9	4
+Brand#24	ECONOMY ANODIZED BRASS	23	4
+Brand#24	ECONOMY ANODIZED COPPER	9	4
+Brand#24	ECONOMY ANODIZED COPPER	49	4
+Brand#24	ECONOMY BRUSHED BRASS	36	4
+Brand#24	ECONOMY BRUSHED COPPER	23	4
+Brand#24	ECONOMY BURNISHED COPPER	3	4
+Brand#24	ECONOMY BURNISHED NICKEL	19	4
+Brand#24	ECONOMY BURNISHED STEEL	45	4
+Brand#24	ECONOMY PLATED BRASS	23	4
+Brand#24	ECONOMY PLATED COPPER	36	4
+Brand#24	ECONOMY PLATED STEEL	45	4
+Brand#24	ECONOMY POLISHED BRASS	23	4
+Brand#24	ECONOMY POLISHED COPPER	45	4
+Brand#24	ECONOMY POLISHED NICKEL	36	4
+Brand#24	ECONOMY POLISHED STEEL	14	4
+Brand#24	ECONOMY POLISHED STEEL	36	4
+Brand#24	LARGE ANODIZED NICKEL	23	4
+Brand#24	LARGE ANODIZED NICKEL	45	4
+Brand#24	LARGE ANODIZED TIN	45	4
+Brand#24	LARGE BRUSHED BRASS	14	4
+Brand#24	LARGE BRUSHED BRASS	23	4
+Brand#24	LARGE BRUSHED STEEL	9	4
+Brand#24	LARGE BRUSHED STEEL	23	4
+Brand#24	LARGE BRUSHED STEEL	45	4
+Brand#24	LARGE BRUSHED TIN	49	4
+Brand#24	LARGE BURNISHED BRASS	3	4
+Brand#24	LARGE BURNISHED NICKEL	19	4
+Brand#24	LARGE PLATED BRASS	9	4
+Brand#24	LARGE PLATED NICKEL	36	4
+Brand#24	LARGE PLATED NICKEL	49	4
+Brand#24	LARGE PLATED TIN	9	4
+Brand#24	LARGE PLATED TIN	19	4
+Brand#24	LARGE PLATED TIN	36	4
+Brand#24	LARGE PLATED TIN	49	4
+Brand#24	LARGE POLISHED BRASS	9	4
+Brand#24	LARGE POLISHED COPPER	9	4
+Brand#24	LARGE POLISHED COPPER	49	4
+Brand#24	LARGE POLISHED NICKEL	19	4
+Brand#24	LARGE POLISHED STEEL	23	4
+Brand#24	LARGE POLISHED TIN	14	4
+Brand#24	MEDIUM ANODIZED COPPER	45	4
+Brand#24	MEDIUM BRUSHED COPPER	9	4
+Brand#24	MEDIUM BRUSHED COPPER	14	4
+Brand#24	MEDIUM BRUSHED NICKEL	9	4
+Brand#24	MEDIUM BRUSHED NICKEL	23	4
+Brand#24	MEDIUM BRUSHED STEEL	14	4
+Brand#24	MEDIUM BRUSHED STEEL	45	4
+Brand#24	MEDIUM BRUSHED STEEL	49	4
+Brand#24	MEDIUM BURNISHED BRASS	36	4
+Brand#24	MEDIUM BURNISHED NICKEL	36	4
+Brand#24	MEDIUM BURNISHED STEEL	36	4
+Brand#24	MEDIUM PLATED COPPER	14	4
+Brand#24	MEDIUM PLATED STEEL	3	4
+Brand#24	MEDIUM PLATED STEEL	19	4
+Brand#24	PROMO ANODIZED NICKEL	9	4
+Brand#24	PROMO ANODIZED NICKEL	19	4
+Brand#24	PROMO ANODIZED NICKEL	45	4
+Brand#24	PROMO ANODIZED STEEL	3	4
+Brand#24	PROMO ANODIZED TIN	45	4
+Brand#24	PROMO BRUSHED BRASS	19	4
+Brand#24	PROMO BRUSHED NICKEL	19	4
+Brand#24	PROMO BRUSHED NICKEL	45	4
+Brand#24	PROMO BRUSHED STEEL	49	4
+Brand#24	PROMO BURNISHED BRASS	3	4
+Brand#24	PROMO BURNISHED BRASS	45	4
+Brand#24	PROMO BURNISHED STEEL	49	4
+Brand#24	PROMO PLATED BRASS	3	4
+Brand#24	PROMO PLATED COPPER	23	4
+Brand#24	PROMO PLATED COPPER	49	4
+Brand#24	PROMO POLISHED BRASS	3	4
+Brand#24	PROMO POLISHED BRASS	14	4
+Brand#24	PROMO POLISHED NICKEL	3	4
+Brand#24	PROMO POLISHED STEEL	14	4
+Brand#24	PROMO POLISHED STEEL	19	4
+Brand#24	PROMO POLISHED STEEL	23	4
+Brand#24	SMALL ANODIZED BRASS	19	4
+Brand#24	SMALL ANODIZED COPPER	3	4
+Brand#24	SMALL ANODIZED NICKEL	14	4
+Brand#24	SMALL ANODIZED STEEL	36	4
+Brand#24	SMALL ANODIZED TIN	3	4
+Brand#24	SMALL ANODIZED TIN	36	4
+Brand#24	SMALL BRUSHED COPPER	49	4
+Brand#24	SMALL BRUSHED NICKEL	49	4
+Brand#24	SMALL BURNISHED BRASS	14	4
+Brand#24	SMALL BURNISHED BRASS	19	4
+Brand#24	SMALL BURNISHED TIN	9	4
+Brand#24	SMALL PLATED BRASS	3	4
+Brand#24	SMALL PLATED COPPER	14	4
+Brand#24	SMALL PLATED COPPER	36	4
+Brand#24	SMALL PLATED NICKEL	14	4
+Brand#24	SMALL PLATED NICKEL	49	4
+Brand#24	SMALL POLISHED BRASS	3	4
+Brand#24	SMALL POLISHED NICKEL	9	4
+Brand#24	SMALL POLISHED NICKEL	19	4
+Brand#24	SMALL POLISHED NICKEL	36	4
+Brand#24	SMALL POLISHED STEEL	9	4
+Brand#24	SMALL POLISHED STEEL	36	4
+Brand#24	STANDARD ANODIZED TIN	9	4
+Brand#24	STANDARD ANODIZED TIN	49	4
+Brand#24	STANDARD BRUSHED BRASS	14	4
+Brand#24	STANDARD BRUSHED COPPER	23	4
+Brand#24	STANDARD BRUSHED NICKEL	19	4
+Brand#24	STANDARD BRUSHED STEEL	14	4
+Brand#24	STANDARD BRUSHED TIN	36	4
+Brand#24	STANDARD BURNISHED COPPER	19	4
+Brand#24	STANDARD BURNISHED COPPER	36	4
+Brand#24	STANDARD BURNISHED NICKEL	45	4
+Brand#24	STANDARD PLATED BRASS	36	4
+Brand#24	STANDARD PLATED COPPER	45	4
+Brand#24	STANDARD PLATED NICKEL	36	4
+Brand#24	STANDARD PLATED TIN	36	4
+Brand#24	STANDARD POLISHED COPPER	45	4
+Brand#24	STANDARD POLISHED NICKEL	14	4
+Brand#25	ECONOMY ANODIZED BRASS	14	4
+Brand#25	ECONOMY ANODIZED BRASS	49	4
+Brand#25	ECONOMY ANODIZED TIN	9	4
+Brand#25	ECONOMY ANODIZED TIN	19	4
+Brand#25	ECONOMY ANODIZED TIN	49	4
+Brand#25	ECONOMY BRUSHED COPPER	36	4
+Brand#25	ECONOMY BURNISHED COPPER	45	4
+Brand#25	ECONOMY BURNISHED TIN	19	4
+Brand#25	ECONOMY PLATED NICKEL	23	4
+Brand#25	ECONOMY PLATED TIN	14	4
+Brand#25	ECONOMY POLISHED BRASS	23	4
+Brand#25	ECONOMY POLISHED COPPER	9	4
+Brand#25	ECONOMY POLISHED NICKEL	3	4
+Brand#25	ECONOMY POLISHED TIN	9	4
+Brand#25	ECONOMY POLISHED TIN	45	4
+Brand#25	LARGE ANODIZED BRASS	3	4
+Brand#25	LARGE ANODIZED BRASS	14	4
+Brand#25	LARGE ANODIZED COPPER	36	4
+Brand#25	LARGE ANODIZED NICKEL	23	4
+Brand#25	LARGE ANODIZED STEEL	23	4
+Brand#25	LARGE BRUSHED NICKEL	19	4
+Brand#25	LARGE BRUSHED NICKEL	49	4
+Brand#25	LARGE BRUSHED TIN	3	4
+Brand#25	LARGE BRUSHED TIN	9	4
+Brand#25	LARGE BURNISHED BRASS	19	4
+Brand#25	LARGE BURNISHED BRASS	23	4
+Brand#25	LARGE BURNISHED BRASS	49	4
+Brand#25	LARGE BURNISHED NICKEL	14	4
+Brand#25	LARGE BURNISHED TIN	49	4
+Brand#25	LARGE PLATED BRASS	14	4
+Brand#25	LARGE PLATED NICKEL	23	4
+Brand#25	LARGE PLATED NICKEL	45	4
+Brand#25	LARGE PLATED TIN	19	4
+Brand#25	LARGE PLATED TIN	23	4
+Brand#25	LARGE POLISHED BRASS	9	4
+Brand#25	LARGE POLISHED COPPER	14	4
+Brand#25	LARGE POLISHED COPPER	36	4
+Brand#25	MEDIUM ANODIZED TIN	36	4
+Brand#25	MEDIUM BRUSHED COPPER	9	4
+Brand#25	MEDIUM BRUSHED COPPER	36	4
+Brand#25	MEDIUM BRUSHED COPPER	49	4
+Brand#25	MEDIUM BURNISHED COPPER	49	4
+Brand#25	MEDIUM BURNISHED NICKEL	9	4
+Brand#25	MEDIUM BURNISHED NICKEL	49	4
+Brand#25	MEDIUM BURNISHED STEEL	3	4
+Brand#25	MEDIUM BURNISHED STEEL	36	4
+Brand#25	MEDIUM BURNISHED STEEL	45	4
+Brand#25	MEDIUM BURNISHED STEEL	49	4
+Brand#25	MEDIUM BURNISHED TIN	9	4
+Brand#25	MEDIUM BURNISHED TIN	36	4
+Brand#25	MEDIUM PLATED BRASS	45	4
+Brand#25	MEDIUM PLATED COPPER	14	4
+Brand#25	MEDIUM PLATED NICKEL	45	4
+Brand#25	MEDIUM PLATED STEEL	9	4
+Brand#25	MEDIUM PLATED STEEL	36	4
+Brand#25	PROMO ANODIZED COPPER	14	4
+Brand#25	PROMO ANODIZED COPPER	19	4
+Brand#25	PROMO ANODIZED STEEL	36	4
+Brand#25	PROMO ANODIZED TIN	3	4
+Brand#25	PROMO ANODIZED TIN	14	4
+Brand#25	PROMO BRUSHED NICKEL	3	4
+Brand#25	PROMO BRUSHED STEEL	19	4
+Brand#25	PROMO BRUSHED TIN	14	4
+Brand#25	PROMO BRUSHED TIN	36	4
+Brand#25	PROMO BURNISHED COPPER	19	4
+Brand#25	PROMO BURNISHED COPPER	45	4
+Brand#25	PROMO BURNISHED COPPER	49	4
+Brand#25	PROMO BURNISHED NICKEL	36	4
+Brand#25	PROMO BURNISHED TIN	3	4
+Brand#25	PROMO PLATED BRASS	45	4
+Brand#25	PROMO PLATED COPPER	19	4
+Brand#25	PROMO PLATED NICKEL	45	4
+Brand#25	PROMO PLATED NICKEL	49	4
+Brand#25	PROMO PLATED STEEL	23	4
+Brand#25	PROMO POLISHED BRASS	23	4
+Brand#25	SMALL ANODIZED BRASS	45	4
+Brand#25	SMALL ANODIZED NICKEL	19	4
+Brand#25	SMALL ANODIZED STEEL	23	4
+Brand#25	SMALL ANODIZED TIN	14	4
+Brand#25	SMALL ANODIZED TIN	19	4
+Brand#25	SMALL BRUSHED COPPER	45	4
+Brand#25	SMALL BRUSHED NICKEL	9	4
+Brand#25	SMALL BURNISHED COPPER	3	4
+Brand#25	SMALL BURNISHED STEEL	3	4
+Brand#25	SMALL BURNISHED STEEL	14	4
+Brand#25	SMALL BURNISHED TIN	3	4
+Brand#25	SMALL PLATED BRASS	19	4
+Brand#25	SMALL PLATED COPPER	23	4
+Brand#25	SMALL PLATED STEEL	45	4
+Brand#25	SMALL PLATED TIN	36	4
+Brand#25	SMALL POLISHED BRASS	23	4
+Brand#25	SMALL POLISHED COPPER	9	4
+Brand#25	SMALL POLISHED STEEL	14	4
+Brand#25	STANDARD ANODIZED STEEL	3	4
+Brand#25	STANDARD ANODIZED STEEL	19	4
+Brand#25	STANDARD ANODIZED TIN	9	4
+Brand#25	STANDARD BRUSHED BRASS	14	4
+Brand#25	STANDARD BRUSHED NICKEL	19	4
+Brand#25	STANDARD BRUSHED TIN	9	4
+Brand#25	STANDARD BURNISHED NICKEL	9	4
+Brand#25	STANDARD PLATED BRASS	3	4
+Brand#25	STANDARD PLATED COPPER	14	4
+Brand#25	STANDARD PLATED NICKEL	36	4
+Brand#25	STANDARD POLISHED BRASS	45	4
+Brand#25	STANDARD POLISHED COPPER	23	4
+Brand#25	STANDARD POLISHED NICKEL	3	4
+Brand#25	STANDARD POLISHED NICKEL	49	4
+Brand#25	STANDARD POLISHED TIN	36	4
+Brand#25	STANDARD POLISHED TIN	45	4
+Brand#31	ECONOMY ANODIZED BRASS	3	4
+Brand#31	ECONOMY ANODIZED COPPER	45	4
+Brand#31	ECONOMY ANODIZED STEEL	3	4
+Brand#31	ECONOMY ANODIZED TIN	45	4
+Brand#31	ECONOMY BRUSHED BRASS	14	4
+Brand#31	ECONOMY BRUSHED COPPER	19	4
+Brand#31	ECONOMY BRUSHED NICKEL	9	4
+Brand#31	ECONOMY BRUSHED NICKEL	14	4
+Brand#31	ECONOMY BRUSHED NICKEL	49	4
+Brand#31	ECONOMY BURNISHED COPPER	36	4
+Brand#31	ECONOMY BURNISHED STEEL	3	4
+Brand#31	ECONOMY BURNISHED TIN	49	4
+Brand#31	ECONOMY PLATED COPPER	49	4
+Brand#31	ECONOMY PLATED NICKEL	9	4
+Brand#31	ECONOMY PLATED STEEL	23	4
+Brand#31	ECONOMY PLATED TIN	36	4
+Brand#31	ECONOMY PLATED TIN	49	4
+Brand#31	ECONOMY POLISHED COPPER	3	4
+Brand#31	ECONOMY POLISHED COPPER	36	4
+Brand#31	ECONOMY POLISHED COPPER	49	4
+Brand#31	ECONOMY POLISHED NICKEL	3	4
+Brand#31	LARGE ANODIZED BRASS	19	4
+Brand#31	LARGE ANODIZED STEEL	45	4
+Brand#31	LARGE BRUSHED BRASS	36	4
+Brand#31	LARGE BRUSHED BRASS	49	4
+Brand#31	LARGE BRUSHED TIN	3	4
+Brand#31	LARGE BURNISHED BRASS	9	4
+Brand#31	LARGE PLATED COPPER	19	4
+Brand#31	LARGE PLATED NICKEL	14	4
+Brand#31	LARGE PLATED TIN	9	4
+Brand#31	LARGE PLATED TIN	14	4
+Brand#31	LARGE POLISHED BRASS	14	4
+Brand#31	LARGE POLISHED STEEL	14	4
+Brand#31	LARGE POLISHED STEEL	45	4
+Brand#31	LARGE POLISHED TIN	19	4
+Brand#31	MEDIUM ANODIZED BRASS	23	4
+Brand#31	MEDIUM ANODIZED BRASS	36	4
+Brand#31	MEDIUM ANODIZED COPPER	14	4
+Brand#31	MEDIUM ANODIZED COPPER	19	4
+Brand#31	MEDIUM ANODIZED COPPER	36	4
+Brand#31	MEDIUM ANODIZED STEEL	14	4
+Brand#31	MEDIUM ANODIZED STEEL	49	4
+Brand#31	MEDIUM ANODIZED TIN	19	4
+Brand#31	MEDIUM ANODIZED TIN	49	4
+Brand#31	MEDIUM BRUSHED BRASS	36	4
+Brand#31	MEDIUM BRUSHED STEEL	14	4
+Brand#31	MEDIUM BURNISHED BRASS	14	4
+Brand#31	MEDIUM BURNISHED COPPER	3	4
+Brand#31	MEDIUM BURNISHED NICKEL	9	4
+Brand#31	MEDIUM BURNISHED STEEL	9	4
+Brand#31	MEDIUM BURNISHED TIN	14	4
+Brand#31	MEDIUM BURNISHED TIN	23	4
+Brand#31	MEDIUM PLATED BRASS	3	4
+Brand#31	MEDIUM PLATED TIN	9	4
+Brand#31	MEDIUM PLATED TIN	36	4
+Brand#31	MEDIUM PLATED TIN	45	4
+Brand#31	PROMO ANODIZED BRASS	3	4
+Brand#31	PROMO ANODIZED NICKEL	9	4
+Brand#31	PROMO BRUSHED BRASS	3	4
+Brand#31	PROMO BRUSHED BRASS	23	4
+Brand#31	PROMO BRUSHED COPPER	23	4
+Brand#31	PROMO BRUSHED NICKEL	45	4
+Brand#31	PROMO BURNISHED COPPER	36	4
+Brand#31	PROMO BURNISHED STEEL	3	4
+Brand#31	PROMO BURNISHED TIN	3	4
+Brand#31	PROMO PLATED BRASS	19	4
+Brand#31	PROMO PLATED NICKEL	36	4
+Brand#31	PROMO POLISHED BRASS	49	4
+Brand#31	PROMO POLISHED COPPER	14	4
+Brand#31	PROMO POLISHED NICKEL	3	4
+Brand#31	PROMO POLISHED NICKEL	9	4
+Brand#31	PROMO POLISHED TIN	3	4
+Brand#31	PROMO POLISHED TIN	23	4
+Brand#31	SMALL ANODIZED COPPER	45	4
+Brand#31	SMALL ANODIZED STEEL	23	4
+Brand#31	SMALL ANODIZED TIN	3	4
+Brand#31	SMALL BRUSHED COPPER	36	4
+Brand#31	SMALL BRUSHED COPPER	49	4
+Brand#31	SMALL BRUSHED NICKEL	19	4
+Brand#31	SMALL BRUSHED NICKEL	23	4
+Brand#31	SMALL BURNISHED BRASS	45	4
+Brand#31	SMALL BURNISHED NICKEL	9	4
+Brand#31	SMALL BURNISHED NICKEL	36	4
+Brand#31	SMALL PLATED COPPER	36	4
+Brand#31	SMALL PLATED NICKEL	9	4
+Brand#31	SMALL PLATED NICKEL	36	4
+Brand#31	SMALL POLISHED BRASS	3	4
+Brand#31	SMALL POLISHED COPPER	45	4
+Brand#31	SMALL POLISHED NICKEL	45	4
+Brand#31	SMALL POLISHED TIN	23	4
+Brand#31	SMALL POLISHED TIN	49	4
+Brand#31	STANDARD BRUSHED STEEL	23	4
+Brand#31	STANDARD BRUSHED STEEL	49	4
+Brand#31	STANDARD BURNISHED BRASS	14	4
+Brand#31	STANDARD BURNISHED NICKEL	45	4
+Brand#31	STANDARD PLATED NICKEL	3	4
+Brand#31	STANDARD POLISHED BRASS	3	4
+Brand#31	STANDARD POLISHED BRASS	45	4
+Brand#31	STANDARD POLISHED STEEL	36	4
+Brand#32	ECONOMY ANODIZED BRASS	19	4
+Brand#32	ECONOMY ANODIZED COPPER	36	4
+Brand#32	ECONOMY ANODIZED STEEL	23	4
+Brand#32	ECONOMY ANODIZED STEEL	36	4
+Brand#32	ECONOMY ANODIZED STEEL	45	4
+Brand#32	ECONOMY ANODIZED TIN	19	4
+Brand#32	ECONOMY BRUSHED COPPER	45	4
+Brand#32	ECONOMY BRUSHED TIN	45	4
+Brand#32	ECONOMY BURNISHED BRASS	23	4
+Brand#32	ECONOMY BURNISHED COPPER	36	4
+Brand#32	ECONOMY BURNISHED COPPER	45	4
+Brand#32	ECONOMY BURNISHED STEEL	19	4
+Brand#32	ECONOMY PLATED BRASS	9	4
+Brand#32	ECONOMY PLATED COPPER	9	4
+Brand#32	ECONOMY PLATED NICKEL	23	4
+Brand#32	ECONOMY PLATED TIN	45	4
+Brand#32	ECONOMY POLISHED STEEL	3	4
+Brand#32	LARGE ANODIZED BRASS	23	4
+Brand#32	LARGE ANODIZED BRASS	36	4
+Brand#32	LARGE ANODIZED NICKEL	45	4
+Brand#32	LARGE ANODIZED STEEL	3	4
+Brand#32	LARGE ANODIZED STEEL	14	4
+Brand#32	LARGE BRUSHED STEEL	45	4
+Brand#32	LARGE BRUSHED TIN	45	4
+Brand#32	LARGE BURNISHED NICKEL	36	4
+Brand#32	LARGE BURNISHED TIN	19	4
+Brand#32	LARGE BURNISHED TIN	45	4
+Brand#32	LARGE PLATED BRASS	3	4
+Brand#32	LARGE PLATED NICKEL	49	4
+Brand#32	LARGE PLATED STEEL	19	4
+Brand#32	LARGE PLATED STEEL	36	4
+Brand#32	LARGE POLISHED BRASS	45	4
+Brand#32	LARGE POLISHED COPPER	9	4
+Brand#32	LARGE POLISHED COPPER	49	4
+Brand#32	LARGE POLISHED NICKEL	3	4
+Brand#32	MEDIUM ANODIZED BRASS	3	4
+Brand#32	MEDIUM ANODIZED BRASS	9	4
+Brand#32	MEDIUM ANODIZED TIN	23	4
+Brand#32	MEDIUM BRUSHED BRASS	23	4
+Brand#32	MEDIUM BRUSHED BRASS	49	4
+Brand#32	MEDIUM BRUSHED COPPER	9	4
+Brand#32	MEDIUM BRUSHED COPPER	19	4
+Brand#32	MEDIUM BRUSHED TIN	49	4
+Brand#32	MEDIUM BURNISHED BRASS	9	4
+Brand#32	MEDIUM BURNISHED BRASS	36	4
+Brand#32	MEDIUM BURNISHED BRASS	49	4
+Brand#32	MEDIUM BURNISHED COPPER	9	4
+Brand#32	MEDIUM BURNISHED COPPER	45	4
+Brand#32	MEDIUM BURNISHED NICKEL	49	4
+Brand#32	MEDIUM BURNISHED TIN	9	4
+Brand#32	MEDIUM BURNISHED TIN	45	4
+Brand#32	MEDIUM PLATED BRASS	3	4
+Brand#32	MEDIUM PLATED BRASS	49	4
+Brand#32	MEDIUM PLATED COPPER	3	4
+Brand#32	MEDIUM PLATED STEEL	9	4
+Brand#32	MEDIUM PLATED TIN	9	4
+Brand#32	PROMO ANODIZED BRASS	3	4
+Brand#32	PROMO ANODIZED COPPER	19	4
+Brand#32	PROMO ANODIZED NICKEL	23	4
+Brand#32	PROMO BRUSHED COPPER	23	4
+Brand#32	PROMO BRUSHED NICKEL	14	4
+Brand#32	PROMO BRUSHED NICKEL	36	4
+Brand#32	PROMO BRUSHED STEEL	14	4
+Brand#32	PROMO BRUSHED STEEL	23	4
+Brand#32	PROMO BRUSHED STEEL	49	4
+Brand#32	PROMO BURNISHED BRASS	45	4
+Brand#32	PROMO BURNISHED NICKEL	45	4
+Brand#32	PROMO BURNISHED TIN	14	4
+Brand#32	PROMO BURNISHED TIN	45	4
+Brand#32	PROMO PLATED TIN	19	4
+Brand#32	PROMO POLISHED NICKEL	36	4
+Brand#32	PROMO POLISHED TIN	3	4
+Brand#32	SMALL ANODIZED BRASS	3	4
+Brand#32	SMALL ANODIZED NICKEL	3	4
+Brand#32	SMALL ANODIZED NICKEL	14	4
+Brand#32	SMALL ANODIZED TIN	9	4
+Brand#32	SMALL BRUSHED BRASS	9	4
+Brand#32	SMALL BRUSHED BRASS	19	4
+Brand#32	SMALL BRUSHED COPPER	3	4
+Brand#32	SMALL BRUSHED COPPER	23	4
+Brand#32	SMALL BRUSHED NICKEL	9	4
+Brand#32	SMALL BRUSHED NICKEL	45	4
+Brand#32	SMALL BRUSHED STEEL	23	4
+Brand#32	SMALL BRUSHED TIN	9	4
+Brand#32	SMALL BURNISHED NICKEL	36	4
+Brand#32	SMALL BURNISHED STEEL	3	4
+Brand#32	SMALL BURNISHED TIN	23	4
+Brand#32	SMALL PLATED BRASS	49	4
+Brand#32	SMALL PLATED COPPER	36	4
+Brand#32	SMALL PLATED COPPER	45	4
+Brand#32	SMALL PLATED NICKEL	45	4
+Brand#32	SMALL PLATED STEEL	45	4
+Brand#32	SMALL PLATED TIN	23	4
+Brand#32	SMALL PLATED TIN	36	4
+Brand#32	SMALL PLATED TIN	45	4
+Brand#32	SMALL POLISHED NICKEL	36	4
+Brand#32	SMALL POLISHED STEEL	14	4
+Brand#32	SMALL POLISHED STEEL	23	4
+Brand#32	SMALL POLISHED STEEL	36	4
+Brand#32	SMALL POLISHED TIN	36	4
+Brand#32	SMALL POLISHED TIN	45	4
+Brand#32	STANDARD ANODIZED NICKEL	19	4
+Brand#32	STANDARD ANODIZED TIN	9	4
+Brand#32	STANDARD ANODIZED TIN	14	4
+Brand#32	STANDARD ANODIZED TIN	19	4
+Brand#32	STANDARD BRUSHED NICKEL	23	4
+Brand#32	STANDARD BURNISHED BRASS	36	4
+Brand#32	STANDARD BURNISHED BRASS	45	4
+Brand#32	STANDARD BURNISHED COPPER	3	4
+Brand#32	STANDARD BURNISHED COPPER	36	4
+Brand#32	STANDARD BURNISHED NICKEL	49	4
+Brand#32	STANDARD BURNISHED STEEL	49	4
+Brand#32	STANDARD BURNISHED TIN	23	4
+Brand#32	STANDARD PLATED BRASS	9	4
+Brand#32	STANDARD PLATED BRASS	45	4
+Brand#32	STANDARD PLATED STEEL	36	4
+Brand#32	STANDARD POLISHED BRASS	14	4
+Brand#32	STANDARD POLISHED COPPER	36	4
+Brand#32	STANDARD POLISHED STEEL	14	4
+Brand#33	ECONOMY ANODIZED BRASS	23	4
+Brand#33	ECONOMY ANODIZED COPPER	9	4
+Brand#33	ECONOMY ANODIZED NICKEL	3	4
+Brand#33	ECONOMY ANODIZED NICKEL	9	4
+Brand#33	ECONOMY ANODIZED NICKEL	23	4
+Brand#33	ECONOMY ANODIZED NICKEL	36	4
+Brand#33	ECONOMY BRUSHED BRASS	14	4
+Brand#33	ECONOMY BRUSHED COPPER	23	4
+Brand#33	ECONOMY BURNISHED BRASS	49	4
+Brand#33	ECONOMY BURNISHED COPPER	3	4
+Brand#33	ECONOMY BURNISHED COPPER	14	4
+Brand#33	ECONOMY BURNISHED STEEL	3	4
+Brand#33	ECONOMY BURNISHED TIN	36	4
+Brand#33	ECONOMY BURNISHED TIN	45	4
+Brand#33	ECONOMY PLATED COPPER	19	4
+Brand#33	ECONOMY PLATED COPPER	45	4
+Brand#33	ECONOMY PLATED NICKEL	14	4
+Brand#33	ECONOMY PLATED NICKEL	36	4
+Brand#33	ECONOMY PLATED STEEL	3	4
+Brand#33	ECONOMY PLATED STEEL	23	4
+Brand#33	ECONOMY PLATED STEEL	36	4
+Brand#33	ECONOMY POLISHED BRASS	14	4
+Brand#33	ECONOMY POLISHED NICKEL	19	4
+Brand#33	ECONOMY POLISHED TIN	9	4
+Brand#33	LARGE ANODIZED BRASS	36	4
+Brand#33	LARGE ANODIZED COPPER	19	4
+Brand#33	LARGE ANODIZED COPPER	45	4
+Brand#33	LARGE ANODIZED NICKEL	36	4
+Brand#33	LARGE ANODIZED NICKEL	45	4
+Brand#33	LARGE ANODIZED STEEL	3	4
+Brand#33	LARGE ANODIZED STEEL	45	4
+Brand#33	LARGE ANODIZED TIN	45	4
+Brand#33	LARGE BRUSHED BRASS	3	4
+Brand#33	LARGE BRUSHED BRASS	49	4
+Brand#33	LARGE BRUSHED STEEL	19	4
+Brand#33	LARGE BRUSHED TIN	36	4
+Brand#33	LARGE BURNISHED COPPER	45	4
+Brand#33	LARGE BURNISHED NICKEL	23	4
+Brand#33	LARGE BURNISHED STEEL	19	4
+Brand#33	LARGE PLATED BRASS	3	4
+Brand#33	LARGE PLATED COPPER	19	4
+Brand#33	LARGE PLATED STEEL	3	4
+Brand#33	LARGE PLATED STEEL	19	4
+Brand#33	LARGE PLATED TIN	45	4
+Brand#33	LARGE POLISHED BRASS	45	4
+Brand#33	LARGE POLISHED STEEL	14	4
+Brand#33	LARGE POLISHED STEEL	23	4
+Brand#33	LARGE POLISHED TIN	23	4
+Brand#33	MEDIUM ANODIZED BRASS	3	4
+Brand#33	MEDIUM ANODIZED COPPER	9	4
+Brand#33	MEDIUM ANODIZED COPPER	36	4
+Brand#33	MEDIUM ANODIZED COPPER	49	4
+Brand#33	MEDIUM ANODIZED NICKEL	3	4
+Brand#33	MEDIUM ANODIZED NICKEL	19	4
+Brand#33	MEDIUM BRUSHED BRASS	3	4
+Brand#33	MEDIUM BRUSHED STEEL	19	4
+Brand#33	MEDIUM BRUSHED TIN	14	4
+Brand#33	MEDIUM BURNISHED COPPER	14	4
+Brand#33	MEDIUM BURNISHED COPPER	49	4
+Brand#33	MEDIUM BURNISHED TIN	36	4
+Brand#33	MEDIUM PLATED BRASS	3	4
+Brand#33	MEDIUM PLATED STEEL	3	4
+Brand#33	MEDIUM PLATED STEEL	49	4
+Brand#33	PROMO ANODIZED BRASS	3	4
+Brand#33	PROMO BRUSHED BRASS	49	4
+Brand#33	PROMO BURNISHED COPPER	23	4
+Brand#33	PROMO BURNISHED NICKEL	14	4
+Brand#33	PROMO BURNISHED NICKEL	36	4
+Brand#33	PROMO BURNISHED TIN	19	4
+Brand#33	PROMO BURNISHED TIN	23	4
+Brand#33	PROMO PLATED COPPER	14	4
+Brand#33	PROMO PLATED STEEL	45	4
+Brand#33	PROMO PLATED STEEL	49	4
+Brand#33	PROMO PLATED TIN	49	4
+Brand#33	PROMO POLISHED COPPER	3	4
+Brand#33	PROMO POLISHED STEEL	3	4
+Brand#33	PROMO POLISHED STEEL	9	4
+Brand#33	PROMO POLISHED STEEL	23	4
+Brand#33	SMALL ANODIZED BRASS	19	4
+Brand#33	SMALL ANODIZED COPPER	23	4
+Brand#33	SMALL ANODIZED COPPER	49	4
+Brand#33	SMALL ANODIZED STEEL	9	4
+Brand#33	SMALL BRUSHED BRASS	3	4
+Brand#33	SMALL BRUSHED COPPER	3	4
+Brand#33	SMALL BRUSHED NICKEL	45	4
+Brand#33	SMALL BRUSHED STEEL	3	4
+Brand#33	SMALL BRUSHED TIN	9	4
+Brand#33	SMALL BURNISHED BRASS	19	4
+Brand#33	SMALL BURNISHED NICKEL	3	4
+Brand#33	SMALL PLATED BRASS	3	4
+Brand#33	SMALL PLATED STEEL	14	4
+Brand#33	SMALL PLATED STEEL	45	4
+Brand#33	SMALL PLATED TIN	23	4
+Brand#33	SMALL PLATED TIN	36	4
+Brand#33	SMALL POLISHED NICKEL	23	4
+Brand#33	SMALL POLISHED TIN	19	4
+Brand#33	SMALL POLISHED TIN	23	4
+Brand#33	SMALL POLISHED TIN	45	4
+Brand#33	STANDARD ANODIZED COPPER	49	4
+Brand#33	STANDARD ANODIZED STEEL	14	4
+Brand#33	STANDARD ANODIZED STEEL	45	4
+Brand#33	STANDARD ANODIZED STEEL	49	4
+Brand#33	STANDARD ANODIZED TIN	45	4
+Brand#33	STANDARD BRUSHED BRASS	9	4
+Brand#33	STANDARD BRUSHED NICKEL	45	4
+Brand#33	STANDARD BRUSHED STEEL	9	4
+Brand#33	STANDARD BRUSHED TIN	36	4
+Brand#33	STANDARD BURNISHED BRASS	9	4
+Brand#33	STANDARD BURNISHED BRASS	23	4
+Brand#33	STANDARD BURNISHED NICKEL	49	4
+Brand#33	STANDARD PLATED BRASS	49	4
+Brand#33	STANDARD PLATED COPPER	3	4
+Brand#33	STANDARD PLATED COPPER	14	4
+Brand#33	STANDARD PLATED NICKEL	36	4
+Brand#33	STANDARD PLATED STEEL	3	4
+Brand#33	STANDARD PLATED STEEL	36	4
+Brand#33	STANDARD PLATED TIN	14	4
+Brand#33	STANDARD POLISHED BRASS	9	4
+Brand#33	STANDARD POLISHED BRASS	19	4
+Brand#33	STANDARD POLISHED STEEL	3	4
+Brand#33	STANDARD POLISHED STEEL	9	4
+Brand#33	STANDARD POLISHED STEEL	14	4
+Brand#34	ECONOMY ANODIZED BRASS	9	4
+Brand#34	ECONOMY ANODIZED COPPER	3	4
+Brand#34	ECONOMY ANODIZED COPPER	14	4
+Brand#34	ECONOMY ANODIZED COPPER	19	4
+Brand#34	ECONOMY ANODIZED STEEL	9	4
+Brand#34	ECONOMY ANODIZED TIN	49	4
+Brand#34	ECONOMY BRUSHED BRASS	14	4
+Brand#34	ECONOMY BRUSHED NICKEL	49	4
+Brand#34	ECONOMY BURNISHED COPPER	9	4
+Brand#34	ECONOMY BURNISHED STEEL	19	4
+Brand#34	ECONOMY BURNISHED TIN	3	4
+Brand#34	ECONOMY BURNISHED TIN	23	4
+Brand#34	ECONOMY PLATED BRASS	9	4
+Brand#34	ECONOMY PLATED BRASS	14	4
+Brand#34	ECONOMY PLATED COPPER	3	4
+Brand#34	ECONOMY PLATED NICKEL	45	4
+Brand#34	ECONOMY PLATED TIN	14	4
+Brand#34	ECONOMY PLATED TIN	45	4
+Brand#34	ECONOMY POLISHED BRASS	45	4
+Brand#34	LARGE ANODIZED BRASS	14	4
+Brand#34	LARGE ANODIZED BRASS	23	4
+Brand#34	LARGE ANODIZED BRASS	36	4
+Brand#34	LARGE ANODIZED NICKEL	3	4
+Brand#34	LARGE ANODIZED TIN	49	4
+Brand#34	LARGE BRUSHED BRASS	49	4
+Brand#34	LARGE BRUSHED COPPER	23	4
+Brand#34	LARGE BRUSHED NICKEL	23	4
+Brand#34	LARGE BRUSHED STEEL	14	4
+Brand#34	LARGE BRUSHED STEEL	19	4
+Brand#34	LARGE BRUSHED TIN	9	4
+Brand#34	LARGE BURNISHED BRASS	23	4
+Brand#34	LARGE BURNISHED COPPER	3	4
+Brand#34	LARGE BURNISHED COPPER	36	4
+Brand#34	LARGE BURNISHED NICKEL	19	4
+Brand#34	LARGE PLATED BRASS	23	4
+Brand#34	LARGE PLATED BRASS	36	4
+Brand#34	LARGE PLATED BRASS	45	4
+Brand#34	LARGE PLATED COPPER	23	4
+Brand#34	LARGE PLATED COPPER	49	4
+Brand#34	LARGE PLATED STEEL	49	4
+Brand#34	LARGE POLISHED NICKEL	49	4
+Brand#34	MEDIUM ANODIZED COPPER	36	4
+Brand#34	MEDIUM ANODIZED TIN	3	4
+Brand#34	MEDIUM BRUSHED BRASS	49	4
+Brand#34	MEDIUM BRUSHED COPPER	9	4
+Brand#34	MEDIUM BRUSHED NICKEL	9	4
+Brand#34	MEDIUM BRUSHED NICKEL	23	4
+Brand#34	MEDIUM BRUSHED TIN	3	4
+Brand#34	MEDIUM BRUSHED TIN	14	4
+Brand#34	MEDIUM BURNISHED STEEL	45	4
+Brand#34	MEDIUM BURNISHED STEEL	49	4
+Brand#34	MEDIUM PLATED COPPER	36	4
+Brand#34	MEDIUM PLATED TIN	3	4
+Brand#34	MEDIUM PLATED TIN	14	4
+Brand#34	PROMO ANODIZED COPPER	45	4
+Brand#34	PROMO ANODIZED NICKEL	14	4
+Brand#34	PROMO ANODIZED STEEL	49	4
+Brand#34	PROMO ANODIZED TIN	14	4
+Brand#34	PROMO BRUSHED BRASS	9	4
+Brand#34	PROMO BRUSHED BRASS	23	4
+Brand#34	PROMO BRUSHED COPPER	36	4
+Brand#34	PROMO BRUSHED STEEL	36	4
+Brand#34	PROMO BURNISHED BRASS	49	4
+Brand#34	PROMO BURNISHED STEEL	3	4
+Brand#34	PROMO PLATED BRASS	9	4
+Brand#34	PROMO PLATED STEEL	49	4
+Brand#34	PROMO POLISHED BRASS	23	4
+Brand#34	PROMO POLISHED NICKEL	3	4
+Brand#34	PROMO POLISHED NICKEL	36	4
+Brand#34	SMALL ANODIZED BRASS	36	4
+Brand#34	SMALL ANODIZED COPPER	45	4
+Brand#34	SMALL ANODIZED NICKEL	14	4
+Brand#34	SMALL ANODIZED NICKEL	36	4
+Brand#34	SMALL ANODIZED STEEL	3	4
+Brand#34	SMALL ANODIZED STEEL	19	4
+Brand#34	SMALL ANODIZED STEEL	23	4
+Brand#34	SMALL ANODIZED STEEL	36	4
+Brand#34	SMALL BRUSHED BRASS	14	4
+Brand#34	SMALL BRUSHED BRASS	36	4
+Brand#34	SMALL BRUSHED NICKEL	14	4
+Brand#34	SMALL BRUSHED NICKEL	36	4
+Brand#34	SMALL BRUSHED NICKEL	45	4
+Brand#34	SMALL BRUSHED TIN	9	4
+Brand#34	SMALL BRUSHED TIN	23	4
+Brand#34	SMALL BRUSHED TIN	36	4
+Brand#34	SMALL BURNISHED COPPER	9	4
+Brand#34	SMALL BURNISHED TIN	36	4
+Brand#34	SMALL PLATED BRASS	14	4
+Brand#34	SMALL PLATED COPPER	36	4
+Brand#34	SMALL PLATED TIN	45	4
+Brand#34	SMALL POLISHED NICKEL	14	4
+Brand#34	SMALL POLISHED NICKEL	45	4
+Brand#34	SMALL POLISHED TIN	9	4
+Brand#34	SMALL POLISHED TIN	14	4
+Brand#34	SMALL POLISHED TIN	19	4
+Brand#34	STANDARD ANODIZED BRASS	23	4
+Brand#34	STANDARD ANODIZED BRASS	36	4
+Brand#34	STANDARD ANODIZED COPPER	45	4
+Brand#34	STANDARD ANODIZED NICKEL	36	4
+Brand#34	STANDARD ANODIZED STEEL	9	4
+Brand#34	STANDARD ANODIZED STEEL	49	4
+Brand#34	STANDARD ANODIZED TIN	9	4
+Brand#34	STANDARD BRUSHED BRASS	19	4
+Brand#34	STANDARD BRUSHED BRASS	23	4
+Brand#34	STANDARD BRUSHED NICKEL	23	4
+Brand#34	STANDARD BRUSHED STEEL	3	4
+Brand#34	STANDARD BRUSHED TIN	19	4
+Brand#34	STANDARD BURNISHED COPPER	45	4
+Brand#34	STANDARD BURNISHED NICKEL	19	4
+Brand#34	STANDARD BURNISHED NICKEL	45	4
+Brand#34	STANDARD BURNISHED STEEL	36	4
+Brand#34	STANDARD BURNISHED TIN	45	4
+Brand#34	STANDARD PLATED BRASS	9	4
+Brand#34	STANDARD PLATED COPPER	9	4
+Brand#34	STANDARD PLATED NICKEL	36	4
+Brand#35	ECONOMY ANODIZED COPPER	3	4
+Brand#35	ECONOMY ANODIZED STEEL	45	4
+Brand#35	ECONOMY BRUSHED BRASS	3	4
+Brand#35	ECONOMY BRUSHED NICKEL	49	4
+Brand#35	ECONOMY BRUSHED STEEL	23	4
+Brand#35	ECONOMY BRUSHED STEEL	45	4
+Brand#35	ECONOMY BRUSHED TIN	14	4
+Brand#35	ECONOMY BRUSHED TIN	23	4
+Brand#35	ECONOMY BURNISHED NICKEL	19	4
+Brand#35	ECONOMY BURNISHED STEEL	36	4
+Brand#35	ECONOMY BURNISHED TIN	9	4
+Brand#35	ECONOMY BURNISHED TIN	19	4
+Brand#35	ECONOMY BURNISHED TIN	49	4
+Brand#35	ECONOMY POLISHED COPPER	9	4
+Brand#35	ECONOMY POLISHED TIN	19	4
+Brand#35	LARGE ANODIZED BRASS	3	4
+Brand#35	LARGE ANODIZED BRASS	23	4
+Brand#35	LARGE ANODIZED COPPER	49	4
+Brand#35	LARGE ANODIZED STEEL	36	4
+Brand#35	LARGE ANODIZED TIN	9	4
+Brand#35	LARGE BRUSHED COPPER	9	4
+Brand#35	LARGE BRUSHED COPPER	23	4
+Brand#35	LARGE BRUSHED STEEL	3	4
+Brand#35	LARGE BRUSHED STEEL	9	4
+Brand#35	LARGE BURNISHED BRASS	36	4
+Brand#35	LARGE BURNISHED BRASS	45	4
+Brand#35	LARGE BURNISHED COPPER	23	4
+Brand#35	LARGE BURNISHED NICKEL	23	4
+Brand#35	LARGE PLATED BRASS	9	4
+Brand#35	LARGE PLATED COPPER	36	4
+Brand#35	LARGE POLISHED BRASS	49	4
+Brand#35	LARGE POLISHED STEEL	9	4
+Brand#35	LARGE POLISHED TIN	14	4
+Brand#35	MEDIUM ANODIZED BRASS	9	4
+Brand#35	MEDIUM ANODIZED BRASS	36	4
+Brand#35	MEDIUM ANODIZED COPPER	9	4
+Brand#35	MEDIUM BRUSHED BRASS	14	4
+Brand#35	MEDIUM BRUSHED COPPER	9	4
+Brand#35	MEDIUM BRUSHED COPPER	36	4
+Brand#35	MEDIUM BURNISHED BRASS	49	4
+Brand#35	MEDIUM BURNISHED NICKEL	45	4
+Brand#35	MEDIUM BURNISHED TIN	36	4
+Brand#35	MEDIUM PLATED BRASS	23	4
+Brand#35	MEDIUM PLATED COPPER	9	4
+Brand#35	MEDIUM PLATED NICKEL	45	4
+Brand#35	MEDIUM PLATED NICKEL	49	4
+Brand#35	MEDIUM PLATED STEEL	49	4
+Brand#35	PROMO ANODIZED COPPER	49	4
+Brand#35	PROMO ANODIZED NICKEL	19	4
+Brand#35	PROMO ANODIZED NICKEL	23	4
+Brand#35	PROMO ANODIZED TIN	3	4
+Brand#35	PROMO ANODIZED TIN	14	4
+Brand#35	PROMO BRUSHED BRASS	49	4
+Brand#35	PROMO BRUSHED NICKEL	14	4
+Brand#35	PROMO BRUSHED NICKEL	19	4
+Brand#35	PROMO BURNISHED BRASS	3	4
+Brand#35	PROMO BURNISHED STEEL	3	4
+Brand#35	PROMO PLATED BRASS	19	4
+Brand#35	PROMO PLATED COPPER	14	4
+Brand#35	PROMO PLATED STEEL	23	4
+Brand#35	PROMO PLATED STEEL	36	4
+Brand#35	PROMO PLATED TIN	19	4
+Brand#35	PROMO POLISHED BRASS	9	4
+Brand#35	PROMO POLISHED BRASS	36	4
+Brand#35	PROMO POLISHED NICKEL	36	4
+Brand#35	PROMO POLISHED STEEL	23	4
+Brand#35	PROMO POLISHED TIN	36	4
+Brand#35	PROMO POLISHED TIN	45	4
+Brand#35	SMALL ANODIZED COPPER	9	4
+Brand#35	SMALL ANODIZED STEEL	19	4
+Brand#35	SMALL ANODIZED TIN	19	4
+Brand#35	SMALL BRUSHED BRASS	36	4
+Brand#35	SMALL BRUSHED STEEL	49	4
+Brand#35	SMALL BRUSHED TIN	3	4
+Brand#35	SMALL BRUSHED TIN	19	4
+Brand#35	SMALL BRUSHED TIN	23	4
+Brand#35	SMALL BURNISHED BRASS	23	4
+Brand#35	SMALL BURNISHED STEEL	36	4
+Brand#35	SMALL BURNISHED TIN	3	4
+Brand#35	SMALL BURNISHED TIN	36	4
+Brand#35	SMALL BURNISHED TIN	49	4
+Brand#35	SMALL PLATED BRASS	23	4
+Brand#35	SMALL PLATED STEEL	14	4
+Brand#35	SMALL POLISHED BRASS	36	4
+Brand#35	SMALL POLISHED STEEL	3	4
+Brand#35	SMALL POLISHED STEEL	49	4
+Brand#35	SMALL POLISHED TIN	23	4
+Brand#35	SMALL POLISHED TIN	45	4
+Brand#35	STANDARD ANODIZED NICKEL	14	4
+Brand#35	STANDARD ANODIZED STEEL	23	4
+Brand#35	STANDARD ANODIZED STEEL	45	4
+Brand#35	STANDARD ANODIZED TIN	9	4
+Brand#35	STANDARD ANODIZED TIN	19	4
+Brand#35	STANDARD BRUSHED BRASS	3	4
+Brand#35	STANDARD BRUSHED BRASS	23	4
+Brand#35	STANDARD BRUSHED BRASS	36	4
+Brand#35	STANDARD BRUSHED COPPER	36	4
+Brand#35	STANDARD BRUSHED NICKEL	36	4
+Brand#35	STANDARD BRUSHED NICKEL	49	4
+Brand#35	STANDARD BRUSHED TIN	9	4
+Brand#35	STANDARD BURNISHED BRASS	9	4
+Brand#35	STANDARD BURNISHED BRASS	19	4
+Brand#35	STANDARD BURNISHED BRASS	23	4
+Brand#35	STANDARD BURNISHED COPPER	36	4
+Brand#35	STANDARD BURNISHED STEEL	14	4
+Brand#35	STANDARD PLATED COPPER	19	4
+Brand#35	STANDARD PLATED NICKEL	23	4
+Brand#35	STANDARD PLATED STEEL	14	4
+Brand#35	STANDARD PLATED STEEL	23	4
+Brand#35	STANDARD PLATED TIN	49	4
+Brand#35	STANDARD POLISHED NICKEL	23	4
+Brand#35	STANDARD POLISHED TIN	23	4
+Brand#35	STANDARD POLISHED TIN	45	4
+Brand#41	ECONOMY ANODIZED STEEL	49	4
+Brand#41	ECONOMY BRUSHED BRASS	3	4
+Brand#41	ECONOMY BRUSHED COPPER	36	4
+Brand#41	ECONOMY BRUSHED NICKEL	23	4
+Brand#41	ECONOMY BRUSHED STEEL	36	4
+Brand#41	ECONOMY BRUSHED STEEL	45	4
+Brand#41	ECONOMY BRUSHED TIN	14	4
+Brand#41	ECONOMY PLATED COPPER	3	4
+Brand#41	ECONOMY PLATED STEEL	3	4
+Brand#41	ECONOMY PLATED TIN	23	4
+Brand#41	ECONOMY POLISHED COPPER	19	4
+Brand#41	ECONOMY POLISHED NICKEL	9	4
+Brand#41	ECONOMY POLISHED NICKEL	14	4
+Brand#41	ECONOMY POLISHED NICKEL	23	4
+Brand#41	ECONOMY POLISHED NICKEL	49	4
+Brand#41	ECONOMY POLISHED STEEL	9	4
+Brand#41	ECONOMY POLISHED STEEL	19	4
+Brand#41	ECONOMY POLISHED STEEL	45	4
+Brand#41	ECONOMY POLISHED TIN	19	4
+Brand#41	LARGE ANODIZED BRASS	14	4
+Brand#41	LARGE ANODIZED BRASS	23	4
+Brand#41	LARGE ANODIZED COPPER	49	4
+Brand#41	LARGE ANODIZED STEEL	3	4
+Brand#41	LARGE ANODIZED STEEL	23	4
+Brand#41	LARGE BRUSHED COPPER	23	4
+Brand#41	LARGE BRUSHED COPPER	49	4
+Brand#41	LARGE BRUSHED STEEL	19	4
+Brand#41	LARGE BURNISHED BRASS	45	4
+Brand#41	LARGE BURNISHED COPPER	3	4
+Brand#41	LARGE BURNISHED NICKEL	23	4
+Brand#41	LARGE BURNISHED TIN	9	4
+Brand#41	LARGE PLATED NICKEL	3	4
+Brand#41	LARGE PLATED NICKEL	23	4
+Brand#41	LARGE PLATED STEEL	9	4
+Brand#41	LARGE PLATED STEEL	36	4
+Brand#41	LARGE PLATED TIN	9	4
+Brand#41	LARGE POLISHED BRASS	36	4
+Brand#41	LARGE POLISHED COPPER	19	4
+Brand#41	LARGE POLISHED COPPER	49	4
+Brand#41	LARGE POLISHED NICKEL	36	4
+Brand#41	LARGE POLISHED STEEL	14	4
+Brand#41	MEDIUM ANODIZED BRASS	9	4
+Brand#41	MEDIUM ANODIZED COPPER	14	4
+Brand#41	MEDIUM ANODIZED NICKEL	3	4
+Brand#41	MEDIUM ANODIZED NICKEL	9	4
+Brand#41	MEDIUM ANODIZED STEEL	14	4
+Brand#41	MEDIUM BRUSHED COPPER	3	4
+Brand#41	MEDIUM BRUSHED TIN	9	4
+Brand#41	MEDIUM BURNISHED COPPER	23	4
+Brand#41	MEDIUM BURNISHED STEEL	9	4
+Brand#41	MEDIUM BURNISHED STEEL	45	4
+Brand#41	MEDIUM BURNISHED TIN	3	4
+Brand#41	MEDIUM PLATED BRASS	19	4
+Brand#41	MEDIUM PLATED BRASS	45	4
+Brand#41	MEDIUM PLATED COPPER	19	4
+Brand#41	MEDIUM PLATED STEEL	19	4
+Brand#41	MEDIUM PLATED STEEL	23	4
+Brand#41	PROMO ANODIZED BRASS	19	4
+Brand#41	PROMO ANODIZED COPPER	9	4
+Brand#41	PROMO ANODIZED NICKEL	9	4
+Brand#41	PROMO BRUSHED BRASS	14	4
+Brand#41	PROMO BRUSHED COPPER	36	4
+Brand#41	PROMO BRUSHED NICKEL	14	4
+Brand#41	PROMO BURNISHED BRASS	49	4
+Brand#41	PROMO BURNISHED NICKEL	36	4
+Brand#41	PROMO BURNISHED TIN	3	4
+Brand#41	PROMO PLATED NICKEL	14	4
+Brand#41	PROMO PLATED NICKEL	45	4
+Brand#41	PROMO PLATED STEEL	3	4
+Brand#41	PROMO PLATED TIN	3	4
+Brand#41	PROMO POLISHED COPPER	23	4
+Brand#41	SMALL ANODIZED BRASS	3	4
+Brand#41	SMALL ANODIZED BRASS	14	4
+Brand#41	SMALL ANODIZED STEEL	45	4
+Brand#41	SMALL ANODIZED TIN	9	4
+Brand#41	SMALL BRUSHED TIN	19	4
+Brand#41	SMALL BURNISHED COPPER	9	4
+Brand#41	SMALL BURNISHED NICKEL	3	4
+Brand#41	SMALL BURNISHED TIN	45	4
+Brand#41	SMALL PLATED COPPER	14	4
+Brand#41	SMALL PLATED COPPER	36	4
+Brand#41	SMALL PLATED COPPER	49	4
+Brand#41	SMALL PLATED TIN	19	4
+Brand#41	SMALL POLISHED COPPER	14	4
+Brand#41	SMALL POLISHED COPPER	19	4
+Brand#41	SMALL POLISHED COPPER	36	4
+Brand#41	SMALL POLISHED TIN	45	4
+Brand#41	STANDARD ANODIZED COPPER	19	4
+Brand#41	STANDARD ANODIZED NICKEL	9	4
+Brand#41	STANDARD ANODIZED STEEL	49	4
+Brand#41	STANDARD ANODIZED TIN	9	4
+Brand#41	STANDARD ANODIZED TIN	36	4
+Brand#41	STANDARD ANODIZED TIN	49	4
+Brand#41	STANDARD BRUSHED BRASS	19	4
+Brand#41	STANDARD BRUSHED NICKEL	3	4
+Brand#41	STANDARD BRUSHED NICKEL	9	4
+Brand#41	STANDARD BRUSHED STEEL	45	4
+Brand#41	STANDARD BRUSHED TIN	45	4
+Brand#41	STANDARD BURNISHED BRASS	23	4
+Brand#41	STANDARD BURNISHED BRASS	36	4
+Brand#41	STANDARD BURNISHED COPPER	49	4
+Brand#41	STANDARD BURNISHED STEEL	45	4
+Brand#41	STANDARD PLATED BRASS	45	4
+Brand#41	STANDARD PLATED NICKEL	14	4
+Brand#41	STANDARD PLATED STEEL	45	4
+Brand#41	STANDARD PLATED TIN	49	4
+Brand#41	STANDARD POLISHED STEEL	9	4
+Brand#41	STANDARD POLISHED STEEL	19	4
+Brand#41	STANDARD POLISHED TIN	45	4
+Brand#42	ECONOMY ANODIZED NICKEL	19	4
+Brand#42	ECONOMY BRUSHED BRASS	14	4
+Brand#42	ECONOMY BRUSHED COPPER	3	4
+Brand#42	ECONOMY BRUSHED COPPER	14	4
+Brand#42	ECONOMY BRUSHED NICKEL	14	4
+Brand#42	ECONOMY BRUSHED STEEL	14	4
+Brand#42	ECONOMY BRUSHED TIN	19	4
+Brand#42	ECONOMY BRUSHED TIN	49	4
+Brand#42	ECONOMY BURNISHED BRASS	19	4
+Brand#42	ECONOMY BURNISHED COPPER	23	4
+Brand#42	ECONOMY BURNISHED NICKEL	14	4
+Brand#42	ECONOMY BURNISHED TIN	14	4
+Brand#42	ECONOMY PLATED COPPER	23	4
+Brand#42	ECONOMY POLISHED BRASS	3	4
+Brand#42	ECONOMY POLISHED COPPER	9	4
+Brand#42	ECONOMY POLISHED STEEL	9	4
+Brand#42	ECONOMY POLISHED STEEL	36	4
+Brand#42	ECONOMY POLISHED TIN	14	4
+Brand#42	LARGE ANODIZED BRASS	49	4
+Brand#42	LARGE ANODIZED COPPER	14	4
+Brand#42	LARGE ANODIZED COPPER	49	4
+Brand#42	LARGE ANODIZED NICKEL	45	4
+Brand#42	LARGE ANODIZED NICKEL	49	4
+Brand#42	LARGE ANODIZED TIN	45	4
+Brand#42	LARGE BRUSHED BRASS	49	4
+Brand#42	LARGE BURNISHED BRASS	45	4
+Brand#42	LARGE BURNISHED BRASS	49	4
+Brand#42	LARGE BURNISHED COPPER	9	4
+Brand#42	LARGE BURNISHED TIN	9	4
+Brand#42	LARGE PLATED BRASS	45	4
+Brand#42	LARGE PLATED COPPER	9	4
+Brand#42	LARGE PLATED NICKEL	36	4
+Brand#42	LARGE PLATED TIN	23	4
+Brand#42	LARGE POLISHED BRASS	9	4
+Brand#42	LARGE POLISHED NICKEL	3	4
+Brand#42	LARGE POLISHED NICKEL	23	4
+Brand#42	LARGE POLISHED STEEL	9	4
+Brand#42	MEDIUM ANODIZED BRASS	23	4
+Brand#42	MEDIUM ANODIZED COPPER	19	4
+Brand#42	MEDIUM ANODIZED NICKEL	14	4
+Brand#42	MEDIUM ANODIZED NICKEL	19	4
+Brand#42	MEDIUM ANODIZED NICKEL	23	4
+Brand#42	MEDIUM ANODIZED STEEL	9	4
+Brand#42	MEDIUM ANODIZED STEEL	14	4
+Brand#42	MEDIUM ANODIZED STEEL	23	4
+Brand#42	MEDIUM ANODIZED TIN	14	4
+Brand#42	MEDIUM ANODIZED TIN	19	4
+Brand#42	MEDIUM BRUSHED COPPER	45	4
+Brand#42	MEDIUM BRUSHED COPPER	49	4
+Brand#42	MEDIUM BRUSHED STEEL	36	4
+Brand#42	MEDIUM BURNISHED COPPER	49	4
+Brand#42	MEDIUM BURNISHED TIN	3	4
+Brand#42	MEDIUM BURNISHED TIN	49	4
+Brand#42	MEDIUM PLATED NICKEL	45	4
+Brand#42	MEDIUM PLATED STEEL	3	4
+Brand#42	MEDIUM PLATED STEEL	23	4
+Brand#42	MEDIUM PLATED STEEL	45	4
+Brand#42	PROMO ANODIZED NICKEL	3	4
+Brand#42	PROMO ANODIZED NICKEL	19	4
+Brand#42	PROMO ANODIZED STEEL	49	4
+Brand#42	PROMO BRUSHED COPPER	45	4
+Brand#42	PROMO BRUSHED STEEL	19	4
+Brand#42	PROMO BRUSHED TIN	45	4
+Brand#42	PROMO BURNISHED COPPER	45	4
+Brand#42	PROMO BURNISHED NICKEL	3	4
+Brand#42	PROMO BURNISHED STEEL	9	4
+Brand#42	PROMO BURNISHED TIN	49	4
+Brand#42	PROMO PLATED BRASS	45	4
+Brand#42	PROMO PLATED NICKEL	23	4
+Brand#42	PROMO PLATED STEEL	19	4
+Brand#42	PROMO PLATED STEEL	45	4
+Brand#42	PROMO POLISHED COPPER	36	4
+Brand#42	PROMO POLISHED NICKEL	3	4
+Brand#42	SMALL ANODIZED BRASS	23	4
+Brand#42	SMALL ANODIZED COPPER	14	4
+Brand#42	SMALL ANODIZED COPPER	19	4
+Brand#42	SMALL ANODIZED NICKEL	23	4
+Brand#42	SMALL BRUSHED TIN	49	4
+Brand#42	SMALL BURNISHED BRASS	3	4
+Brand#42	SMALL BURNISHED BRASS	36	4
+Brand#42	SMALL BURNISHED COPPER	9	4
+Brand#42	SMALL BURNISHED NICKEL	9	4
+Brand#42	SMALL BURNISHED TIN	9	4
+Brand#42	SMALL PLATED NICKEL	9	4
+Brand#42	SMALL PLATED TIN	36	4
+Brand#42	SMALL POLISHED BRASS	3	4
+Brand#42	SMALL POLISHED COPPER	36	4
+Brand#42	SMALL POLISHED NICKEL	23	4
+Brand#42	SMALL POLISHED STEEL	49	4
+Brand#42	SMALL POLISHED TIN	3	4
+Brand#42	STANDARD ANODIZED BRASS	49	4
+Brand#42	STANDARD ANODIZED COPPER	49	4
+Brand#42	STANDARD ANODIZED NICKEL	36	4
+Brand#42	STANDARD ANODIZED NICKEL	45	4
+Brand#42	STANDARD BRUSHED NICKEL	23	4
+Brand#42	STANDARD BURNISHED NICKEL	49	4
+Brand#42	STANDARD BURNISHED STEEL	3	4
+Brand#42	STANDARD BURNISHED TIN	19	4
+Brand#42	STANDARD PLATED BRASS	19	4
+Brand#42	STANDARD PLATED COPPER	9	4
+Brand#42	STANDARD PLATED NICKEL	45	4
+Brand#42	STANDARD PLATED STEEL	3	4
+Brand#42	STANDARD POLISHED BRASS	36	4
+Brand#42	STANDARD POLISHED BRASS	45	4
+Brand#42	STANDARD POLISHED COPPER	14	4
+Brand#42	STANDARD POLISHED NICKEL	45	4
+Brand#42	STANDARD POLISHED TIN	9	4
+Brand#42	STANDARD POLISHED TIN	19	4
+Brand#42	STANDARD POLISHED TIN	23	4
+Brand#42	STANDARD POLISHED TIN	36	4
+Brand#43	ECONOMY ANODIZED COPPER	19	4
+Brand#43	ECONOMY ANODIZED COPPER	45	4
+Brand#43	ECONOMY ANODIZED NICKEL	3	4
+Brand#43	ECONOMY ANODIZED NICKEL	49	4
+Brand#43	ECONOMY ANODIZED STEEL	23	4
+Brand#43	ECONOMY ANODIZED TIN	49	4
+Brand#43	ECONOMY BRUSHED BRASS	49	4
+Brand#43	ECONOMY BRUSHED COPPER	45	4
+Brand#43	ECONOMY BRUSHED NICKEL	9	4
+Brand#43	ECONOMY BURNISHED NICKEL	9	4
+Brand#43	ECONOMY BURNISHED TIN	19	4
+Brand#43	ECONOMY PLATED COPPER	36	4
+Brand#43	ECONOMY PLATED STEEL	9	4
+Brand#43	ECONOMY PLATED TIN	14	4
+Brand#43	ECONOMY PLATED TIN	19	4
+Brand#43	ECONOMY PLATED TIN	49	4
+Brand#43	ECONOMY POLISHED COPPER	19	4
+Brand#43	ECONOMY POLISHED NICKEL	36	4
+Brand#43	ECONOMY POLISHED TIN	14	4
+Brand#43	ECONOMY POLISHED TIN	45	4
+Brand#43	LARGE ANODIZED BRASS	14	4
+Brand#43	LARGE ANODIZED BRASS	36	4
+Brand#43	LARGE ANODIZED COPPER	45	4
+Brand#43	LARGE BRUSHED COPPER	3	4
+Brand#43	LARGE BRUSHED NICKEL	14	4
+Brand#43	LARGE BRUSHED NICKEL	19	4
+Brand#43	LARGE BRUSHED NICKEL	45	4
+Brand#43	LARGE BRUSHED NICKEL	49	4
+Brand#43	LARGE BURNISHED COPPER	3	4
+Brand#43	LARGE BURNISHED TIN	23	4
+Brand#43	LARGE BURNISHED TIN	45	4
+Brand#43	LARGE PLATED BRASS	45	4
+Brand#43	LARGE PLATED STEEL	14	4
+Brand#43	LARGE PLATED TIN	36	4
+Brand#43	LARGE PLATED TIN	45	4
+Brand#43	LARGE POLISHED BRASS	9	4
+Brand#43	LARGE POLISHED COPPER	9	4
+Brand#43	LARGE POLISHED COPPER	19	4
+Brand#43	LARGE POLISHED STEEL	14	4
+Brand#43	LARGE POLISHED TIN	45	4
+Brand#43	MEDIUM ANODIZED BRASS	14	4
+Brand#43	MEDIUM ANODIZED COPPER	36	4
+Brand#43	MEDIUM ANODIZED COPPER	49	4
+Brand#43	MEDIUM ANODIZED STEEL	19	4
+Brand#43	MEDIUM ANODIZED STEEL	36	4
+Brand#43	MEDIUM BRUSHED BRASS	9	4
+Brand#43	MEDIUM BRUSHED BRASS	49	4
+Brand#43	MEDIUM BRUSHED COPPER	3	4
+Brand#43	MEDIUM BRUSHED NICKEL	9	4
+Brand#43	MEDIUM BRUSHED STEEL	23	4
+Brand#43	MEDIUM BURNISHED COPPER	14	4
+Brand#43	MEDIUM BURNISHED COPPER	45	4
+Brand#43	MEDIUM BURNISHED TIN	23	4
+Brand#43	MEDIUM PLATED BRASS	3	4
+Brand#43	MEDIUM PLATED COPPER	14	4
+Brand#43	MEDIUM PLATED NICKEL	36	4
+Brand#43	MEDIUM PLATED NICKEL	45	4
+Brand#43	MEDIUM PLATED TIN	49	4
+Brand#43	PROMO ANODIZED NICKEL	45	4
+Brand#43	PROMO ANODIZED TIN	14	4
+Brand#43	PROMO BRUSHED NICKEL	14	4
+Brand#43	PROMO BRUSHED STEEL	14	4
+Brand#43	PROMO BRUSHED TIN	45	4
+Brand#43	PROMO BURNISHED BRASS	49	4
+Brand#43	PROMO BURNISHED NICKEL	9	4
+Brand#43	PROMO BURNISHED STEEL	3	4
+Brand#43	PROMO BURNISHED STEEL	36	4
+Brand#43	PROMO BURNISHED TIN	36	4
+Brand#43	PROMO PLATED BRASS	19	4
+Brand#43	PROMO PLATED COPPER	45	4
+Brand#43	PROMO PLATED COPPER	49	4
+Brand#43	PROMO PLATED TIN	3	4
+Brand#43	PROMO POLISHED BRASS	19	4
+Brand#43	PROMO POLISHED BRASS	23	4
+Brand#43	PROMO POLISHED NICKEL	49	4
+Brand#43	PROMO POLISHED STEEL	14	4
+Brand#43	PROMO POLISHED STEEL	19	4
+Brand#43	PROMO POLISHED STEEL	23	4
+Brand#43	PROMO POLISHED STEEL	36	4
+Brand#43	SMALL ANODIZED BRASS	19	4
+Brand#43	SMALL ANODIZED NICKEL	9	4
+Brand#43	SMALL BRUSHED NICKEL	3	4
+Brand#43	SMALL BRUSHED NICKEL	9	4
+Brand#43	SMALL BURNISHED BRASS	49	4
+Brand#43	SMALL BURNISHED STEEL	23	4
+Brand#43	SMALL PLATED BRASS	14	4
+Brand#43	SMALL PLATED BRASS	36	4
+Brand#43	SMALL PLATED COPPER	23	4
+Brand#43	SMALL PLATED COPPER	49	4
+Brand#43	SMALL PLATED NICKEL	36	4
+Brand#43	SMALL PLATED NICKEL	49	4
+Brand#43	SMALL PLATED STEEL	14	4
+Brand#43	SMALL PLATED TIN	49	4
+Brand#43	SMALL POLISHED STEEL	19	4
+Brand#43	STANDARD ANODIZED BRASS	3	4
+Brand#43	STANDARD ANODIZED COPPER	49	4
+Brand#43	STANDARD ANODIZED NICKEL	14	4
+Brand#43	STANDARD BRUSHED TIN	14	4
+Brand#43	STANDARD BURNISHED BRASS	23	4
+Brand#43	STANDARD BURNISHED STEEL	19	4
+Brand#43	STANDARD BURNISHED STEEL	23	4
+Brand#43	STANDARD PLATED BRASS	9	4
+Brand#43	STANDARD PLATED BRASS	19	4
+Brand#43	STANDARD PLATED BRASS	49	4
+Brand#43	STANDARD PLATED COPPER	36	4
+Brand#43	STANDARD PLATED NICKEL	14	4
+Brand#43	STANDARD PLATED NICKEL	19	4
+Brand#43	STANDARD PLATED TIN	14	4
+Brand#43	STANDARD POLISHED BRASS	23	4
+Brand#43	STANDARD POLISHED TIN	9	4
+Brand#44	ECONOMY ANODIZED BRASS	3	4
+Brand#44	ECONOMY ANODIZED BRASS	45	4
+Brand#44	ECONOMY ANODIZED NICKEL	36	4
+Brand#44	ECONOMY ANODIZED STEEL	19	4
+Brand#44	ECONOMY BRUSHED COPPER	23	4
+Brand#44	ECONOMY BRUSHED TIN	49	4
+Brand#44	ECONOMY BURNISHED COPPER	19	4
+Brand#44	ECONOMY BURNISHED STEEL	45	4
+Brand#44	ECONOMY PLATED STEEL	19	4
+Brand#44	ECONOMY PLATED STEEL	23	4
+Brand#44	ECONOMY PLATED TIN	23	4
+Brand#44	ECONOMY POLISHED BRASS	23	4
+Brand#44	ECONOMY POLISHED COPPER	9	4
+Brand#44	ECONOMY POLISHED COPPER	45	4
+Brand#44	ECONOMY POLISHED NICKEL	14	4
+Brand#44	ECONOMY POLISHED NICKEL	23	4
+Brand#44	ECONOMY POLISHED STEEL	49	4
+Brand#44	ECONOMY POLISHED TIN	23	4
+Brand#44	ECONOMY POLISHED TIN	36	4
+Brand#44	LARGE ANODIZED BRASS	19	4
+Brand#44	LARGE ANODIZED TIN	3	4
+Brand#44	LARGE ANODIZED TIN	14	4
+Brand#44	LARGE BRUSHED TIN	3	4
+Brand#44	LARGE BRUSHED TIN	23	4
+Brand#44	LARGE BURNISHED BRASS	23	4
+Brand#44	LARGE BURNISHED BRASS	49	4
+Brand#44	LARGE BURNISHED COPPER	3	4
+Brand#44	LARGE BURNISHED COPPER	19	4
+Brand#44	LARGE BURNISHED COPPER	36	4
+Brand#44	LARGE BURNISHED TIN	14	4
+Brand#44	LARGE PLATED BRASS	9	4
+Brand#44	LARGE PLATED BRASS	49	4
+Brand#44	LARGE PLATED NICKEL	14	4
+Brand#44	LARGE PLATED STEEL	14	4
+Brand#44	LARGE PLATED TIN	19	4
+Brand#44	LARGE PLATED TIN	23	4
+Brand#44	LARGE POLISHED STEEL	23	4
+Brand#44	LARGE POLISHED STEEL	49	4
+Brand#44	MEDIUM ANODIZED COPPER	45	4
+Brand#44	MEDIUM ANODIZED NICKEL	45	4
+Brand#44	MEDIUM BRUSHED BRASS	49	4
+Brand#44	MEDIUM BRUSHED COPPER	3	4
+Brand#44	MEDIUM BRUSHED COPPER	45	4
+Brand#44	MEDIUM BRUSHED STEEL	19	4
+Brand#44	MEDIUM BRUSHED TIN	49	4
+Brand#44	MEDIUM BURNISHED COPPER	45	4
+Brand#44	MEDIUM BURNISHED NICKEL	23	4
+Brand#44	MEDIUM BURNISHED TIN	23	4
+Brand#44	MEDIUM PLATED COPPER	14	4
+Brand#44	PROMO ANODIZED COPPER	23	4
+Brand#44	PROMO ANODIZED STEEL	36	4
+Brand#44	PROMO BRUSHED COPPER	23	4
+Brand#44	PROMO BRUSHED COPPER	36	4
+Brand#44	PROMO BRUSHED TIN	19	4
+Brand#44	PROMO PLATED BRASS	3	4
+Brand#44	PROMO PLATED COPPER	36	4
+Brand#44	PROMO PLATED STEEL	3	4
+Brand#44	PROMO PLATED STEEL	36	4
+Brand#44	PROMO PLATED STEEL	49	4
+Brand#44	PROMO POLISHED BRASS	3	4
+Brand#44	PROMO POLISHED BRASS	19	4
+Brand#44	PROMO POLISHED COPPER	45	4
+Brand#44	PROMO POLISHED STEEL	36	4
+Brand#44	PROMO POLISHED TIN	9	4
+Brand#44	SMALL ANODIZED COPPER	23	4
+Brand#44	SMALL ANODIZED STEEL	23	4
+Brand#44	SMALL ANODIZED TIN	45	4
+Brand#44	SMALL BRUSHED COPPER	14	4
+Brand#44	SMALL BRUSHED STEEL	45	4
+Brand#44	SMALL BURNISHED COPPER	14	4
+Brand#44	SMALL BURNISHED COPPER	49	4
+Brand#44	SMALL BURNISHED NICKEL	14	4
+Brand#44	SMALL BURNISHED STEEL	23	4
+Brand#44	SMALL BURNISHED TIN	49	4
+Brand#44	SMALL PLATED BRASS	36	4
+Brand#44	SMALL PLATED COPPER	19	4
+Brand#44	SMALL PLATED NICKEL	3	4
+Brand#44	SMALL POLISHED COPPER	3	4
+Brand#44	SMALL POLISHED COPPER	49	4
+Brand#44	SMALL POLISHED STEEL	3	4
+Brand#44	STANDARD ANODIZED BRASS	3	4
+Brand#44	STANDARD ANODIZED COPPER	3	4
+Brand#44	STANDARD ANODIZED NICKEL	3	4
+Brand#44	STANDARD ANODIZED NICKEL	36	4
+Brand#44	STANDARD ANODIZED STEEL	14	4
+Brand#44	STANDARD ANODIZED TIN	3	4
+Brand#44	STANDARD ANODIZED TIN	9	4
+Brand#44	STANDARD ANODIZED TIN	36	4
+Brand#44	STANDARD BRUSHED COPPER	36	4
+Brand#44	STANDARD BRUSHED COPPER	45	4
+Brand#44	STANDARD BRUSHED TIN	9	4
+Brand#44	STANDARD BRUSHED TIN	49	4
+Brand#44	STANDARD BURNISHED COPPER	9	4
+Brand#44	STANDARD BURNISHED STEEL	23	4
+Brand#44	STANDARD PLATED BRASS	14	4
+Brand#44	STANDARD PLATED BRASS	23	4
+Brand#44	STANDARD PLATED BRASS	49	4
+Brand#44	STANDARD PLATED COPPER	14	4
+Brand#44	STANDARD POLISHED NICKEL	19	4
+Brand#44	STANDARD POLISHED TIN	9	4
+Brand#51	ECONOMY ANODIZED BRASS	9	4
+Brand#51	ECONOMY ANODIZED BRASS	23	4
+Brand#51	ECONOMY ANODIZED NICKEL	3	4
+Brand#51	ECONOMY ANODIZED NICKEL	23	4
+Brand#51	ECONOMY ANODIZED STEEL	19	4
+Brand#51	ECONOMY ANODIZED STEEL	23	4
+Brand#51	ECONOMY ANODIZED STEEL	49	4
+Brand#51	ECONOMY BRUSHED BRASS	3	4
+Brand#51	ECONOMY BRUSHED BRASS	49	4
+Brand#51	ECONOMY BRUSHED NICKEL	14	4
+Brand#51	ECONOMY BRUSHED STEEL	45	4
+Brand#51	ECONOMY BRUSHED TIN	36	4
+Brand#51	ECONOMY BURNISHED BRASS	14	4
+Brand#51	ECONOMY BURNISHED COPPER	45	4
+Brand#51	ECONOMY PLATED NICKEL	49	4
+Brand#51	ECONOMY PLATED TIN	36	4
+Brand#51	ECONOMY POLISHED COPPER	9	4
+Brand#51	ECONOMY POLISHED STEEL	14	4
+Brand#51	ECONOMY POLISHED STEEL	49	4
+Brand#51	LARGE ANODIZED COPPER	9	4
+Brand#51	LARGE ANODIZED COPPER	49	4
+Brand#51	LARGE ANODIZED NICKEL	14	4
+Brand#51	LARGE ANODIZED STEEL	36	4
+Brand#51	LARGE BRUSHED NICKEL	3	4
+Brand#51	LARGE BRUSHED NICKEL	9	4
+Brand#51	LARGE BURNISHED BRASS	19	4
+Brand#51	LARGE BURNISHED BRASS	36	4
+Brand#51	LARGE BURNISHED COPPER	14	4
+Brand#51	LARGE BURNISHED NICKEL	14	4
+Brand#51	LARGE PLATED BRASS	36	4
+Brand#51	LARGE POLISHED COPPER	14	4
+Brand#51	LARGE POLISHED NICKEL	23	4
+Brand#51	LARGE POLISHED NICKEL	36	4
+Brand#51	LARGE POLISHED STEEL	19	4
+Brand#51	MEDIUM ANODIZED COPPER	9	4
+Brand#51	MEDIUM ANODIZED STEEL	3	4
+Brand#51	MEDIUM BRUSHED BRASS	36	4
+Brand#51	MEDIUM BRUSHED BRASS	45	4
+Brand#51	MEDIUM BRUSHED STEEL	3	4
+Brand#51	MEDIUM BRUSHED TIN	36	4
+Brand#51	MEDIUM BURNISHED NICKEL	3	4
+Brand#51	MEDIUM BURNISHED NICKEL	36	4
+Brand#51	MEDIUM BURNISHED STEEL	14	4
+Brand#51	MEDIUM BURNISHED TIN	9	4
+Brand#51	MEDIUM PLATED STEEL	19	4
+Brand#51	MEDIUM PLATED TIN	3	4
+Brand#51	PROMO ANODIZED NICKEL	14	4
+Brand#51	PROMO ANODIZED STEEL	23	4
+Brand#51	PROMO ANODIZED TIN	19	4
+Brand#51	PROMO BRUSHED BRASS	23	4
+Brand#51	PROMO BRUSHED COPPER	45	4
+Brand#51	PROMO BRUSHED STEEL	45	4
+Brand#51	PROMO BRUSHED TIN	9	4
+Brand#51	PROMO BURNISHED BRASS	19	4
+Brand#51	PROMO BURNISHED BRASS	23	4
+Brand#51	PROMO BURNISHED NICKEL	14	4
+Brand#51	PROMO PLATED BRASS	3	4
+Brand#51	PROMO PLATED BRASS	23	4
+Brand#51	PROMO PLATED TIN	19	4
+Brand#51	PROMO PLATED TIN	23	4
+Brand#51	PROMO POLISHED BRASS	23	4
+Brand#51	PROMO POLISHED COPPER	9	4
+Brand#51	PROMO POLISHED NICKEL	9	4
+Brand#51	PROMO POLISHED STEEL	49	4
+Brand#51	SMALL ANODIZED STEEL	14	4
+Brand#51	SMALL BRUSHED BRASS	23	4
+Brand#51	SMALL BRUSHED TIN	19	4
+Brand#51	SMALL BURNISHED NICKEL	23	4
+Brand#51	SMALL PLATED COPPER	49	4
+Brand#51	SMALL PLATED NICKEL	3	4
+Brand#51	SMALL PLATED NICKEL	14	4
+Brand#51	SMALL PLATED STEEL	45	4
+Brand#51	SMALL POLISHED NICKEL	14	4
+Brand#51	SMALL POLISHED NICKEL	23	4
+Brand#51	SMALL POLISHED STEEL	3	4
+Brand#51	SMALL POLISHED STEEL	19	4
+Brand#51	SMALL POLISHED STEEL	49	4
+Brand#51	STANDARD ANODIZED NICKEL	3	4
+Brand#51	STANDARD ANODIZED NICKEL	49	4
+Brand#51	STANDARD BRUSHED BRASS	3	4
+Brand#51	STANDARD BRUSHED COPPER	3	4
+Brand#51	STANDARD BRUSHED NICKEL	19	4
+Brand#51	STANDARD BRUSHED STEEL	36	4
+Brand#51	STANDARD BURNISHED COPPER	19	4
+Brand#51	STANDARD BURNISHED NICKEL	49	4
+Brand#51	STANDARD BURNISHED STEEL	23	4
+Brand#51	STANDARD BURNISHED STEEL	36	4
+Brand#51	STANDARD BURNISHED TIN	45	4
+Brand#51	STANDARD PLATED BRASS	36	4
+Brand#51	STANDARD PLATED BRASS	49	4
+Brand#51	STANDARD PLATED COPPER	14	4
+Brand#51	STANDARD PLATED COPPER	23	4
+Brand#51	STANDARD POLISHED BRASS	14	4
+Brand#51	STANDARD POLISHED BRASS	45	4
+Brand#51	STANDARD POLISHED STEEL	36	4
+Brand#51	STANDARD POLISHED STEEL	49	4
+Brand#51	STANDARD POLISHED TIN	45	4
+Brand#52	ECONOMY ANODIZED BRASS	14	4
+Brand#52	ECONOMY ANODIZED BRASS	23	4
+Brand#52	ECONOMY ANODIZED COPPER	36	4
+Brand#52	ECONOMY ANODIZED NICKEL	49	4
+Brand#52	ECONOMY ANODIZED STEEL	19	4
+Brand#52	ECONOMY BRUSHED COPPER	49	4
+Brand#52	ECONOMY BURNISHED BRASS	36	4
+Brand#52	ECONOMY BURNISHED COPPER	19	4
+Brand#52	ECONOMY BURNISHED COPPER	45	4
+Brand#52	ECONOMY BURNISHED NICKEL	19	4
+Brand#52	ECONOMY BURNISHED STEEL	36	4
+Brand#52	ECONOMY PLATED TIN	14	4
+Brand#52	ECONOMY PLATED TIN	23	4
+Brand#52	ECONOMY POLISHED BRASS	23	4
+Brand#52	ECONOMY POLISHED BRASS	45	4
+Brand#52	ECONOMY POLISHED NICKEL	36	4
+Brand#52	ECONOMY POLISHED STEEL	49	4
+Brand#52	LARGE ANODIZED COPPER	14	4
+Brand#52	LARGE ANODIZED NICKEL	3	4
+Brand#52	LARGE ANODIZED NICKEL	45	4
+Brand#52	LARGE ANODIZED TIN	45	4
+Brand#52	LARGE BRUSHED COPPER	19	4
+Brand#52	LARGE BRUSHED NICKEL	3	4
+Brand#52	LARGE BRUSHED NICKEL	19	4
+Brand#52	LARGE BRUSHED NICKEL	23	4
+Brand#52	LARGE BRUSHED STEEL	49	4
+Brand#52	LARGE BRUSHED TIN	14	4
+Brand#52	LARGE BURNISHED NICKEL	9	4
+Brand#52	LARGE BURNISHED TIN	23	4
+Brand#52	LARGE BURNISHED TIN	45	4
+Brand#52	LARGE PLATED BRASS	14	4
+Brand#52	LARGE PLATED COPPER	14	4
+Brand#52	LARGE PLATED COPPER	19	4
+Brand#52	LARGE PLATED NICKEL	45	4
+Brand#52	LARGE PLATED STEEL	9	4
+Brand#52	LARGE PLATED TIN	9	4
+Brand#52	LARGE POLISHED NICKEL	19	4
+Brand#52	LARGE POLISHED NICKEL	23	4
+Brand#52	LARGE POLISHED NICKEL	36	4
+Brand#52	LARGE POLISHED TIN	9	4
+Brand#52	MEDIUM ANODIZED COPPER	36	4
+Brand#52	MEDIUM ANODIZED STEEL	14	4
+Brand#52	MEDIUM ANODIZED TIN	3	4
+Brand#52	MEDIUM ANODIZED TIN	49	4
+Brand#52	MEDIUM BRUSHED COPPER	9	4
+Brand#52	MEDIUM BRUSHED NICKEL	9	4
+Brand#52	MEDIUM BRUSHED STEEL	23	4
+Brand#52	MEDIUM BRUSHED STEEL	49	4
+Brand#52	MEDIUM BURNISHED STEEL	23	4
+Brand#52	MEDIUM BURNISHED TIN	45	4
+Brand#52	MEDIUM BURNISHED TIN	49	4
+Brand#52	MEDIUM PLATED BRASS	36	4
+Brand#52	MEDIUM PLATED STEEL	9	4
+Brand#52	MEDIUM PLATED STEEL	49	4
+Brand#52	MEDIUM PLATED TIN	9	4
+Brand#52	MEDIUM PLATED TIN	49	4
+Brand#52	PROMO ANODIZED BRASS	9	4
+Brand#52	PROMO ANODIZED BRASS	23	4
+Brand#52	PROMO ANODIZED BRASS	36	4
+Brand#52	PROMO ANODIZED NICKEL	45	4
+Brand#52	PROMO ANODIZED STEEL	36	4
+Brand#52	PROMO BRUSHED COPPER	3	4
+Brand#52	PROMO BRUSHED NICKEL	3	4
+Brand#52	PROMO BRUSHED NICKEL	49	4
+Brand#52	PROMO BRUSHED STEEL	14	4
+Brand#52	PROMO BRUSHED TIN	3	4
+Brand#52	PROMO BRUSHED TIN	19	4
+Brand#52	PROMO BRUSHED TIN	36	4
+Brand#52	PROMO BURNISHED COPPER	49	4
+Brand#52	PROMO BURNISHED NICKEL	9	4
+Brand#52	PROMO BURNISHED STEEL	9	4
+Brand#52	PROMO BURNISHED STEEL	23	4
+Brand#52	PROMO BURNISHED TIN	19	4
+Brand#52	PROMO BURNISHED TIN	36	4
+Brand#52	PROMO PLATED BRASS	19	4
+Brand#52	PROMO PLATED BRASS	45	4
+Brand#52	PROMO PLATED BRASS	49	4
+Brand#52	PROMO PLATED COPPER	9	4
+Brand#52	PROMO PLATED NICKEL	3	4
+Brand#52	PROMO PLATED NICKEL	23	4
+Brand#52	PROMO POLISHED NICKEL	14	4
+Brand#52	PROMO POLISHED NICKEL	49	4
+Brand#52	PROMO POLISHED TIN	36	4
+Brand#52	SMALL ANODIZED BRASS	3	4
+Brand#52	SMALL ANODIZED BRASS	14	4
+Brand#52	SMALL ANODIZED COPPER	3	4
+Brand#52	SMALL ANODIZED NICKEL	36	4
+Brand#52	SMALL ANODIZED STEEL	9	4
+Brand#52	SMALL ANODIZED STEEL	19	4
+Brand#52	SMALL BRUSHED NICKEL	19	4
+Brand#52	SMALL BRUSHED STEEL	23	4
+Brand#52	SMALL BRUSHED TIN	14	4
+Brand#52	SMALL BRUSHED TIN	19	4
+Brand#52	SMALL BURNISHED NICKEL	14	4
+Brand#52	SMALL BURNISHED NICKEL	49	4
+Brand#52	SMALL BURNISHED TIN	9	4
+Brand#52	SMALL POLISHED BRASS	36	4
+Brand#52	SMALL POLISHED BRASS	49	4
+Brand#52	SMALL POLISHED TIN	45	4
+Brand#52	STANDARD ANODIZED BRASS	45	4
+Brand#52	STANDARD BRUSHED BRASS	23	4
+Brand#52	STANDARD BRUSHED COPPER	14	4
+Brand#52	STANDARD BRUSHED TIN	36	4
+Brand#52	STANDARD BURNISHED BRASS	49	4
+Brand#52	STANDARD BURNISHED STEEL	19	4
+Brand#52	STANDARD BURNISHED TIN	9	4
+Brand#52	STANDARD BURNISHED TIN	19	4
+Brand#52	STANDARD PLATED NICKEL	36	4
+Brand#52	STANDARD PLATED STEEL	36	4
+Brand#52	STANDARD POLISHED BRASS	36	4
+Brand#52	STANDARD POLISHED COPPER	45	4
+Brand#52	STANDARD POLISHED STEEL	19	4
+Brand#52	STANDARD POLISHED TIN	19	4
+Brand#53	ECONOMY ANODIZED BRASS	45	4
+Brand#53	ECONOMY ANODIZED COPPER	9	4
+Brand#53	ECONOMY ANODIZED NICKEL	3	4
+Brand#53	ECONOMY ANODIZED NICKEL	19	4
+Brand#53	ECONOMY ANODIZED STEEL	45	4
+Brand#53	ECONOMY ANODIZED TIN	14	4
+Brand#53	ECONOMY ANODIZED TIN	36	4
+Brand#53	ECONOMY BRUSHED TIN	45	4
+Brand#53	ECONOMY BURNISHED BRASS	14	4
+Brand#53	ECONOMY BURNISHED COPPER	45	4
+Brand#53	ECONOMY BURNISHED NICKEL	3	4
+Brand#53	ECONOMY BURNISHED NICKEL	49	4
+Brand#53	ECONOMY BURNISHED TIN	45	4
+Brand#53	ECONOMY PLATED BRASS	3	4
+Brand#53	ECONOMY PLATED NICKEL	14	4
+Brand#53	ECONOMY PLATED STEEL	23	4
+Brand#53	ECONOMY PLATED STEEL	36	4
+Brand#53	ECONOMY POLISHED TIN	36	4
+Brand#53	LARGE ANODIZED NICKEL	49	4
+Brand#53	LARGE ANODIZED STEEL	19	4
+Brand#53	LARGE BRUSHED COPPER	3	4
+Brand#53	LARGE BRUSHED COPPER	14	4
+Brand#53	LARGE BRUSHED NICKEL	23	4
+Brand#53	LARGE BRUSHED NICKEL	36	4
+Brand#53	LARGE BRUSHED TIN	36	4
+Brand#53	LARGE BURNISHED BRASS	45	4
+Brand#53	LARGE BURNISHED COPPER	19	4
+Brand#53	LARGE BURNISHED COPPER	36	4
+Brand#53	LARGE BURNISHED NICKEL	23	4
+Brand#53	LARGE BURNISHED STEEL	19	4
+Brand#53	LARGE BURNISHED STEEL	23	4
+Brand#53	LARGE PLATED BRASS	9	4
+Brand#53	LARGE PLATED BRASS	45	4
+Brand#53	LARGE PLATED BRASS	49	4
+Brand#53	LARGE PLATED COPPER	23	4
+Brand#53	LARGE PLATED NICKEL	23	4
+Brand#53	LARGE PLATED NICKEL	49	4
+Brand#53	LARGE PLATED STEEL	49	4
+Brand#53	LARGE PLATED TIN	14	4
+Brand#53	LARGE POLISHED COPPER	49	4
+Brand#53	LARGE POLISHED STEEL	36	4
+Brand#53	LARGE POLISHED TIN	9	4
+Brand#53	MEDIUM ANODIZED BRASS	23	4
+Brand#53	MEDIUM ANODIZED STEEL	14	4
+Brand#53	MEDIUM ANODIZED STEEL	36	4
+Brand#53	MEDIUM ANODIZED TIN	3	4
+Brand#53	MEDIUM ANODIZED TIN	9	4
+Brand#53	MEDIUM BRUSHED BRASS	3	4
+Brand#53	MEDIUM BRUSHED COPPER	3	4
+Brand#53	MEDIUM BRUSHED NICKEL	14	4
+Brand#53	MEDIUM BRUSHED NICKEL	36	4
+Brand#53	MEDIUM BRUSHED NICKEL	49	4
+Brand#53	MEDIUM BRUSHED STEEL	45	4
+Brand#53	MEDIUM BURNISHED BRASS	3	4
+Brand#53	MEDIUM BURNISHED BRASS	36	4
+Brand#53	MEDIUM BURNISHED TIN	9	4
+Brand#53	MEDIUM BURNISHED TIN	14	4
+Brand#53	MEDIUM BURNISHED TIN	36	4
+Brand#53	MEDIUM PLATED BRASS	23	4
+Brand#53	MEDIUM PLATED COPPER	14	4
+Brand#53	MEDIUM PLATED NICKEL	45	4
+Brand#53	MEDIUM PLATED TIN	19	4
+Brand#53	MEDIUM PLATED TIN	45	4
+Brand#53	PROMO ANODIZED BRASS	36	4
+Brand#53	PROMO ANODIZED NICKEL	3	4
+Brand#53	PROMO ANODIZED NICKEL	19	4
+Brand#53	PROMO BRUSHED BRASS	45	4
+Brand#53	PROMO BRUSHED COPPER	3	4
+Brand#53	PROMO BRUSHED COPPER	23	4
+Brand#53	PROMO BRUSHED COPPER	45	4
+Brand#53	PROMO BURNISHED BRASS	23	4
+Brand#53	PROMO BURNISHED BRASS	36	4
+Brand#53	PROMO BURNISHED NICKEL	23	4
+Brand#53	PROMO BURNISHED STEEL	23	4
+Brand#53	PROMO BURNISHED STEEL	49	4
+Brand#53	PROMO PLATED TIN	19	4
+Brand#53	PROMO PLATED TIN	23	4
+Brand#53	PROMO PLATED TIN	36	4
+Brand#53	PROMO POLISHED STEEL	23	4
+Brand#53	PROMO POLISHED TIN	3	4
+Brand#53	SMALL ANODIZED COPPER	23	4
+Brand#53	SMALL ANODIZED COPPER	36	4
+Brand#53	SMALL ANODIZED COPPER	49	4
+Brand#53	SMALL ANODIZED NICKEL	36	4
+Brand#53	SMALL BRUSHED BRASS	36	4
+Brand#53	SMALL BRUSHED COPPER	3	4
+Brand#53	SMALL BRUSHED TIN	3	4
+Brand#53	SMALL BRUSHED TIN	36	4
+Brand#53	SMALL BURNISHED BRASS	9	4
+Brand#53	SMALL BURNISHED BRASS	49	4
+Brand#53	SMALL BURNISHED COPPER	19	4
+Brand#53	SMALL BURNISHED COPPER	45	4
+Brand#53	SMALL PLATED BRASS	9	4
+Brand#53	SMALL PLATED COPPER	3	4
+Brand#53	SMALL PLATED NICKEL	14	4
+Brand#53	SMALL POLISHED NICKEL	19	4
+Brand#53	SMALL POLISHED STEEL	36	4
+Brand#53	SMALL POLISHED TIN	23	4
+Brand#53	STANDARD ANODIZED BRASS	14	4
+Brand#53	STANDARD ANODIZED NICKEL	9	4
+Brand#53	STANDARD ANODIZED NICKEL	23	4
+Brand#53	STANDARD ANODIZED NICKEL	45	4
+Brand#53	STANDARD ANODIZED STEEL	45	4
+Brand#53	STANDARD BRUSHED COPPER	3	4
+Brand#53	STANDARD BRUSHED NICKEL	23	4
+Brand#53	STANDARD BRUSHED TIN	14	4
+Brand#53	STANDARD BURNISHED NICKEL	49	4
+Brand#53	STANDARD BURNISHED STEEL	9	4
+Brand#53	STANDARD PLATED BRASS	36	4
+Brand#53	STANDARD PLATED COPPER	45	4
+Brand#53	STANDARD PLATED NICKEL	36	4
+Brand#53	STANDARD PLATED STEEL	3	4
+Brand#53	STANDARD PLATED STEEL	49	4
+Brand#53	STANDARD PLATED TIN	23	4
+Brand#53	STANDARD POLISHED STEEL	3	4
+Brand#54	ECONOMY ANODIZED BRASS	9	4
+Brand#54	ECONOMY ANODIZED BRASS	45	4
+Brand#54	ECONOMY ANODIZED COPPER	9	4
+Brand#54	ECONOMY ANODIZED STEEL	19	4
+Brand#54	ECONOMY BRUSHED BRASS	45	4
+Brand#54	ECONOMY BRUSHED NICKEL	19	4
+Brand#54	ECONOMY BRUSHED STEEL	3	4
+Brand#54	ECONOMY BRUSHED TIN	19	4
+Brand#54	ECONOMY BURNISHED BRASS	45	4
+Brand#54	ECONOMY BURNISHED COPPER	14	4
+Brand#54	ECONOMY BURNISHED NICKEL	9	4
+Brand#54	ECONOMY BURNISHED NICKEL	36	4
+Brand#54	ECONOMY BURNISHED STEEL	36	4
+Brand#54	ECONOMY BURNISHED TIN	9	4
+Brand#54	ECONOMY BURNISHED TIN	14	4
+Brand#54	ECONOMY BURNISHED TIN	23	4
+Brand#54	ECONOMY PLATED TIN	23	4
+Brand#54	ECONOMY POLISHED BRASS	9	4
+Brand#54	ECONOMY POLISHED BRASS	19	4
+Brand#54	ECONOMY POLISHED COPPER	23	4
+Brand#54	ECONOMY POLISHED STEEL	23	4
+Brand#54	ECONOMY POLISHED TIN	3	4
+Brand#54	LARGE ANODIZED BRASS	14	4
+Brand#54	LARGE ANODIZED BRASS	49	4
+Brand#54	LARGE ANODIZED TIN	9	4
+Brand#54	LARGE BRUSHED BRASS	14	4
+Brand#54	LARGE BRUSHED STEEL	9	4
+Brand#54	LARGE BRUSHED STEEL	23	4
+Brand#54	LARGE BRUSHED TIN	14	4
+Brand#54	LARGE BURNISHED BRASS	49	4
+Brand#54	LARGE BURNISHED COPPER	19	4
+Brand#54	LARGE BURNISHED NICKEL	14	4
+Brand#54	LARGE BURNISHED TIN	14	4
+Brand#54	LARGE PLATED BRASS	19	4
+Brand#54	LARGE PLATED BRASS	23	4
+Brand#54	LARGE POLISHED BRASS	19	4
+Brand#54	LARGE POLISHED BRASS	23	4
+Brand#54	LARGE POLISHED NICKEL	3	4
+Brand#54	LARGE POLISHED NICKEL	14	4
+Brand#54	LARGE POLISHED STEEL	19	4
+Brand#54	LARGE POLISHED TIN	3	4
+Brand#54	LARGE POLISHED TIN	9	4
+Brand#54	LARGE POLISHED TIN	36	4
+Brand#54	MEDIUM ANODIZED NICKEL	9	4
+Brand#54	MEDIUM ANODIZED NICKEL	14	4
+Brand#54	MEDIUM ANODIZED NICKEL	36	4
+Brand#54	MEDIUM BRUSHED NICKEL	9	4
+Brand#54	MEDIUM BRUSHED NICKEL	19	4
+Brand#54	MEDIUM BURNISHED STEEL	3	4
+Brand#54	MEDIUM BURNISHED STEEL	19	4
+Brand#54	MEDIUM BURNISHED STEEL	23	4
+Brand#54	MEDIUM PLATED BRASS	3	4
+Brand#54	MEDIUM PLATED NICKEL	45	4
+Brand#54	PROMO ANODIZED NICKEL	45	4
+Brand#54	PROMO BRUSHED BRASS	3	4
+Brand#54	PROMO BRUSHED STEEL	23	4
+Brand#54	PROMO BRUSHED TIN	14	4
+Brand#54	PROMO BURNISHED COPPER	49	4
+Brand#54	PROMO BURNISHED TIN	9	4
+Brand#54	PROMO PLATED BRASS	14	4
+Brand#54	PROMO PLATED NICKEL	3	4
+Brand#54	PROMO PLATED STEEL	19	4
+Brand#54	PROMO PLATED TIN	23	4
+Brand#54	PROMO PLATED TIN	49	4
+Brand#54	PROMO POLISHED BRASS	3	4
+Brand#54	PROMO POLISHED NICKEL	9	4
+Brand#54	PROMO POLISHED TIN	49	4
+Brand#54	SMALL ANODIZED COPPER	49	4
+Brand#54	SMALL ANODIZED NICKEL	9	4
+Brand#54	SMALL ANODIZED NICKEL	36	4
+Brand#54	SMALL ANODIZED TIN	19	4
+Brand#54	SMALL BRUSHED BRASS	14	4
+Brand#54	SMALL BRUSHED BRASS	19	4
+Brand#54	SMALL BRUSHED BRASS	36	4
+Brand#54	SMALL BRUSHED COPPER	3	4
+Brand#54	SMALL BRUSHED COPPER	9	4
+Brand#54	SMALL BRUSHED COPPER	19	4
+Brand#54	SMALL BRUSHED TIN	9	4
+Brand#54	SMALL BRUSHED TIN	36	4
+Brand#54	SMALL BURNISHED COPPER	9	4
+Brand#54	SMALL BURNISHED COPPER	36	4
+Brand#54	SMALL BURNISHED STEEL	14	4
+Brand#54	SMALL BURNISHED STEEL	19	4
+Brand#54	SMALL BURNISHED TIN	9	4
+Brand#54	SMALL BURNISHED TIN	36	4
+Brand#54	SMALL PLATED BRASS	23	4
+Brand#54	SMALL PLATED COPPER	9	4
+Brand#54	SMALL PLATED COPPER	36	4
+Brand#54	SMALL PLATED COPPER	49	4
+Brand#54	SMALL PLATED NICKEL	9	4
+Brand#54	SMALL PLATED TIN	23	4
+Brand#54	SMALL PLATED TIN	36	4
+Brand#54	SMALL POLISHED BRASS	9	4
+Brand#54	SMALL POLISHED COPPER	9	4
+Brand#54	SMALL POLISHED TIN	9	4
+Brand#54	STANDARD ANODIZED BRASS	3	4
+Brand#54	STANDARD ANODIZED BRASS	9	4
+Brand#54	STANDARD ANODIZED COPPER	3	4
+Brand#54	STANDARD ANODIZED TIN	3	4
+Brand#54	STANDARD BRUSHED COPPER	3	4
+Brand#54	STANDARD BRUSHED NICKEL	45	4
+Brand#54	STANDARD BRUSHED TIN	36	4
+Brand#54	STANDARD BURNISHED BRASS	23	4
+Brand#54	STANDARD BURNISHED BRASS	49	4
+Brand#54	STANDARD BURNISHED COPPER	19	4
+Brand#54	STANDARD BURNISHED NICKEL	23	4
+Brand#54	STANDARD BURNISHED STEEL	45	4
+Brand#54	STANDARD PLATED BRASS	3	4
+Brand#54	STANDARD PLATED BRASS	45	4
+Brand#54	STANDARD PLATED BRASS	49	4
+Brand#54	STANDARD PLATED STEEL	3	4
+Brand#54	STANDARD POLISHED BRASS	36	4
+Brand#54	STANDARD POLISHED STEEL	3	4
+Brand#54	STANDARD POLISHED STEEL	14	4
+Brand#54	STANDARD POLISHED STEEL	45	4
+Brand#55	ECONOMY ANODIZED BRASS	3	4
+Brand#55	ECONOMY BRUSHED BRASS	19	4
+Brand#55	ECONOMY BRUSHED COPPER	9	4
+Brand#55	ECONOMY BRUSHED COPPER	23	4
+Brand#55	ECONOMY BRUSHED COPPER	45	4
+Brand#55	ECONOMY BRUSHED STEEL	23	4
+Brand#55	ECONOMY BURNISHED NICKEL	36	4
+Brand#55	ECONOMY BURNISHED NICKEL	45	4
+Brand#55	ECONOMY BURNISHED TIN	45	4
+Brand#55	ECONOMY PLATED NICKEL	19	4
+Brand#55	ECONOMY POLISHED NICKEL	9	4
+Brand#55	LARGE BRUSHED BRASS	23	4
+Brand#55	LARGE BRUSHED BRASS	45	4
+Brand#55	LARGE BRUSHED COPPER	49	4
+Brand#55	LARGE BRUSHED NICKEL	9	4
+Brand#55	LARGE BRUSHED NICKEL	14	4
+Brand#55	LARGE BURNISHED BRASS	3	4
+Brand#55	LARGE BURNISHED COPPER	14	4
+Brand#55	LARGE BURNISHED COPPER	36	4
+Brand#55	LARGE PLATED BRASS	45	4
+Brand#55	LARGE PLATED COPPER	19	4
+Brand#55	LARGE PLATED NICKEL	9	4
+Brand#55	LARGE PLATED STEEL	9	4
+Brand#55	LARGE PLATED TIN	9	4
+Brand#55	LARGE PLATED TIN	14	4
+Brand#55	LARGE PLATED TIN	23	4
+Brand#55	LARGE POLISHED NICKEL	3	4
+Brand#55	LARGE POLISHED STEEL	36	4
+Brand#55	LARGE POLISHED STEEL	45	4
+Brand#55	MEDIUM ANODIZED COPPER	9	4
+Brand#55	MEDIUM BRUSHED BRASS	3	4
+Brand#55	MEDIUM BRUSHED NICKEL	23	4
+Brand#55	MEDIUM BRUSHED TIN	45	4
+Brand#55	MEDIUM BURNISHED BRASS	23	4
+Brand#55	MEDIUM BURNISHED COPPER	36	4
+Brand#55	MEDIUM BURNISHED NICKEL	3	4
+Brand#55	MEDIUM BURNISHED STEEL	14	4
+Brand#55	MEDIUM BURNISHED STEEL	36	4
+Brand#55	MEDIUM PLATED NICKEL	23	4
+Brand#55	PROMO ANODIZED COPPER	14	4
+Brand#55	PROMO ANODIZED COPPER	49	4
+Brand#55	PROMO ANODIZED STEEL	36	4
+Brand#55	PROMO ANODIZED TIN	23	4
+Brand#55	PROMO BRUSHED NICKEL	36	4
+Brand#55	PROMO BRUSHED STEEL	3	4
+Brand#55	PROMO BRUSHED STEEL	36	4
+Brand#55	PROMO BRUSHED TIN	9	4
+Brand#55	PROMO BURNISHED COPPER	3	4
+Brand#55	PROMO BURNISHED STEEL	14	4
+Brand#55	PROMO BURNISHED TIN	23	4
+Brand#55	PROMO BURNISHED TIN	49	4
+Brand#55	PROMO PLATED COPPER	3	4
+Brand#55	PROMO PLATED NICKEL	3	4
+Brand#55	PROMO PLATED NICKEL	14	4
+Brand#55	PROMO PLATED NICKEL	23	4
+Brand#55	PROMO PLATED TIN	3	4
+Brand#55	PROMO POLISHED COPPER	3	4
+Brand#55	SMALL ANODIZED BRASS	19	4
+Brand#55	SMALL ANODIZED NICKEL	45	4
+Brand#55	SMALL BRUSHED COPPER	14	4
+Brand#55	SMALL BRUSHED COPPER	45	4
+Brand#55	SMALL BURNISHED BRASS	14	4
+Brand#55	SMALL BURNISHED TIN	3	4
+Brand#55	SMALL BURNISHED TIN	49	4
+Brand#55	SMALL PLATED BRASS	45	4
+Brand#55	SMALL PLATED COPPER	23	4
+Brand#55	SMALL PLATED COPPER	36	4
+Brand#55	SMALL PLATED COPPER	45	4
+Brand#55	SMALL PLATED COPPER	49	4
+Brand#55	SMALL PLATED NICKEL	9	4
+Brand#55	SMALL PLATED STEEL	9	4
+Brand#55	SMALL PLATED TIN	14	4
+Brand#55	SMALL PLATED TIN	36	4
+Brand#55	SMALL POLISHED NICKEL	45	4
+Brand#55	SMALL POLISHED STEEL	19	4
+Brand#55	SMALL POLISHED TIN	19	4
+Brand#55	STANDARD ANODIZED BRASS	36	4
+Brand#55	STANDARD ANODIZED BRASS	49	4
+Brand#55	STANDARD ANODIZED STEEL	19	4
+Brand#55	STANDARD ANODIZED TIN	36	4
+Brand#55	STANDARD ANODIZED TIN	49	4
+Brand#55	STANDARD BRUSHED BRASS	36	4
+Brand#55	STANDARD BRUSHED COPPER	3	4
+Brand#55	STANDARD BRUSHED COPPER	9	4
+Brand#55	STANDARD BRUSHED COPPER	23	4
+Brand#55	STANDARD BRUSHED STEEL	19	4
+Brand#55	STANDARD BRUSHED TIN	23	4
+Brand#55	STANDARD BRUSHED TIN	45	4
+Brand#55	STANDARD BURNISHED BRASS	19	4
+Brand#55	STANDARD BURNISHED NICKEL	3	4
+Brand#55	STANDARD BURNISHED NICKEL	36	4
+Brand#55	STANDARD BURNISHED STEEL	19	4
+Brand#55	STANDARD PLATED BRASS	23	4
+Brand#55	STANDARD PLATED NICKEL	9	4
+Brand#55	STANDARD PLATED TIN	36	4
+Brand#55	STANDARD POLISHED BRASS	3	4
+Brand#55	STANDARD POLISHED BRASS	49	4
+Brand#55	STANDARD POLISHED COPPER	19	4
+Brand#55	STANDARD POLISHED COPPER	36	4
+Brand#55	STANDARD POLISHED NICKEL	14	4
+Brand#55	STANDARD POLISHED STEEL	9	4
+Brand#55	STANDARD POLISHED STEEL	36	4
+Brand#12	LARGE BURNISHED NICKEL	14	3
+Brand#12	PROMO POLISHED TIN	3	3
+Brand#21	MEDIUM ANODIZED TIN	9	3
+Brand#22	PROMO BRUSHED BRASS	19	3
+Brand#22	PROMO BURNISHED COPPER	14	3
+Brand#43	STANDARD BRUSHED BRASS	23	3
+Brand#44	MEDIUM ANODIZED NICKEL	9	3
+Brand#53	MEDIUM BURNISHED BRASS	49	3
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q17.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q17.out
new file mode 100644
index 00000000000000..fbe66eba846ec7
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q17.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q17 --
+23512.752857
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q18.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q18.out
new file mode 100644
index 00000000000000..55ec421f0555e0
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q18.out
@@ -0,0 +1,8 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q18 --
+Customer#000001639	1639	502886	1994-04-12	456423.88	312.00
+Customer#000006655	6655	29158	1995-10-21	452805.02	305.00
+Customer#000014110	14110	565574	1995-09-24	425099.85	301.00
+Customer#000001775	1775	6882	1997-04-09	408368.10	303.00
+Customer#000011459	11459	551136	1993-05-19	386812.74	308.00
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q19.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q19.out
new file mode 100644
index 00000000000000..e9d14e6a51eeb6
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q19.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q19 --
+168597.2860
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q20.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q20.out
new file mode 100644
index 00000000000000..66dc3b4db568c8
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q20.out
@@ -0,0 +1,12 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q20 --
+Supplier#000000157	,mEGorBfVIm
+Supplier#000000197	YC2Acon6kjY3zj3Fbxs2k4Vdf7X0cd2F
+Supplier#000000287	7a9SP7qW5Yku5PvSg
+Supplier#000000378	FfbhyCxWvcPrO8ltp9
+Supplier#000000530	0qwCMwobKY OcmLyfRXlagA8ukENJv,
+Supplier#000000555	TfB,a5bfl3Ah 3Z 74GqnNs6zKVGM
+Supplier#000000557	jj0wUYh9K3fG5Jhdhrkuy ,4
+Supplier#000000729	pqck2ppy758TQpZCUAjPvlU55K3QjfL7Bi
+Supplier#000000935	ij98czM 2KzWe7dDTOxB8sq0UfCdvrX
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q21.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q21.out
new file mode 100644
index 00000000000000..b03644955e707c
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q21.out
@@ -0,0 +1,50 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q21 --
+Supplier#000000445	16
+Supplier#000000825	16
+Supplier#000000709	15
+Supplier#000000762	15
+Supplier#000000357	14
+Supplier#000000399	14
+Supplier#000000496	14
+Supplier#000000977	13
+Supplier#000000144	12
+Supplier#000000188	12
+Supplier#000000415	12
+Supplier#000000472	12
+Supplier#000000633	12
+Supplier#000000708	12
+Supplier#000000889	12
+Supplier#000000380	11
+Supplier#000000602	11
+Supplier#000000659	11
+Supplier#000000821	11
+Supplier#000000929	11
+Supplier#000000262	10
+Supplier#000000460	10
+Supplier#000000486	10
+Supplier#000000669	10
+Supplier#000000718	10
+Supplier#000000778	10
+Supplier#000000167	9
+Supplier#000000578	9
+Supplier#000000673	9
+Supplier#000000687	9
+Supplier#000000074	8
+Supplier#000000565	8
+Supplier#000000648	8
+Supplier#000000918	8
+Supplier#000000427	7
+Supplier#000000503	7
+Supplier#000000610	7
+Supplier#000000670	7
+Supplier#000000811	7
+Supplier#000000114	6
+Supplier#000000379	6
+Supplier#000000436	6
+Supplier#000000500	6
+Supplier#000000660	6
+Supplier#000000788	6
+Supplier#000000846	6
+Supplier#000000920	4
+
diff --git a/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q22.out b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q22.out
new file mode 100644
index 00000000000000..76407081f65495
--- /dev/null
+++ b/regression-test/data/tpch_unique_sql_zstd_bucket1_p0/sql/q22.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !q22 --
+13	94	714035.05
+17	96	722560.15
+18	99	738012.52
+23	93	708285.25
+29	85	632693.46
+30	87	646748.02
+31	87	647372.50
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer.sql
new file mode 100644
index 00000000000000..e6fd43649d215e
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer.sql
@@ -0,0 +1,17 @@
+CREATE TABLE IF NOT EXISTS customer (
+  C_CUSTKEY     INTEGER NOT NULL,
+  C_NAME        VARCHAR(25) NOT NULL,
+  C_ADDRESS     VARCHAR(40) NOT NULL,
+  C_NATIONKEY   INTEGER NOT NULL,
+  C_PHONE       CHAR(15) NOT NULL,
+  C_ACCTBAL     DECIMAL(15,2)   NOT NULL,
+  C_MKTSEGMENT  CHAR(10) NOT NULL,
+  C_COMMENT     VARCHAR(117) NOT NULL
+)
+UNIQUE KEY(C_CUSTKEY, C_NAME)
+DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer_delete.sql
new file mode 100644
index 00000000000000..a214fe920c914c
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/customer_delete.sql
@@ -0,0 +1 @@
+DELETE FROM customer where C_CUSTKEY >= 0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem.sql
new file mode 100644
index 00000000000000..1e9d4756c240e3
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem.sql
@@ -0,0 +1,25 @@
+CREATE TABLE IF NOT EXISTS lineitem (
+  L_ORDERKEY    INTEGER NOT NULL,
+  L_PARTKEY     INTEGER NOT NULL,
+  L_SUPPKEY     INTEGER NOT NULL,
+  L_LINENUMBER  INTEGER NOT NULL,
+  L_QUANTITY    DECIMAL(15,2) NOT NULL,
+  L_EXTENDEDPRICE  DECIMAL(15,2) NOT NULL,
+  L_DISCOUNT    DECIMAL(15,2) NOT NULL,
+  L_TAX         DECIMAL(15,2) NOT NULL,
+  L_RETURNFLAG  CHAR(1) NOT NULL,
+  L_LINESTATUS  CHAR(1) NOT NULL,
+  L_SHIPDATE    DATE NOT NULL,
+  L_COMMITDATE  DATE NOT NULL,
+  L_RECEIPTDATE DATE NOT NULL,
+  L_SHIPINSTRUCT CHAR(25) NOT NULL,
+  L_SHIPMODE     CHAR(10) NOT NULL,
+  L_COMMENT      VARCHAR(44) NOT NULL
+)
+UNIQUE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem_delete.sql
new file mode 100644
index 00000000000000..6dbbe49ed0b52d
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/lineitem_delete.sql
@@ -0,0 +1 @@
+DELETE from lineitem where L_ORDERKEY >= 0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation.sql
new file mode 100644
index 00000000000000..248b257b523cf0
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation.sql
@@ -0,0 +1,13 @@
+CREATE TABLE IF NOT EXISTS nation  (
+  N_NATIONKEY  INTEGER NOT NULL,
+  N_NAME       CHAR(25) NOT NULL,
+  N_REGIONKEY  INTEGER NOT NULL,
+  N_COMMENT    VARCHAR(152)
+)
+UNIQUE KEY(N_NATIONKEY, N_NAME)
+DISTRIBUTED BY HASH(N_NATIONKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation_delete.sql
new file mode 100644
index 00000000000000..9fb7c085a6fea3
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/nation_delete.sql
@@ -0,0 +1 @@
+DELETE from nation where N_NATIONKEY >= 0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders.sql
new file mode 100644
index 00000000000000..d138a62da366ca
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders.sql
@@ -0,0 +1,18 @@
+CREATE TABLE IF NOT EXISTS orders  (
+  O_ORDERKEY       INTEGER NOT NULL,
+  O_CUSTKEY        INTEGER NOT NULL,
+  O_ORDERSTATUS    CHAR(1) NOT NULL,
+  O_TOTALPRICE     DECIMAL(15,2) NOT NULL,
+  O_ORDERDATE      DATE NOT NULL,
+  O_ORDERPRIORITY  CHAR(15) NOT NULL,  
+  O_CLERK          CHAR(15) NOT NULL, 
+  O_SHIPPRIORITY   INTEGER NOT NULL,
+  O_COMMENT        VARCHAR(79) NOT NULL
+)
+UNIQUE KEY(O_ORDERKEY, O_CUSTKEY)
+DISTRIBUTED BY HASH(O_ORDERKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders_delete.sql
new file mode 100644
index 00000000000000..5199d8f296dc90
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/orders_delete.sql
@@ -0,0 +1 @@
+delete from orders where O_ORDERKEY >= 0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part.sql
new file mode 100644
index 00000000000000..eb9d74fd598656
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part.sql
@@ -0,0 +1,18 @@
+CREATE TABLE IF NOT EXISTS part (
+  P_PARTKEY     INTEGER NOT NULL,
+  P_NAME        VARCHAR(55) NOT NULL,
+  P_MFGR        CHAR(25) NOT NULL,
+  P_BRAND       CHAR(10) NOT NULL,
+  P_TYPE        VARCHAR(25) NOT NULL,
+  P_SIZE        INTEGER NOT NULL,
+  P_CONTAINER   CHAR(10) NOT NULL,
+  P_RETAILPRICE DECIMAL(15,2) NOT NULL,
+  P_COMMENT     VARCHAR(23) NOT NULL 
+)
+UNIQUE KEY(P_PARTKEY, P_NAME)
+DISTRIBUTED BY HASH(P_PARTKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part_delete.sql
new file mode 100644
index 00000000000000..ee1a2e5104503a
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/part_delete.sql
@@ -0,0 +1 @@
+delete from part where P_PARTKEY>=0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp.sql
new file mode 100644
index 00000000000000..2ec8accabfa11c
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp.sql
@@ -0,0 +1,14 @@
+CREATE TABLE IF NOT EXISTS partsupp (
+  PS_PARTKEY     INTEGER NOT NULL,
+  PS_SUPPKEY     INTEGER NOT NULL,
+  PS_AVAILQTY    INTEGER NOT NULL,
+  PS_SUPPLYCOST  DECIMAL(15,2)  NOT NULL,
+  PS_COMMENT     VARCHAR(199) NOT NULL 
+)
+UNIQUE KEY(PS_PARTKEY, PS_SUPPKEY)
+DISTRIBUTED BY HASH(PS_PARTKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp_delete.sql
new file mode 100644
index 00000000000000..067cff3a6f0f77
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/partsupp_delete.sql
@@ -0,0 +1 @@
+delete from partsupp where PS_PARTKEY>=0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region.sql
new file mode 100644
index 00000000000000..4791cf0c14abec
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region.sql
@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS region (
+  R_REGIONKEY  INTEGER NOT NULL,
+  R_NAME       CHAR(25) NOT NULL,
+  R_COMMENT    VARCHAR(152)
+)
+UNIQUE KEY(R_REGIONKEY, R_NAME)
+DISTRIBUTED BY HASH(R_REGIONKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region_delete.sql
new file mode 100644
index 00000000000000..4945d5105ab6cf
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/region_delete.sql
@@ -0,0 +1 @@
+delete from region where R_REGIONKEY>=0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1.sql
new file mode 100644
index 00000000000000..a12a60a2848a2d
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1.sql
@@ -0,0 +1,11 @@
+CREATE VIEW IF NOT EXISTS revenue1 AS
+  SELECT
+    l_suppkey AS supplier_no,
+    sum(l_extendedprice * (1 - l_discount)) AS total_revenue
+  FROM
+    lineitem
+  WHERE
+    l_shipdate >= DATE '1996-01-01'
+    AND l_shipdate < DATE '1996-01-01' + INTERVAL '3' MONTH
+GROUP BY
+  l_suppkey;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1_delete.sql
new file mode 100644
index 00000000000000..12b50f19605a16
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/revenue1_delete.sql
@@ -0,0 +1 @@
+drop view IF EXISTS revenue1;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier.sql
new file mode 100644
index 00000000000000..e98ec7baf80e69
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier.sql
@@ -0,0 +1,15 @@
+CREATE TABLE IF NOT EXISTS supplier (
+  S_SUPPKEY     INTEGER NOT NULL,
+  S_NAME        CHAR(25) NOT NULL,
+  S_ADDRESS     VARCHAR(40) NOT NULL,
+  S_NATIONKEY   INTEGER NOT NULL,
+  S_PHONE       CHAR(15) NOT NULL,
+  S_ACCTBAL     DECIMAL(15,2) NOT NULL,
+  S_COMMENT     VARCHAR(101) NOT NULL
+)
+UNIQUE KEY(S_SUPPKEY, S_NAME)
+DISTRIBUTED BY HASH(S_SUPPKEY) BUCKETS 1
+PROPERTIES (
+  "compression"="zstd",
+  "replication_num" = "1"
+)
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier_delete.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier_delete.sql
new file mode 100644
index 00000000000000..d8327a8f856334
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/ddl/supplier_delete.sql
@@ -0,0 +1 @@
+delete from supplier where S_SUPPKEY>=0;
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/load.groovy b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/load.groovy
new file mode 100644
index 00000000000000..444c7d413ef486
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/load.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Most of the cases are copied from https://github.com/trinodb/trino/tree/master
+// /testing/trino-product-tests/src/main/resources/sql-tests/testcases
+// and modified by Doris.
+
+// syntax error:
+// q06 q13 q15
+// Test 23 suites, failed 3 suites
+
+// Note: To filter out tables from sql files, use the following one-liner comamnd
+// sed -nr 's/.*tables: (.*)$/\1/gp' /path/to/*.sql | sed -nr 's/,/\n/gp' | sort | uniq
+suite("load") {
+    def tables = [customer: ["c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_mktsegment, c_comment,temp"],
+                  lineitem: ["l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag,l_linestatus, l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment,temp"],
+                  nation  : ["n_nationkey, n_name, n_regionkey, n_comment, temp"],
+                  orders  : ["o_orderkey, o_custkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk, o_shippriority, o_comment, temp"],
+                  part    : ["p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment, temp"],
+                  partsupp: ["ps_partkey,ps_suppkey,ps_availqty,ps_supplycost,ps_comment,temp"],
+                  region  : ["r_regionkey, r_name, r_comment,temp"],
+                  supplier: ["s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment,temp"]]
+
+    tables.forEach { tableName, columns ->
+        sql new File("""${context.file.parent}/ddl/${tableName}.sql""").text
+        sql new File("""${context.file.parent}/ddl/${tableName}_delete.sql""").text
+        streamLoad {
+            // a default db 'regression_test' is specified in
+            // ${DORIS_HOME}/conf/regression-conf.groovy
+            table "${tableName}"
+
+            // default label is UUID:
+            // set 'label' UUID.randomUUID().toString()
+
+            // default column_separator is specify in doris fe config, usually is '\t'.
+            // this line change to ','
+            set 'column_separator', '|'
+            set 'compress_type', 'GZ'
+            set 'columns', "${columns[0]}"
+
+            // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+            // also, you can stream load a http stream, e.g. http://xxx/some.csv
+            file """${getS3Url()}/regression/tpch/sf0.1/${tableName}.tbl.gz"""
+
+            time 10000 // limit inflight 10s
+
+            // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+
+            // if declared a check callback, the default check condition will ignore.
+            // So you must check all condition
+            check { result, exception, startTime, endTime ->
+                if (exception != null) {
+                    throw exception
+                }
+                log.info("Stream load result: ${result}".toString())
+                def json = parseJson(result)
+                assertEquals("success", json.Status.toLowerCase())
+                assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+                assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+            }
+        }
+        sql """ ANALYZE TABLE $tableName WITH SYNC """
+    }
+
+    def table = "revenue1"
+    sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text
+    sql new File("""${context.file.parent}/ddl/${table}.sql""").text
+
+    sql """ sync """
+}
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q01.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q01.sql
new file mode 100644
index 00000000000000..7b20cc9febb2cb
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q01.sql
@@ -0,0 +1,22 @@
+-- tables: lineitem
+SELECT
+  l_returnflag,
+  l_linestatus,
+  sum(l_quantity)                                       AS sum_qty,
+  sum(l_extendedprice)                                  AS sum_base_price,
+  sum(l_extendedprice * (1 - l_discount))               AS sum_disc_price,
+  sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge,
+  avg(l_quantity)                                       AS avg_qty,
+  avg(l_extendedprice)                                  AS avg_price,
+  avg(l_discount)                                       AS avg_disc,
+  count(*)                                              AS count_order
+FROM
+  lineitem
+WHERE
+  l_shipdate <= DATE '1998-12-01' - INTERVAL '90' DAY
+GROUP BY
+l_returnflag,
+l_linestatus
+ORDER BY
+l_returnflag,
+l_linestatus
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q02.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q02.sql
new file mode 100644
index 00000000000000..8c60ec1929364d
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q02.sql
@@ -0,0 +1,42 @@
+-- tables: part,supplier,partsupp,nation,region
+SELECT
+  s_acctbal,
+  s_name,
+  n_name,
+  p_partkey,
+  p_mfgr,
+  s_address,
+  s_phone,
+  s_comment
+FROM
+  part,
+  supplier,
+  partsupp,
+  nation,
+  region
+WHERE
+  p_partkey = ps_partkey
+  AND s_suppkey = ps_suppkey
+  AND p_size = 15
+  AND p_type LIKE '%BRASS'
+  AND s_nationkey = n_nationkey
+  AND n_regionkey = r_regionkey
+  AND r_name = 'EUROPE'
+  AND ps_supplycost = (
+    SELECT min(ps_supplycost)
+    FROM
+      partsupp, supplier,
+      nation, region
+    WHERE
+      p_partkey = ps_partkey
+      AND s_suppkey = ps_suppkey
+      AND s_nationkey = n_nationkey
+      AND n_regionkey = r_regionkey
+      AND r_name = 'EUROPE'
+  )
+ORDER BY
+  s_acctbal DESC,
+  n_name,
+  s_name,
+  p_partkey
+LIMIT 100
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q03.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q03.sql
new file mode 100644
index 00000000000000..e689d3cbb0878b
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q03.sql
@@ -0,0 +1,24 @@
+-- tables: customer,orders,lineitem
+SELECT
+  l_orderkey,
+  sum(l_extendedprice * (1 - l_discount)) AS revenue,
+  o_orderdate,
+  o_shippriority
+FROM
+  customer,
+  orders,
+  lineitem
+WHERE
+  c_mktsegment = 'BUILDING'
+  AND c_custkey = o_custkey
+  AND l_orderkey = o_orderkey
+  AND o_orderdate < DATE '1995-03-15'
+  AND l_shipdate > DATE '1995-03-15'
+GROUP BY
+  l_orderkey,
+  o_orderdate,
+  o_shippriority
+ORDER BY
+  revenue DESC,
+  o_orderdate
+LIMIT 10
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q04.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q04.sql
new file mode 100644
index 00000000000000..c4c7925eb70ff9
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q04.sql
@@ -0,0 +1,19 @@
+-- tables: orders,lineitem
+SELECT
+  o_orderpriority,
+  count(*) AS order_count
+FROM orders
+WHERE
+  o_orderdate >= DATE '1993-07-01'
+  AND o_orderdate < DATE '1993-07-01' + INTERVAL '3' MONTH
+AND EXISTS (
+SELECT *
+FROM lineitem
+WHERE
+l_orderkey = o_orderkey
+AND l_commitdate < l_receiptdate
+)
+GROUP BY
+o_orderpriority
+ORDER BY
+o_orderpriority
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q05.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q05.sql
new file mode 100644
index 00000000000000..059725c12ac8a0
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q05.sql
@@ -0,0 +1,25 @@
+-- tables: customer,orders,lineitem,supplier,nation,region
+SELECT
+  n_name,
+  sum(l_extendedprice * (1 - l_discount)) AS revenue
+FROM
+  customer,
+  orders,
+  lineitem,
+  supplier,
+  nation,
+  region
+WHERE
+  c_custkey = o_custkey
+  AND l_orderkey = o_orderkey
+  AND l_suppkey = s_suppkey
+  AND c_nationkey = s_nationkey
+  AND s_nationkey = n_nationkey
+  AND n_regionkey = r_regionkey
+  AND r_name = 'ASIA'
+  AND o_orderdate >= DATE '1994-01-01'
+  AND o_orderdate < DATE '1994-01-01' + INTERVAL '1' YEAR
+GROUP BY
+n_name
+ORDER BY
+revenue DESC
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q06.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q06.sql
new file mode 100644
index 00000000000000..e81a352c2410e7
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q06.sql
@@ -0,0 +1,11 @@
+-- tables: lineitem
+
+SELECT sum(l_extendedprice * l_discount) AS revenue
+FROM
+  lineitem
+WHERE
+  l_shipdate >= DATE '1994-01-01'
+  AND l_shipdate < DATE '1994-01-01' + INTERVAL '1' YEAR
+AND l_discount BETWEEN 0.06 - 0.01 AND .06 + 0.01
+AND l_quantity < 24
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q07.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q07.sql
new file mode 100644
index 00000000000000..edeaf35f7f0c33
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q07.sql
@@ -0,0 +1,39 @@
+-- tables: supplier,lineitem,orders,customer,nation
+SELECT
+  supp_nation,
+  cust_nation,
+  l_year,
+  sum(volume) AS revenue
+FROM (
+       SELECT
+         n1.n_name                          AS supp_nation,
+         n2.n_name                          AS cust_nation,
+         extract(YEAR FROM l_shipdate)      AS l_year,
+         l_extendedprice * (1 - l_discount) AS volume
+       FROM
+         supplier,
+         lineitem,
+         orders,
+         customer,
+         nation n1,
+         nation n2
+       WHERE
+         s_suppkey = l_suppkey
+         AND o_orderkey = l_orderkey
+         AND c_custkey = o_custkey
+         AND s_nationkey = n1.n_nationkey
+         AND c_nationkey = n2.n_nationkey
+         AND (
+           (n1.n_name = 'FRANCE' AND n2.n_name = 'GERMANY')
+           OR (n1.n_name = 'GERMANY' AND n2.n_name = 'FRANCE')
+         )
+         AND l_shipdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31'
+     ) AS shipping
+GROUP BY
+  supp_nation,
+  cust_nation,
+  l_year
+ORDER BY
+  supp_nation,
+  cust_nation,
+  l_year
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q08.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q08.sql
new file mode 100644
index 00000000000000..2657711d79fe1c
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q08.sql
@@ -0,0 +1,38 @@
+-- tables: part,supplier,lineitem,orders,customer,nation,region
+SELECT
+  o_year,
+  sum(CASE
+      WHEN nation = 'BRAZIL'
+        THEN volume
+      ELSE 0
+      END) / sum(volume) AS mkt_share
+FROM (
+       SELECT
+         extract(YEAR FROM o_orderdate)     AS o_year,
+         l_extendedprice * (1 - l_discount) AS volume,
+         n2.n_name                          AS nation
+       FROM
+         part,
+         supplier,
+         lineitem,
+         orders,
+         customer,
+         nation n1,
+         nation n2,
+         region
+       WHERE
+         p_partkey = l_partkey
+         AND s_suppkey = l_suppkey
+         AND l_orderkey = o_orderkey
+         AND o_custkey = c_custkey
+         AND c_nationkey = n1.n_nationkey
+         AND n1.n_regionkey = r_regionkey
+         AND r_name = 'AMERICA'
+         AND s_nationkey = n2.n_nationkey
+         AND o_orderdate BETWEEN DATE '1995-01-01' AND DATE '1996-12-31'
+         AND p_type = 'ECONOMY ANODIZED STEEL'
+     ) AS all_nations
+GROUP BY
+  o_year
+ORDER BY
+  o_year
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q09.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q09.sql
new file mode 100644
index 00000000000000..9236941881f39b
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q09.sql
@@ -0,0 +1,32 @@
+-- tables: part,supplier,lineitem,partsupp,orders,nation
+SELECT
+  nation,
+  o_year,
+  sum(amount) AS sum_profit
+FROM (
+       SELECT
+         n_name                                                          AS nation,
+         extract(YEAR FROM o_orderdate)                                  AS o_year,
+         l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity AS amount
+       FROM
+         part,
+         supplier,
+         lineitem,
+         partsupp,
+         orders,
+         nation
+       WHERE
+         s_suppkey = l_suppkey
+         AND ps_suppkey = l_suppkey
+         AND ps_partkey = l_partkey
+         AND p_partkey = l_partkey
+         AND o_orderkey = l_orderkey
+         AND s_nationkey = n_nationkey
+         AND p_name LIKE '%green%'
+     ) AS profit
+GROUP BY
+  nation,
+  o_year
+ORDER BY
+  nation,
+  o_year DESC
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q10.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q10.sql
new file mode 100644
index 00000000000000..965bc17b3c0de7
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q10.sql
@@ -0,0 +1,33 @@
+-- tables: customer,orders,lineitem,nation
+SELECT
+  c_custkey,
+  c_name,
+  sum(l_extendedprice * (1 - l_discount)) AS revenue,
+  c_acctbal,
+  n_name,
+  c_address,
+  c_phone,
+  c_comment
+FROM
+  customer,
+  orders,
+  lineitem,
+  nation
+WHERE
+  c_custkey = o_custkey
+  AND l_orderkey = o_orderkey
+  AND o_orderdate >= DATE '1993-10-01'
+  AND o_orderdate < DATE '1993-10-01' + INTERVAL '3' MONTH
+  AND l_returnflag = 'R'
+  AND c_nationkey = n_nationkey
+GROUP BY
+  c_custkey,
+  c_name,
+  c_acctbal,
+  c_phone,
+  n_name,
+  c_address,
+  c_comment
+ORDER BY
+  revenue DESC
+LIMIT 20
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q11.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q11.sql
new file mode 100644
index 00000000000000..18856461ea2e2d
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q11.sql
@@ -0,0 +1,28 @@
+-- tables: partsupp,supplier,nation
+SELECT
+  ps_partkey,
+  sum(ps_supplycost * ps_availqty) AS value
+FROM
+  partsupp,
+  supplier,
+  nation
+WHERE
+  ps_suppkey = s_suppkey
+  AND s_nationkey = n_nationkey
+  AND n_name = 'GERMANY'
+GROUP BY
+  ps_partkey
+HAVING
+  sum(ps_supplycost * ps_availqty) > (
+    SELECT sum(ps_supplycost * ps_availqty) * 0.0001
+    FROM
+      partsupp,
+      supplier,
+      nation
+    WHERE
+      ps_suppkey = s_suppkey
+      AND s_nationkey = n_nationkey
+      AND n_name = 'GERMANY'
+  )
+ORDER BY
+  value DESC
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q12.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q12.sql
new file mode 100644
index 00000000000000..3312d53dc4ad1c
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q12.sql
@@ -0,0 +1,29 @@
+-- tables: orders,lineitem
+SELECT
+  l_shipmode,
+  sum(CASE
+      WHEN o_orderpriority = '1-URGENT'
+           OR o_orderpriority = '2-HIGH'
+        THEN 1
+      ELSE 0
+      END) AS high_line_count,
+  sum(CASE
+      WHEN o_orderpriority <> '1-URGENT'
+           AND o_orderpriority <> '2-HIGH'
+        THEN 1
+      ELSE 0
+      END) AS low_line_count
+FROM
+  orders,
+  lineitem
+WHERE
+  o_orderkey = l_orderkey
+  AND l_shipmode IN ('MAIL', 'SHIP')
+  AND l_commitdate < l_receiptdate
+  AND l_shipdate < l_commitdate
+  AND l_receiptdate >= DATE '1994-01-01'
+  AND l_receiptdate < DATE '1994-01-01' + INTERVAL '1' YEAR
+GROUP BY
+  l_shipmode
+ORDER BY
+  l_shipmode
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q13.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q13.sql
new file mode 100644
index 00000000000000..f7d69a955f94d1
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q13.sql
@@ -0,0 +1,21 @@
+-- tables: customer
+SELECT
+  c_count,
+  count(*) AS custdist
+FROM (
+       SELECT
+         c_custkey,
+         count(o_orderkey) AS c_count
+       FROM
+         customer
+         LEFT OUTER JOIN orders ON
+                                  c_custkey = o_custkey
+                                  AND o_comment NOT LIKE '%special%requests%'
+       GROUP BY
+         c_custkey
+     ) AS c_orders
+GROUP BY
+  c_count
+ORDER BY
+  custdist DESC,
+  c_count DESC
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q14.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q14.sql
new file mode 100644
index 00000000000000..8fb3730bc23138
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q14.sql
@@ -0,0 +1,13 @@
+-- tables: lineitem,part
+SELECT 100.00 * sum(CASE
+                    WHEN p_type LIKE 'PROMO%'
+                      THEN l_extendedprice * (1 - l_discount)
+                    ELSE 0
+                    END) / sum(l_extendedprice * (1 - l_discount)) AS promo_revenue
+FROM
+  lineitem,
+  part
+WHERE
+  l_partkey = p_partkey
+  AND l_shipdate >= DATE '1995-09-01'
+  AND l_shipdate < DATE '1995-09-01' + INTERVAL '1' MONTH
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q15.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q15.sql
new file mode 100644
index 00000000000000..80cd87be5d2e2e
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q15.sql
@@ -0,0 +1,20 @@
+SELECT
+  s_suppkey,
+  s_name,
+  s_address,
+  s_phone,
+  total_revenue
+FROM
+  supplier,
+  revenue1
+WHERE
+  s_suppkey = supplier_no
+  AND total_revenue = (
+    SELECT max(total_revenue)
+    FROM
+      revenue1
+  )
+ORDER BY
+  s_suppkey;
+
+
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q16.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q16.sql
new file mode 100644
index 00000000000000..277777cb13acdc
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q16.sql
@@ -0,0 +1,30 @@
+-- tables: partsupp,part,supplier
+SELECT
+  p_brand,
+  p_type,
+  p_size,
+  count(DISTINCT ps_suppkey) AS supplier_cnt
+FROM
+  partsupp,
+  part
+WHERE
+  p_partkey = ps_partkey
+  AND p_brand <> 'Brand#45'
+  AND p_type NOT LIKE 'MEDIUM POLISHED%'
+  AND p_size IN (49, 14, 23, 45, 19, 3, 36, 9)
+  AND ps_suppkey NOT IN (
+    SELECT s_suppkey
+    FROM
+      supplier
+    WHERE
+      s_comment LIKE '%Customer%Complaints%'
+  )
+GROUP BY
+  p_brand,
+  p_type,
+  p_size
+ORDER BY
+  supplier_cnt DESC,
+  p_brand,
+  p_type,
+  p_size
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q17.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q17.sql
new file mode 100644
index 00000000000000..c37f2ceedba52e
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q17.sql
@@ -0,0 +1,16 @@
+-- tables: lineitem,part
+SELECT sum(l_extendedprice) / 7.0 AS avg_yearly
+FROM
+  lineitem,
+  part
+WHERE
+  p_partkey = l_partkey
+  AND p_brand = 'Brand#23'
+  AND p_container = 'MED BOX'
+  AND l_quantity < (
+    SELECT 0.2 * avg(l_quantity)
+    FROM
+      lineitem
+    WHERE
+      l_partkey = p_partkey
+  )
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q18.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q18.sql
new file mode 100644
index 00000000000000..98db1d60466178
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q18.sql
@@ -0,0 +1,34 @@
+-- tables: customer,orders,lineitem
+SELECT
+  c_name,
+  c_custkey,
+  o_orderkey,
+  o_orderdate,
+  o_totalprice,
+  sum(l_quantity)
+FROM
+  customer,
+  orders,
+  lineitem
+WHERE
+  o_orderkey IN (
+    SELECT l_orderkey
+    FROM
+      lineitem
+    GROUP BY
+      l_orderkey
+    HAVING
+      sum(l_quantity) > 300
+  )
+  AND c_custkey = o_custkey
+  AND o_orderkey = l_orderkey
+GROUP BY
+  c_name,
+  c_custkey,
+  o_orderkey,
+  o_orderdate,
+  o_totalprice
+ORDER BY
+  o_totalprice DESC,
+  o_orderdate
+LIMIT 100
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q19.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q19.sql
new file mode 100644
index 00000000000000..c4e107cd68bc01
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q19.sql
@@ -0,0 +1,35 @@
+-- tables: lineitem,part
+SELECT sum(l_extendedprice * (1 - l_discount)) AS revenue
+FROM
+  lineitem,
+  part
+WHERE
+  (
+    p_partkey = l_partkey
+    AND p_brand = 'Brand#12'
+    AND p_container IN ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG')
+    AND l_quantity >= 1 AND l_quantity <= 1 + 10
+    AND p_size BETWEEN 1 AND 5
+    AND l_shipmode IN ('AIR', 'AIR REG')
+    AND l_shipinstruct = 'DELIVER IN PERSON'
+  )
+  OR
+  (
+    p_partkey = l_partkey
+    AND p_brand = 'Brand#23'
+    AND p_container IN ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK')
+    AND l_quantity >= 10 AND l_quantity <= 10 + 10
+    AND p_size BETWEEN 1 AND 10
+    AND l_shipmode IN ('AIR', 'AIR REG')
+    AND l_shipinstruct = 'DELIVER IN PERSON'
+  )
+  OR
+  (
+    p_partkey = l_partkey
+    AND p_brand = 'Brand#34'
+    AND p_container IN ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG')
+    AND l_quantity >= 20 AND l_quantity <= 20 + 10
+    AND p_size BETWEEN 1 AND 15
+    AND l_shipmode IN ('AIR', 'AIR REG')
+    AND l_shipinstruct = 'DELIVER IN PERSON'
+  )
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q20.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q20.sql
new file mode 100644
index 00000000000000..6d4ef5bcfc9155
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q20.sql
@@ -0,0 +1,33 @@
+-- tables: supplier,nation,partsupp,lineitem,part
+SELECT
+  s_name,
+  s_address
+FROM
+  supplier, nation
+WHERE
+  s_suppkey IN (
+    SELECT ps_suppkey
+    FROM
+      partsupp
+    WHERE
+      ps_partkey IN (
+        SELECT p_partkey
+        FROM
+          part
+        WHERE
+          p_name LIKE 'forest%'
+      )
+      AND ps_availqty > (
+        SELECT 0.5 * sum(l_quantity)
+        FROM
+          lineitem
+        WHERE
+          l_partkey = ps_partkey
+          AND l_suppkey = ps_suppkey
+          AND l_shipdate >= date('1994-01-01')
+          AND l_shipdate < date('1994-01-01') + interval '1' YEAR
+)
+)
+AND s_nationkey = n_nationkey
+AND n_name = 'CANADA'
+ORDER BY s_name
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q21.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q21.sql
new file mode 100644
index 00000000000000..961f2749ddfe82
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q21.sql
@@ -0,0 +1,39 @@
+-- tables: supplier,lineitem,orders,nation
+SELECT
+  s_name,
+  count(*) AS numwait
+FROM
+  supplier,
+  lineitem l1,
+  orders,
+  nation
+WHERE
+  s_suppkey = l1.l_suppkey
+  AND o_orderkey = l1.l_orderkey
+  AND o_orderstatus = 'F'
+  AND l1.l_receiptdate > l1.l_commitdate
+  AND exists(
+    SELECT *
+    FROM
+      lineitem l2
+    WHERE
+      l2.l_orderkey = l1.l_orderkey
+      AND l2.l_suppkey <> l1.l_suppkey
+  )
+  AND NOT exists(
+    SELECT *
+    FROM
+      lineitem l3
+    WHERE
+      l3.l_orderkey = l1.l_orderkey
+      AND l3.l_suppkey <> l1.l_suppkey
+      AND l3.l_receiptdate > l3.l_commitdate
+  )
+  AND s_nationkey = n_nationkey
+  AND n_name = 'SAUDI ARABIA'
+GROUP BY
+  s_name
+ORDER BY
+  numwait DESC,
+  s_name
+LIMIT 100
diff --git a/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q22.sql b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q22.sql
new file mode 100644
index 00000000000000..01da1804587b47
--- /dev/null
+++ b/regression-test/suites/tpch_unique_sql_zstd_bucket1_p0/sql/q22.sql
@@ -0,0 +1,35 @@
+-- tables: orders,customer
+SELECT
+  cntrycode,
+  count(*)       AS numcust,
+  sum(c_acctbal) AS totacctbal
+FROM (
+       SELECT
+         substr(c_phone, 1, 2) AS cntrycode,
+         c_acctbal
+       FROM
+         customer
+       WHERE
+         substr(c_phone, 1, 2) IN
+         ('13', '31', '23', '29', '30', '18', '17')
+         AND c_acctbal > (
+           SELECT avg(c_acctbal)
+           FROM
+             customer
+           WHERE
+             c_acctbal > 0.00
+             AND substr(c_phone, 1, 2) IN
+                 ('13', '31', '23', '29', '30', '18', '17')
+         )
+         AND NOT exists(
+           SELECT *
+           FROM
+             orders
+           WHERE
+             o_custkey = c_custkey
+         )
+     ) AS custsale
+GROUP BY
+  cntrycode
+ORDER BY
+  cntrycode

From c98489fc09a6b428a590303e05b7d8bec7c1bc0d Mon Sep 17 00:00:00 2001
From: Pxl <pxl290@qq.com>
Date: Thu, 28 Dec 2023 14:56:33 +0800
Subject: [PATCH 111/299] [Feature](materialized-view) support visitBitmapUnion
 mv rewrite (#29200)

* support visitBitmapUnion rewrite

* add case
---
 .../SelectMaterializedIndexWithAggregate.java | 101 +++++++++++++-----
 .../testBitmapUnionInQuery.out                |   3 +
 .../testBitmapUnionInQuery.groovy             |   6 ++
 3 files changed, 86 insertions(+), 24 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
index b36afa35c41565..ae627523e36d2c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java
@@ -52,7 +52,6 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
 import org.apache.doris.nereids.trees.expressions.functions.combinator.MergeCombinator;
 import org.apache.doris.nereids.trees.expressions.functions.combinator.StateCombinator;
-import org.apache.doris.nereids.trees.expressions.functions.scalar.BitmapHash;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.HllHash;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmapWithCheck;
@@ -1282,6 +1281,66 @@ public Expression visitCount(Count count, RewriteContext context) {
             return count;
         }
 
+        /**
+         * bitmap_union(to_bitmap(col)) ->
+         * bitmap_union(mva_BITMAP_UNION__to_bitmap_with_check(col))
+         */
+        @Override
+        public Expression visitBitmapUnion(BitmapUnion bitmapUnion, RewriteContext context) {
+            Expression result = visitAggregateFunction(bitmapUnion, context);
+            if (result != bitmapUnion) {
+                return result;
+            }
+            if (bitmapUnion.child() instanceof ToBitmap) {
+                ToBitmap toBitmap = (ToBitmap) bitmapUnion.child();
+                Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(toBitmap.child());
+                if (slotOpt.isPresent()) {
+                    String bitmapUnionColumn = normalizeName(CreateMaterializedViewStmt
+                            .mvColumnBuilder(AggregateType.BITMAP_UNION, CreateMaterializedViewStmt
+                                    .mvColumnBuilder(new ToBitmapWithCheck(toBitmap.child()).toSql())));
+
+                    Column mvColumn = context.checkContext.getColumn(bitmapUnionColumn);
+                    // has bitmap_union column
+                    if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {
+
+                        Slot bitmapUnionSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index)
+                                .stream().filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
+                                .findFirst().orElseThrow(
+                                        () -> new AnalysisException("cannot find bitmap union slot when select mv"));
+
+                        context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionSlot);
+                        context.exprRewriteMap.projectExprMap.put(toBitmap, bitmapUnionSlot);
+                        BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot);
+                        context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion);
+                        return newBitmapUnion;
+                    }
+                }
+            } else {
+                Expression child = bitmapUnion.child();
+                String bitmapUnionColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(
+                        AggregateType.BITMAP_UNION, CreateMaterializedViewStmt.mvColumnBuilder(child.toSql())));
+
+                Column mvColumn = context.checkContext.getColumn(bitmapUnionColumn);
+                // has bitmap_union column
+                if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {
+
+                    Slot bitmapUnionSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index)
+                            .stream().filter(s -> bitmapUnionColumn.equalsIgnoreCase(normalizeName(s.getName())))
+                            .findFirst()
+                            .orElseThrow(() -> new AnalysisException("cannot find bitmap union slot when select mv"));
+                    if (child instanceof Slot) {
+                        context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionSlot);
+                    }
+                    context.exprRewriteMap.projectExprMap.put(child, bitmapUnionSlot);
+                    BitmapUnion newBitmapUnion = new BitmapUnion(bitmapUnionSlot);
+                    context.exprRewriteMap.aggFuncMap.put(bitmapUnion, newBitmapUnion);
+                    return newBitmapUnion;
+                }
+            }
+
+            return bitmapUnion;
+        }
+
         /**
          * bitmap_union_count(to_bitmap(col)) -> bitmap_union_count(mva_BITMAP_UNION__to_bitmap_with_check(col))
          */
@@ -1318,32 +1377,26 @@ public Expression visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, Rewri
                         return newBitmapUnionCount;
                     }
                 }
-            } else if (bitmapUnionCount.child() instanceof BitmapHash) {
-                BitmapHash bitmapHash = (BitmapHash) bitmapUnionCount.child();
-                Optional<Slot> slotOpt = ExpressionUtils.extractSlotOrCastOnSlot(bitmapHash.child());
-                if (slotOpt.isPresent()) {
-                    String bitmapUnionCountColumn = normalizeName(
-                            CreateMaterializedViewStmt.mvColumnBuilder(AggregateType.BITMAP_UNION,
-                                    CreateMaterializedViewStmt.mvColumnBuilder(bitmapHash.toSql())));
-
-                    Column mvColumn = context.checkContext.getColumn(bitmapUnionCountColumn);
-                    // has bitmap_union_count column
-                    if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {
+            } else {
+                Expression child = bitmapUnionCount.child();
+                String bitmapUnionCountColumn = normalizeName(CreateMaterializedViewStmt.mvColumnBuilder(
+                        AggregateType.BITMAP_UNION, CreateMaterializedViewStmt.mvColumnBuilder(child.toSql())));
 
-                        Slot bitmapUnionCountSlot = context.checkContext.scan
-                                .getOutputByIndex(context.checkContext.index)
-                                .stream()
-                                .filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName())))
-                                .findFirst()
-                                .orElseThrow(() -> new AnalysisException(
-                                        "cannot find bitmap union count slot when select mv"));
+                Column mvColumn = context.checkContext.getColumn(bitmapUnionCountColumn);
+                // has bitmap_union_count column
+                if (mvColumn != null && context.checkContext.valueNameToColumn.containsValue(mvColumn)) {
 
-                        context.exprRewriteMap.slotMap.put(slotOpt.get(), bitmapUnionCountSlot);
-                        context.exprRewriteMap.projectExprMap.put(bitmapHash, bitmapUnionCountSlot);
-                        BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
-                        context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount);
-                        return newBitmapUnionCount;
+                    Slot bitmapUnionCountSlot = context.checkContext.scan.getOutputByIndex(context.checkContext.index)
+                            .stream().filter(s -> bitmapUnionCountColumn.equalsIgnoreCase(normalizeName(s.getName())))
+                            .findFirst().orElseThrow(
+                                    () -> new AnalysisException("cannot find bitmap union count slot when select mv"));
+                    if (child instanceof Slot) {
+                        context.exprRewriteMap.slotMap.put((Slot) child, bitmapUnionCountSlot);
                     }
+                    context.exprRewriteMap.projectExprMap.put(child, bitmapUnionCountSlot);
+                    BitmapUnionCount newBitmapUnionCount = new BitmapUnionCount(bitmapUnionCountSlot);
+                    context.exprRewriteMap.aggFuncMap.put(bitmapUnionCount, newBitmapUnionCount);
+                    return newBitmapUnionCount;
                 }
             }
 
diff --git a/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out b/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out
index 88913c5b65d879..75a47cb33192a5 100644
--- a/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out
+++ b/regression-test/data/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.out
@@ -7,3 +7,6 @@
 -- !select_mv --
 1	2
 
+-- !select_mv --
+1	2
+
diff --git a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy
index 798d350b92cd7b..d4502dd03b19b2 100644
--- a/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy
+++ b/regression-test/suites/mv_p0/ut/testBitmapUnionInQuery/testBitmapUnionInQuery.groovy
@@ -47,4 +47,10 @@ suite ("testBitmapUnionInQuery") {
         contains "(user_tags_mv)"
     }
     qt_select_mv "select user_id, bitmap_union_count(to_bitmap(tag_id)) a from user_tags group by user_id having a>1 order by a;"
+
+    explain {
+        sql("select user_id, bitmap_count(bitmap_union(to_bitmap(tag_id))) a from user_tags group by user_id having a>1 order by a;")
+        contains "(user_tags_mv)"
+    }
+    qt_select_mv "select user_id, bitmap_count(bitmap_union(to_bitmap(tag_id))) a from user_tags group by user_id having a>1 order by a;"
 }

From 118775f913c2fd8e6ac1069ad1486c16c7909a0b Mon Sep 17 00:00:00 2001
From: Pxl <pxl290@qq.com>
Date: Thu, 28 Dec 2023 14:57:31 +0800
Subject: [PATCH 112/299] [Bug](schame-change) fix wrong result after reorder
 mor table (#29045)

* fix wrong result after reorder mor table

* update
---
 be/src/olap/column_mapping.h                  |  2 +
 be/src/olap/schema_change.cpp                 | 35 +++++--------
 .../reorder_mor/reorder_mor.out               |  7 +++
 .../reorder_mor/reorder_mor.groovy            | 52 +++++++++++++++++++
 4 files changed, 74 insertions(+), 22 deletions(-)
 create mode 100644 regression-test/data/schema_change_p0/reorder_mor/reorder_mor.out
 create mode 100644 regression-test/suites/schema_change_p0/reorder_mor/reorder_mor.groovy

diff --git a/be/src/olap/column_mapping.h b/be/src/olap/column_mapping.h
index 41564bda3cc56f..047af1e9d1190b 100644
--- a/be/src/olap/column_mapping.h
+++ b/be/src/olap/column_mapping.h
@@ -30,6 +30,8 @@ struct ColumnMapping {
     ColumnMapping() = default;
     virtual ~ColumnMapping() = default;
 
+    bool has_reference() const { return expr != nullptr || ref_column >= 0; }
+
     // <0: use default value
     // >=0: use origin column
     int32_t ref_column = -1;
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index fd4bf312c90fcd..7f18826b406ef9 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -1181,11 +1181,6 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
     const std::unordered_map<std::string, AlterMaterializedViewParam>& materialized_function_map =
             sc_params.materialized_params_map;
     DescriptorTbl desc_tbl = *sc_params.desc_tbl;
-
-    if (sc_params.alter_tablet_type == ROLLUP) {
-        *sc_directly = true;
-    }
-
     TabletSchemaSPtr new_tablet_schema = sc_params.new_tablet_schema;
 
     // set column mapping
@@ -1246,15 +1241,13 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
         changer->set_where_expr(materialized_function_map.find(WHERE_SIGN_LOWER)->second.expr);
     }
 
-    // Check if re-aggregation is needed.
-    *sc_sorting = false;
     // If the reference sequence of the Key column is out of order, it needs to be reordered
     int num_default_value = 0;
 
     for (int i = 0, new_schema_size = new_tablet->num_key_columns(); i < new_schema_size; ++i) {
         ColumnMapping* column_mapping = changer->get_mutable_column_mapping(i);
 
-        if (column_mapping->expr == nullptr) {
+        if (!column_mapping->has_reference()) {
             num_default_value++;
             continue;
         }
@@ -1288,6 +1281,11 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
         return Status::OK();
     }
 
+    if (sc_params.alter_tablet_type == ROLLUP) {
+        *sc_directly = true;
+        return Status::OK();
+    }
+
     if (new_tablet->enable_unique_key_merge_on_write() &&
         new_tablet->num_key_columns() > base_tablet_schema->num_key_columns()) {
         *sc_directly = true;
@@ -1300,25 +1298,18 @@ Status SchemaChangeHandler::_parse_request(const SchemaChangeParams& sc_params,
         return Status::OK();
     }
 
-    for (size_t i = 0; i < new_tablet->num_columns(); ++i) {
-        ColumnMapping* column_mapping = changer->get_mutable_column_mapping(i);
-        if (column_mapping->expr == nullptr) {
-            continue;
-        } else {
-            *sc_directly = true;
-            return Status::OK();
-        }
-    }
-
     if (!sc_params.delete_handler->empty()) {
         // there exists delete condition in header, can't do linked schema change
         *sc_directly = true;
+        return Status::OK();
     }
 
-    if (base_tablet->tablet_meta()->preferred_rowset_type() !=
-        new_tablet->tablet_meta()->preferred_rowset_type()) {
-        // If the base_tablet and new_tablet rowset types are different, just use directly type
-        *sc_directly = true;
+    for (size_t i = 0; i < new_tablet->num_columns(); ++i) {
+        ColumnMapping* column_mapping = changer->get_mutable_column_mapping(i);
+        if (column_mapping->expr != nullptr) {
+            *sc_directly = true;
+            return Status::OK();
+        }
     }
 
     // if rs_reader has remote files, link schema change is not supported,
diff --git a/regression-test/data/schema_change_p0/reorder_mor/reorder_mor.out b/regression-test/data/schema_change_p0/reorder_mor/reorder_mor.out
new file mode 100644
index 00000000000000..32ec9950499ea7
--- /dev/null
+++ b/regression-test/data/schema_change_p0/reorder_mor/reorder_mor.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_star --
+1	3	5
+2	4	3
+8	1	3
+9	2	3
+
diff --git a/regression-test/suites/schema_change_p0/reorder_mor/reorder_mor.groovy b/regression-test/suites/schema_change_p0/reorder_mor/reorder_mor.groovy
new file mode 100644
index 00000000000000..76d9d650f05bda
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/reorder_mor/reorder_mor.groovy
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("reorder_mor") {
+
+    sql """ DROP TABLE IF EXISTS test; """
+
+    sql """
+            create table test (c0 int, c1 int, c2 int) ENGINE=OLAP UNIQUE KEY(`c0`, c1) COMMENT 'OLAP' DISTRIBUTED BY HASH(`c0`) BUCKETS 1 PROPERTIES ("replication_allocation" = "tag.location.default: 1", "enable_unique_key_merge_on_write"="false");
+        """
+
+    sql "insert into test values(1,8,3),(2,9,3),(3,1,3),(4,2,3);"
+
+    sql "alter table test order by(c1,c0,c2);"
+
+    def getJobState = { tableName ->
+         def jobStateResult = sql """  SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+         return jobStateResult[0][9]
+    }
+
+    int max_try_time = 100
+    while (max_try_time--){
+        String result = getJobState("test")
+        if (result == "FINISHED") {
+            sleep(1000)
+            break
+        } else {
+            sleep(1000)
+            assertTrue(max_try_time>1)
+        }
+    }
+
+    sql "insert into test values(1, 3, 5);"
+
+    qt_select_star "select * from test order by 1,2,3;"
+}

From 31b3be456c53c8c0ec7bb64ecca42866b71e86e0 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@apache.org>
Date: Thu, 28 Dec 2023 15:04:22 +0800
Subject: [PATCH 113/299] add workload scheduler in be (#29116)

---
 be/src/agent/agent_server.cpp                 |   8 ++
 .../agent/workload_sched_policy_listener.cpp  |  78 +++++++++++
 be/src/agent/workload_sched_policy_listener.h |  37 +++++
 be/src/runtime/exec_env.h                     |   4 +
 be/src/runtime/exec_env_init.cpp              |   8 ++
 be/src/runtime/fragment_mgr.cpp               |  20 +++
 be/src/runtime/fragment_mgr.h                 |   3 +
 be/src/runtime/query_context.h                |   2 +
 .../workload_management/workload_action.cpp   |  35 +++++
 .../workload_management/workload_action.h     |  68 +++++++++
 .../workload_management/workload_comparator.h |  77 ++++++++++
 .../workload_condition.cpp                    |  59 ++++++++
 .../workload_management/workload_condition.h  |  97 +++++++++++++
 .../workload_management/workload_query_info.h |  33 +++++
 .../workload_sched_policy.cpp                 |  73 ++++++++++
 .../workload_sched_policy.h                   |  59 ++++++++
 .../workload_sched_policy_mgr.cpp             | 131 ++++++++++++++++++
 .../workload_sched_policy_mgr.h               |  50 +++++++
 .../java/org/apache/doris/catalog/Env.java    |   3 +
 .../common/publish/TopicPublisherThread.java  |   7 +-
 .../publish/WorkloadGroupPublisher.java       |   9 +-
 .../WorkloadSchedPolicy.java                  |  67 +++++++++
 .../WorkloadSchedPolicyMgr.java               |  17 +++
 .../WorkloadSchedPolicyPublisher.java         |  44 ++++++
 gensrc/thrift/BackendService.thrift           |  44 +++++-
 25 files changed, 1027 insertions(+), 6 deletions(-)
 create mode 100644 be/src/agent/workload_sched_policy_listener.cpp
 create mode 100644 be/src/agent/workload_sched_policy_listener.h
 create mode 100644 be/src/runtime/workload_management/workload_action.cpp
 create mode 100644 be/src/runtime/workload_management/workload_action.h
 create mode 100644 be/src/runtime/workload_management/workload_comparator.h
 create mode 100644 be/src/runtime/workload_management/workload_condition.cpp
 create mode 100644 be/src/runtime/workload_management/workload_condition.h
 create mode 100644 be/src/runtime/workload_management/workload_query_info.h
 create mode 100644 be/src/runtime/workload_management/workload_sched_policy.cpp
 create mode 100644 be/src/runtime/workload_management/workload_sched_policy.h
 create mode 100644 be/src/runtime/workload_management/workload_sched_policy_mgr.cpp
 create mode 100644 be/src/runtime/workload_management/workload_sched_policy_mgr.h
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java

diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp
index 9902a0ad726e47..a3b18c5356796e 100644
--- a/be/src/agent/agent_server.cpp
+++ b/be/src/agent/agent_server.cpp
@@ -31,6 +31,7 @@
 #include "agent/topic_subscriber.h"
 #include "agent/utils.h"
 #include "agent/workload_group_listener.h"
+#include "agent/workload_sched_policy_listener.h"
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
@@ -72,6 +73,13 @@ AgentServer::AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info)
     LOG(INFO) << "Register workload group listener";
     _topic_subscriber->register_listener(doris::TTopicInfoType::type::WORKLOAD_GROUP,
                                          std::move(wg_listener));
+
+    std::unique_ptr<TopicListener> policy_listener =
+            std::make_unique<WorkloadschedPolicyListener>(exec_env);
+    LOG(INFO) << "Register workload scheduler policy listener";
+    _topic_subscriber->register_listener(doris::TTopicInfoType::type::WORKLOAD_SCHED_POLICY,
+                                         std::move(policy_listener));
+
 #endif
 }
 
diff --git a/be/src/agent/workload_sched_policy_listener.cpp b/be/src/agent/workload_sched_policy_listener.cpp
new file mode 100644
index 00000000000000..461fd2cbb0f6d0
--- /dev/null
+++ b/be/src/agent/workload_sched_policy_listener.cpp
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "agent/workload_sched_policy_listener.h"
+
+#include "runtime/workload_management/workload_action.h"
+#include "runtime/workload_management/workload_condition.h"
+#include "runtime/workload_management/workload_sched_policy.h"
+#include "runtime/workload_management/workload_sched_policy_mgr.h"
+
+namespace doris {
+
+void WorkloadschedPolicyListener::handle_topic_info(const std::vector<TopicInfo>& topic_info_list) {
+    std::map<uint64_t, std::shared_ptr<WorkloadSchedPolicy>> policy_map;
+    for (const TopicInfo& topic_info : topic_info_list) {
+        if (!topic_info.__isset.workload_sched_policy) {
+            continue;
+        }
+
+        TWorkloadSchedPolicy tpolicy = topic_info.workload_sched_policy;
+        // some metric or action can not exec in be, then need skip
+        bool need_skip_current_policy = false;
+
+        std::vector<std::unique_ptr<WorkloadCondition>> cond_ptr_list;
+        for (TWorkloadCondition& cond : tpolicy.condition_list) {
+            std::unique_ptr<WorkloadCondition> cond_ptr =
+                    WorkloadConditionFactory::create_workload_condition(&cond);
+            if (cond_ptr == nullptr) {
+                need_skip_current_policy = true;
+                break;
+            }
+            cond_ptr_list.push_back(std::move(cond_ptr));
+        }
+        if (need_skip_current_policy) {
+            continue;
+        }
+
+        std::vector<std::unique_ptr<WorkloadAction>> action_ptr_list;
+        for (TWorkloadAction& action : tpolicy.action_list) {
+            std::unique_ptr<WorkloadAction> action_ptr =
+                    WorkloadActionFactory::create_workload_action(&action);
+            if (action_ptr == nullptr) {
+                need_skip_current_policy = true;
+                break;
+            }
+            action_ptr_list.push_back(std::move(action_ptr));
+        }
+        if (need_skip_current_policy) {
+            continue;
+        }
+
+        std::shared_ptr<WorkloadSchedPolicy> policy_ptr = std::make_shared<WorkloadSchedPolicy>();
+        policy_ptr->init(tpolicy.id, tpolicy.name, tpolicy.version, tpolicy.enabled,
+                         tpolicy.priority, std::move(cond_ptr_list), std::move(action_ptr_list));
+        policy_map.emplace(tpolicy.id, std::move(policy_ptr));
+    }
+    size_t new_policy_size = policy_map.size();
+    if (new_policy_size > 0) {
+        _exec_env->workload_sched_policy_mgr()->update_workload_sched_policy(std::move(policy_map));
+    }
+    LOG(INFO) << "[workload_schedule]finish update workload schedule policy, size="
+              << new_policy_size;
+}
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/agent/workload_sched_policy_listener.h b/be/src/agent/workload_sched_policy_listener.h
new file mode 100644
index 00000000000000..f1410b502fdd99
--- /dev/null
+++ b/be/src/agent/workload_sched_policy_listener.h
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <glog/logging.h>
+
+#include "agent/topic_listener.h"
+#include "runtime/exec_env.h"
+
+namespace doris {
+
+class WorkloadschedPolicyListener : public TopicListener {
+public:
+    WorkloadschedPolicyListener(ExecEnv* exec_env) : _exec_env(exec_env) {}
+
+    void handle_topic_info(const std::vector<TopicInfo>& topic_info_list) override;
+
+private:
+    ExecEnv* _exec_env = nullptr;
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 4c1060891f5ffb..f8a52ad1ec402b 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -63,6 +63,7 @@ class InvertedIndexSearcherCache;
 class InvertedIndexQueryCache;
 } // namespace segment_v2
 
+class WorkloadSchedPolicyMgr;
 class BfdParser;
 class BrokerMgr;
 template <class T>
@@ -153,6 +154,7 @@ class ExecEnv {
     pipeline::TaskScheduler* pipeline_task_scheduler() { return _without_group_task_scheduler; }
     pipeline::TaskScheduler* pipeline_task_group_scheduler() { return _with_group_task_scheduler; }
     taskgroup::TaskGroupManager* task_group_manager() { return _task_group_manager; }
+    WorkloadSchedPolicyMgr* workload_sched_policy_mgr() { return _workload_sched_mgr; }
 
     // using template to simplify client cache management
     template <typename T>
@@ -378,6 +380,8 @@ class ExecEnv {
     std::shared_ptr<doris::pipeline::BlockedTaskScheduler> _with_group_block_scheduler;
 
     doris::pipeline::RuntimeFilterTimerQueue* _runtime_filter_timer_queue = nullptr;
+
+    WorkloadSchedPolicyMgr* _workload_sched_mgr = nullptr;
 };
 
 template <>
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 4df69d67c03acc..48d229b9e21961 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -76,6 +76,7 @@
 #include "runtime/task_group/task_group_manager.h"
 #include "runtime/thread_context.h"
 #include "runtime/user_function_cache.h"
+#include "runtime/workload_management/workload_sched_policy_mgr.h"
 #include "service/backend_options.h"
 #include "service/backend_service.h"
 #include "service/point_query_executor.h"
@@ -194,6 +195,10 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
                               .set_max_threads(1)
                               .set_max_queue_size(1000000)
                               .build(&_lazy_release_obj_pool));
+
+    _workload_sched_mgr = new WorkloadSchedPolicyMgr();
+    _workload_sched_mgr->start(this);
+
     init_file_cache_factory();
     RETURN_IF_ERROR(init_pipeline_task_scheduler());
     _task_group_manager = new taskgroup::TaskGroupManager();
@@ -519,6 +524,8 @@ void ExecEnv::destroy() {
     SAFE_STOP(_group_commit_mgr);
     // _routine_load_task_executor should be stopped before _new_load_stream_mgr.
     SAFE_STOP(_routine_load_task_executor);
+    // stop workload scheduler
+    SAFE_STOP(_workload_sched_mgr);
     // stop pipline step 1, non-cgroup execution
     SAFE_SHUTDOWN(_without_group_block_scheduler.get());
     SAFE_STOP(_without_group_task_scheduler);
@@ -593,6 +600,7 @@ void ExecEnv::destroy() {
     SAFE_DELETE(_bfd_parser);
     SAFE_DELETE(_result_cache);
     SAFE_DELETE(_fragment_mgr);
+    SAFE_DELETE(_workload_sched_mgr);
     SAFE_DELETE(_task_group_manager);
     SAFE_DELETE(_with_group_task_scheduler);
     SAFE_DELETE(_without_group_task_scheduler);
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index 65e1ba475ae1c0..9256c3bccfddef 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -73,6 +73,7 @@
 #include "runtime/task_group/task_group_manager.h"
 #include "runtime/thread_context.h"
 #include "runtime/types.h"
+#include "runtime/workload_management/workload_query_info.h"
 #include "service/backend_options.h"
 #include "util/debug_util.h"
 #include "util/doris_metrics.h"
@@ -1567,4 +1568,23 @@ void FragmentMgr::_setup_shared_hashtable_for_broadcast_join(const TPipelineFrag
     }
 }
 
+void FragmentMgr::get_runtime_query_info(std::vector<WorkloadQueryInfo>* query_info_list) {
+    {
+        std::lock_guard<std::mutex> lock(_lock);
+        // todo: use monotonic time
+        VecDateTimeValue now = VecDateTimeValue::local_time();
+        for (const auto& q : _query_ctx_map) {
+            WorkloadQueryInfo workload_query_info;
+            workload_query_info.query_id = print_id(q.first);
+            workload_query_info.tquery_id = q.first;
+
+            uint64_t query_time_millisecond = q.second->query_time(now) * 1000;
+            workload_query_info.metric_map.emplace(WorkloadMetricType::QUERY_TIME,
+                                                   std::to_string(query_time_millisecond));
+            // todo, add scan rows, scan bytes
+            query_info_list->push_back(workload_query_info);
+        }
+    }
+}
+
 } // namespace doris
diff --git a/be/src/runtime/fragment_mgr.h b/be/src/runtime/fragment_mgr.h
index 21d85503803ce3..f9a6d4f72ed1b4 100644
--- a/be/src/runtime/fragment_mgr.h
+++ b/be/src/runtime/fragment_mgr.h
@@ -64,6 +64,7 @@ class TPipelineInstanceParams;
 class TScanColumnDesc;
 class TScanOpenParams;
 class Thread;
+class WorkloadQueryInfo;
 
 std::string to_load_error_http_path(const std::string& file_name);
 
@@ -153,6 +154,8 @@ class FragmentMgr : public RestMonitorIface {
 
     std::string dump_pipeline_tasks();
 
+    void get_runtime_query_info(std::vector<WorkloadQueryInfo>* _query_info_list);
+
 private:
     void cancel_unlocked_impl(const TUniqueId& id, const PPlanFragmentCancelReason& reason,
                               const std::unique_lock<std::mutex>& state_lock, bool is_pipeline,
diff --git a/be/src/runtime/query_context.h b/be/src/runtime/query_context.h
index a230fd653e8eb5..203c5b6e3f4ca0 100644
--- a/be/src/runtime/query_context.h
+++ b/be/src/runtime/query_context.h
@@ -93,6 +93,8 @@ class QueryContext {
         return false;
     }
 
+    int64_t query_time(VecDateTimeValue& now) { return now.second_diff(_start_time); }
+
     void set_thread_token(int concurrency, bool is_serial) {
         _thread_token = _exec_env->scanner_scheduler()->new_limited_scan_pool_token(
                 is_serial ? ThreadPool::ExecutionMode::SERIAL
diff --git a/be/src/runtime/workload_management/workload_action.cpp b/be/src/runtime/workload_management/workload_action.cpp
new file mode 100644
index 00000000000000..39916bc7cc19db
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_action.cpp
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/workload_management/workload_action.h"
+
+#include "runtime/fragment_mgr.h"
+
+namespace doris {
+
+void WorkloadActionCancelQuery::exec(WorkloadQueryInfo* query_info) {
+    LOG(INFO) << "[workload_schedule]workload scheduler cancel query " << query_info->query_id;
+    ExecEnv::GetInstance()->fragment_mgr()->cancel_query(
+            query_info->tquery_id, PPlanFragmentCancelReason::INTERNAL_ERROR,
+            std::string("query canceled by workload scheduler"));
+}
+
+void WorkloadActionMoveQuery::exec(WorkloadQueryInfo* query_info) {
+    LOG(INFO) << "[workload_schedule]move query action run group=" << _wg_name;
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_action.h b/be/src/runtime/workload_management/workload_action.h
new file mode 100644
index 00000000000000..29c01320b7846f
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_action.h
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "runtime/workload_management/workload_query_info.h"
+
+namespace doris {
+
+enum WorkloadActionType { MOVE_QUERY_TO_GROUP = 0, CANCEL_QUERY = 1 };
+
+class WorkloadAction {
+public:
+    WorkloadAction() = default;
+    virtual ~WorkloadAction() = default;
+
+    virtual void exec(WorkloadQueryInfo* query_info) = 0;
+
+    virtual WorkloadActionType get_action_type() = 0;
+};
+
+class WorkloadActionCancelQuery : public WorkloadAction {
+public:
+    void exec(WorkloadQueryInfo* query_info) override;
+
+    WorkloadActionType get_action_type() override { return CANCEL_QUERY; }
+};
+
+//todo(wb) implement it
+class WorkloadActionMoveQuery : public WorkloadAction {
+public:
+    WorkloadActionMoveQuery(std::string wg_name) : _wg_name(wg_name) {}
+    void exec(WorkloadQueryInfo* query_info) override;
+
+    WorkloadActionType get_action_type() override { return MOVE_QUERY_TO_GROUP; }
+
+private:
+    std::string _wg_name;
+};
+
+class WorkloadActionFactory {
+public:
+    static std::unique_ptr<WorkloadAction> create_workload_action(TWorkloadAction* action) {
+        if (TWorkloadActionType::type::MOVE_QUERY_TO_GROUP == action->action) {
+            return std::make_unique<WorkloadActionMoveQuery>(action->action_args);
+        } else if (TWorkloadActionType::type::CANCEL_QUERY == action->action) {
+            return std::make_unique<WorkloadActionCancelQuery>();
+        }
+        LOG(ERROR) << "not find a action " << action->action;
+        return nullptr;
+    }
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_comparator.h b/be/src/runtime/workload_management/workload_comparator.h
new file mode 100644
index 00000000000000..00981a89dc8fd5
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_comparator.h
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+
+#include "common/status.h"
+
+namespace doris {
+
+enum WorkloadCompareOperator { EQUAL, GREATER, GREATER_EQUAL, LESS, LESS_EQUAL, INVALID_OP };
+
+class WorkloadCompareUtils {
+public:
+    static WorkloadCompareOperator get_workload_compare_operator(TCompareOperator::type t_op) {
+        if (TCompareOperator::type::EQUAL == t_op) {
+            return EQUAL;
+        } else if (TCompareOperator::type::GREATER == t_op) {
+            return GREATER;
+        } else if (TCompareOperator::type::GREATER_EQUAL == t_op) {
+            return GREATER_EQUAL;
+        } else if (TCompareOperator::type::LESS == t_op) {
+            return LESS;
+        } else if (TCompareOperator::type::LESS_EQUAL == t_op) {
+            return LESS_EQUAL;
+        }
+        LOG(ERROR) << "can not find a valid op ";
+        return INVALID_OP;
+    }
+
+    static bool compare_signed_integer(WorkloadCompareOperator op, int64_t first_val,
+                                       int64_t second_val) {
+        switch (op) {
+        case EQUAL:
+            return first_val == second_val;
+        case GREATER:
+            return first_val > second_val;
+        case GREATER_EQUAL:
+            return first_val >= second_val;
+        case LESS:
+            return first_val < second_val;
+        case LESS_EQUAL:
+            return first_val <= second_val;
+        default:
+            LOG(ERROR) << "unexpected signed integer compare operator " << op;
+            return false;
+        }
+    }
+
+    static bool compare_string(WorkloadCompareOperator op, std::string first_val,
+                               std::string second_val) {
+        switch (op) {
+        case EQUAL:
+            return first_val == second_val;
+        default:
+            LOG(ERROR) << "unexpected string compare operator " << op;
+            return false;
+        }
+    }
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_condition.cpp b/be/src/runtime/workload_management/workload_condition.cpp
new file mode 100644
index 00000000000000..dff6f2adc24298
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_condition.cpp
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/workload_management/workload_condition.h"
+
+namespace doris {
+
+// query time
+WorkloadConditionQueryTime::WorkloadConditionQueryTime(WorkloadCompareOperator op,
+                                                       std::string str_val) {
+    _op = op;
+    _query_time = std::stol(str_val);
+}
+
+bool WorkloadConditionQueryTime::eval(std::string str_val) {
+    int64_t query_time_args = std::stol(str_val);
+    return WorkloadCompareUtils::compare_signed_integer(_op, query_time_args, _query_time);
+}
+
+// scan rows
+WorkloadConditionScanRows::WorkloadConditionScanRows(WorkloadCompareOperator op,
+                                                     std::string str_val) {
+    _op = op;
+    _scan_rows = std::stol(str_val);
+}
+
+bool WorkloadConditionScanRows::eval(std::string str_val) {
+    int64_t scan_rows_args = std::stol(str_val);
+    return WorkloadCompareUtils::compare_signed_integer(_op, scan_rows_args, _scan_rows);
+}
+
+// scan bytes
+WorkloadConditionScanBytes::WorkloadConditionScanBytes(WorkloadCompareOperator op,
+                                                       std::string str_val) {
+    _op = op;
+    _scan_bytes = std::stol(str_val);
+}
+
+// todo(wb): need handle invalid input value
+bool WorkloadConditionScanBytes::eval(std::string str_val) {
+    int64_t scan_bytes_args = std::stol(str_val);
+    return WorkloadCompareUtils::compare_signed_integer(_op, scan_bytes_args, _scan_bytes);
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_condition.h b/be/src/runtime/workload_management/workload_condition.h
new file mode 100644
index 00000000000000..3486742c9684c7
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_condition.h
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <gen_cpp/BackendService_types.h>
+
+#include "runtime/workload_management/workload_comparator.h"
+
+namespace doris {
+
+enum WorkloadMetricType { QUERY_TIME, SCAN_ROWS, SCAN_BYTES };
+
+class WorkloadCondition {
+public:
+    WorkloadCondition() = default;
+    virtual ~WorkloadCondition() = default;
+
+    virtual bool eval(std::string str_val) = 0;
+
+    virtual WorkloadMetricType get_workload_metric_type() = 0;
+};
+
+class WorkloadConditionQueryTime : public WorkloadCondition {
+public:
+    WorkloadConditionQueryTime(WorkloadCompareOperator op, std::string str_val);
+
+    bool eval(std::string str_val) override;
+
+    WorkloadMetricType get_workload_metric_type() override {
+        return WorkloadMetricType::QUERY_TIME;
+    }
+
+private:
+    int64_t _query_time;
+    WorkloadCompareOperator _op;
+};
+
+class WorkloadConditionScanRows : public WorkloadCondition {
+public:
+    WorkloadConditionScanRows(WorkloadCompareOperator op, std::string str_val);
+    bool eval(std::string str_val) override;
+    WorkloadMetricType get_workload_metric_type() override { return WorkloadMetricType::SCAN_ROWS; }
+
+private:
+    int64_t _scan_rows;
+    WorkloadCompareOperator _op;
+};
+
+class WorkloadConditionScanBytes : public WorkloadCondition {
+public:
+    WorkloadConditionScanBytes(WorkloadCompareOperator op, std::string str_val);
+    bool eval(std::string str_val) override;
+    WorkloadMetricType get_workload_metric_type() override {
+        return WorkloadMetricType::SCAN_BYTES;
+    }
+
+private:
+    int64_t _scan_bytes;
+    WorkloadCompareOperator _op;
+};
+
+class WorkloadConditionFactory {
+public:
+    static std::unique_ptr<WorkloadCondition> create_workload_condition(
+            TWorkloadCondition* t_cond) {
+        WorkloadCompareOperator op =
+                WorkloadCompareUtils::get_workload_compare_operator(t_cond->op);
+        std::string str_val = t_cond->value;
+        TWorkloadMetricType::type metric_name = t_cond->metric_name;
+        if (TWorkloadMetricType::type::QUERY_TIME == metric_name) {
+            return std::make_unique<WorkloadConditionQueryTime>(op, str_val);
+        } else if (TWorkloadMetricType::type::SCAN_ROWS == metric_name) {
+            return std::make_unique<WorkloadConditionScanRows>(op, str_val);
+        } else if (TWorkloadMetricType::type::SCAN_BYTES == metric_name) {
+            return std::make_unique<WorkloadConditionScanBytes>(op, str_val);
+        }
+        LOG(ERROR) << "not find a metric name " << metric_name;
+        return nullptr;
+    }
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_query_info.h b/be/src/runtime/workload_management/workload_query_info.h
new file mode 100644
index 00000000000000..9c24e9dee8aeb1
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_query_info.h
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <map>
+
+#include "runtime/workload_management/workload_condition.h"
+
+namespace doris {
+
+class WorkloadQueryInfo {
+public:
+    std::map<WorkloadMetricType, std::string> metric_map;
+    TUniqueId tquery_id;
+    std::string query_id;
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_sched_policy.cpp b/be/src/runtime/workload_management/workload_sched_policy.cpp
new file mode 100644
index 00000000000000..bc543a7d7709f5
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_sched_policy.cpp
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/workload_management/workload_sched_policy.h"
+
+namespace doris {
+
+void WorkloadSchedPolicy::init(int64_t id, std::string name, int version, bool enabled,
+                               int priority,
+                               std::vector<std::unique_ptr<WorkloadCondition>> condition_list,
+                               std::vector<std::unique_ptr<WorkloadAction>> action_list) {
+    _id = id;
+    _name = name;
+    _version = version;
+    _enabled = enabled;
+    _priority = priority;
+    _condition_list = std::move(condition_list);
+    _action_list = std::move(action_list);
+
+    _first_action_type = _action_list[0]->get_action_type();
+    if (_first_action_type != WorkloadActionType::MOVE_QUERY_TO_GROUP &&
+        _first_action_type != WorkloadActionType::CANCEL_QUERY) {
+        for (int i = 1; i < _action_list.size(); i++) {
+            WorkloadActionType cur_action_type = _action_list[i]->get_action_type();
+            // one policy can not both contains move and cancel
+            if (cur_action_type == WorkloadActionType::MOVE_QUERY_TO_GROUP ||
+                cur_action_type == WorkloadActionType::CANCEL_QUERY) {
+                _first_action_type = cur_action_type;
+                break;
+            }
+        }
+    }
+}
+
+bool WorkloadSchedPolicy::is_match(WorkloadQueryInfo* query_info_ptr) {
+    if (!_enabled) {
+        return false;
+    }
+    auto& metric_val_map = query_info_ptr->metric_map;
+    for (auto& cond : _condition_list) {
+        if (metric_val_map.find(cond->get_workload_metric_type()) == metric_val_map.end()) {
+            return false;
+        }
+
+        std::string val = metric_val_map.at(cond->get_workload_metric_type());
+        if (!cond->eval(val)) {
+            return false;
+        }
+    }
+    return true;
+}
+
+void WorkloadSchedPolicy::exec_action(WorkloadQueryInfo* query_info) {
+    for (int i = 0; i < _action_list.size(); i++) {
+        _action_list[i]->exec(query_info);
+    }
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_sched_policy.h b/be/src/runtime/workload_management/workload_sched_policy.h
new file mode 100644
index 00000000000000..82f42b9a0b4df8
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_sched_policy.h
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <vector>
+
+#include "runtime/workload_management/workload_action.h"
+#include "runtime/workload_management/workload_condition.h"
+
+namespace doris {
+
+class WorkloadSchedPolicy {
+public:
+    WorkloadSchedPolicy() = default;
+    ~WorkloadSchedPolicy() = default;
+
+    void init(int64_t id, std::string name, int version, bool enabled, int priority,
+              std::vector<std::unique_ptr<WorkloadCondition>> condition_list,
+              std::vector<std::unique_ptr<WorkloadAction>> action_list);
+
+    bool enabled() { return _enabled; }
+    int priority() { return _priority; }
+
+    bool is_match(WorkloadQueryInfo* query_info);
+
+    WorkloadActionType get_first_action_type() { return _first_action_type; }
+
+    void exec_action(WorkloadQueryInfo* query_info);
+
+    int version() { return _version; }
+
+private:
+    int64_t _id;
+    std::string _name;
+    int _version;
+    bool _enabled;
+    int _priority;
+
+    std::vector<std::unique_ptr<WorkloadCondition>> _condition_list;
+    std::vector<std::unique_ptr<WorkloadAction>> _action_list;
+
+    WorkloadActionType _first_action_type;
+};
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp b/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp
new file mode 100644
index 00000000000000..8a30b5395eb3e2
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/workload_management/workload_sched_policy_mgr.h"
+
+#include "runtime/fragment_mgr.h"
+
+namespace doris {
+
+void WorkloadSchedPolicyMgr::start(ExecEnv* exec_env) {
+    _stop_latch.reset(1);
+    _exec_env = exec_env;
+
+    Status st;
+    st = Thread::create(
+            "workload", "workload_scheduler", [this]() { this->_schedule_workload(); }, &_thread);
+    if (!st.ok()) {
+        LOG(WARNING) << "create workload scheduler thread failed";
+    } else {
+        LOG(INFO) << "start workload scheduler ";
+    }
+}
+
+void WorkloadSchedPolicyMgr::stop() {
+    std::lock_guard<std::shared_mutex> write_lock(_stop_lock);
+    if (_stop_latch.count() == 0) {
+        LOG(INFO) << "workload schedule manager is already stopped. ";
+        return;
+    }
+    _stop_latch.count_down();
+    if (_thread) {
+        _thread->join();
+    }
+    LOG(INFO) << "workload schedule manager stopped, thread is finished. ";
+}
+
+void WorkloadSchedPolicyMgr::update_workload_sched_policy(
+        std::map<uint64_t, std::shared_ptr<WorkloadSchedPolicy>> new_policy_map) {
+    std::lock_guard<std::shared_mutex> write_lock(_policy_lock);
+    // 1 upsert
+    for (const auto& [id, policy] : new_policy_map) {
+        if (_id_policy_map.find(id) == _id_policy_map.end()) {
+            _id_policy_map.emplace(id, policy);
+        } else {
+            if (policy->version() > _id_policy_map.at(id)->version()) {
+                _id_policy_map[id] = policy;
+            }
+        }
+    }
+
+    // 2 delete
+    for (auto iter = _id_policy_map.begin(); iter != _id_policy_map.end();) {
+        if (new_policy_map.find(iter->first) == new_policy_map.end()) {
+            iter = _id_policy_map.erase(iter);
+        } else {
+            iter++;
+        }
+    }
+}
+
+void WorkloadSchedPolicyMgr::_schedule_workload() {
+    while (!_stop_latch.wait_for(std::chrono::milliseconds(500))) {
+        // 1 get query info
+        std::vector<WorkloadQueryInfo> list;
+        _exec_env->fragment_mgr()->get_runtime_query_info(&list);
+        // todo: add timer
+        LOG(INFO) << "[workload_schedule] get query list size=" << list.size();
+
+        for (int i = 0; i < list.size(); i++) {
+            WorkloadQueryInfo* query_info_ptr = &(list[i]);
+            // 2 get matched policy
+            std::map<WorkloadActionType, std::shared_ptr<WorkloadSchedPolicy>> matched_policy_map;
+            {
+                std::shared_lock<std::shared_mutex> read_lock(_policy_lock);
+                for (auto& entity : _id_policy_map) {
+                    auto& new_policy = entity.second;
+                    if (new_policy->is_match(query_info_ptr)) {
+                        WorkloadActionType new_policy_type = new_policy->get_first_action_type();
+                        if (matched_policy_map.find(new_policy_type) == matched_policy_map.end() ||
+                            new_policy->priority() >
+                                    matched_policy_map.at(new_policy_type)->priority()) {
+                            matched_policy_map[new_policy_type] = new_policy;
+                        }
+                    }
+                }
+            }
+
+            if (matched_policy_map.size() == 0) {
+                continue;
+            }
+            LOG(INFO) << "[workload_schedule] matched policy size=" << matched_policy_map.size();
+            // 3 check action conflicts
+            if (matched_policy_map.find(WorkloadActionType::MOVE_QUERY_TO_GROUP) !=
+                        matched_policy_map.end() &&
+                matched_policy_map.find(WorkloadActionType::CANCEL_QUERY) !=
+                        matched_policy_map.end()) {
+                // compare priority
+                int move_prio =
+                        matched_policy_map.at(WorkloadActionType::MOVE_QUERY_TO_GROUP)->priority();
+                int cancel_prio =
+                        matched_policy_map.at(WorkloadActionType::CANCEL_QUERY)->priority();
+                if (cancel_prio >= move_prio) {
+                    matched_policy_map.erase(WorkloadActionType::MOVE_QUERY_TO_GROUP);
+                } else {
+                    matched_policy_map.erase(WorkloadActionType::CANCEL_QUERY);
+                }
+            }
+
+            // 4 exec policy action
+            for (const auto& [key, value] : matched_policy_map) {
+                value->exec_action(query_info_ptr);
+            }
+        }
+    }
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/workload_management/workload_sched_policy_mgr.h b/be/src/runtime/workload_management/workload_sched_policy_mgr.h
new file mode 100644
index 00000000000000..52b41eacf4b1c9
--- /dev/null
+++ b/be/src/runtime/workload_management/workload_sched_policy_mgr.h
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "runtime/exec_env.h"
+#include "runtime/workload_management/workload_sched_policy.h"
+#include "util/countdown_latch.h"
+
+namespace doris {
+
+class WorkloadSchedPolicyMgr {
+public:
+    WorkloadSchedPolicyMgr() : _stop_latch(0) {}
+    ~WorkloadSchedPolicyMgr() = default;
+
+    void start(ExecEnv* exec_env);
+
+    void stop();
+
+    void update_workload_sched_policy(
+            std::map<uint64_t, std::shared_ptr<WorkloadSchedPolicy>> policy_map);
+
+private:
+    void _schedule_workload();
+
+    std::shared_mutex _policy_lock;
+    std::map<uint64_t, std::shared_ptr<WorkloadSchedPolicy>> _id_policy_map;
+
+    std::shared_mutex _stop_lock;
+    CountDownLatch _stop_latch;
+    scoped_refptr<Thread> _thread;
+    ExecEnv* _exec_env;
+};
+
+}; // namespace doris
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index cd77f70f59c4fa..1ad131abb0ae18 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -230,6 +230,7 @@
 import org.apache.doris.resource.Tag;
 import org.apache.doris.resource.workloadgroup.WorkloadGroupMgr;
 import org.apache.doris.resource.workloadschedpolicy.WorkloadSchedPolicyMgr;
+import org.apache.doris.resource.workloadschedpolicy.WorkloadSchedPolicyPublisher;
 import org.apache.doris.scheduler.manager.TransientTaskManager;
 import org.apache.doris.scheduler.registry.ExportTaskRegister;
 import org.apache.doris.service.ExecuteEnv;
@@ -998,6 +999,8 @@ public void initialize(String[] args) throws Exception {
 
         TopicPublisher wgPublisher = new WorkloadGroupPublisher(this);
         topicPublisherThread.addToTopicPublisherList(wgPublisher);
+        WorkloadSchedPolicyPublisher wpPublisher = new WorkloadSchedPolicyPublisher(this);
+        topicPublisherThread.addToTopicPublisherList(wpPublisher);
         topicPublisherThread.start();
 
         workloadGroupMgr.startUpdateThread();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java b/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java
index 616c8a30b5668a..db5158e24f5a01 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java
@@ -59,9 +59,6 @@ public void addToTopicPublisherList(TopicPublisher topicPublisher) {
 
     @Override
     protected void runAfterCatalogReady() {
-        if (!Config.enable_workload_group) {
-            return;
-        }
         LOG.info("begin publish topic info");
         // step 1: get all publish topic info
         TPublishTopicRequest request = new TPublishTopicRequest();
@@ -69,6 +66,10 @@ protected void runAfterCatalogReady() {
             topicPublisher.getTopicInfo(request);
         }
 
+        if (request.getTopicMap().size() == 0) {
+            return;
+        }
+
         // step 2: publish topic info to all be
         Collection<Backend> nodesToPublish = clusterInfoService.getIdToBackend().values();
         AckResponseHandler handler = new AckResponseHandler(nodesToPublish);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java b/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java
index 6c5ce9e4c11d26..e81ccb06d3e564 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java
@@ -20,6 +20,9 @@
 import org.apache.doris.catalog.Env;
 import org.apache.doris.thrift.TPublishTopicRequest;
 import org.apache.doris.thrift.TTopicInfoType;
+import org.apache.doris.thrift.TopicInfo;
+
+import java.util.List;
 
 public class WorkloadGroupPublisher implements TopicPublisher {
 
@@ -31,7 +34,9 @@ public WorkloadGroupPublisher(Env env) {
 
     @Override
     public void getTopicInfo(TPublishTopicRequest req) {
-        req.putToTopicMap(TTopicInfoType.WORKLOAD_GROUP,
-                env.getWorkloadGroupMgr().getPublishTopicInfo());
+        List<TopicInfo> list = env.getWorkloadGroupMgr().getPublishTopicInfo();
+        if (list.size() > 0) {
+            req.putToTopicMap(TTopicInfoType.WORKLOAD_GROUP, list);
+        }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java
index 827c2367133835..d514ea62d2d0ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java
@@ -22,8 +22,16 @@
 import org.apache.doris.common.io.Writable;
 import org.apache.doris.persist.gson.GsonPostProcessable;
 import org.apache.doris.persist.gson.GsonUtils;
+import org.apache.doris.thrift.TCompareOperator;
+import org.apache.doris.thrift.TWorkloadAction;
+import org.apache.doris.thrift.TWorkloadActionType;
+import org.apache.doris.thrift.TWorkloadCondition;
+import org.apache.doris.thrift.TWorkloadMetricType;
+import org.apache.doris.thrift.TWorkloadSchedPolicy;
+import org.apache.doris.thrift.TopicInfo;
 
 import com.esotericsoftware.minlog.Log;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.gson.annotations.SerializedName;
 
@@ -41,6 +49,23 @@ public class WorkloadSchedPolicy implements Writable, GsonPostProcessable {
     public static final ImmutableSet<String> POLICY_PROPERTIES = new ImmutableSet.Builder<String>()
             .add(ENABLED).add(PRIORITY).build();
 
+    // used for convert fe type to thrift type
+    private static ImmutableMap<WorkloadMetricType, TWorkloadMetricType> METRIC_MAP
+            = new ImmutableMap.Builder<WorkloadMetricType, TWorkloadMetricType>()
+            .put(WorkloadMetricType.QUERY_TIME, TWorkloadMetricType.QUERY_TIME).build();
+    private static ImmutableMap<WorkloadActionType, TWorkloadActionType> ACTION_MAP
+            = new ImmutableMap.Builder<WorkloadActionType, TWorkloadActionType>()
+            .put(WorkloadActionType.MOVE_QUERY_TO_GROUP, TWorkloadActionType.MOVE_QUERY_TO_GROUP)
+            .put(WorkloadActionType.CANCEL_QUERY, TWorkloadActionType.CANCEL_QUERY).build();
+
+    private static ImmutableMap<WorkloadConditionOperator, TCompareOperator> OP_MAP
+            = new ImmutableMap.Builder<WorkloadConditionOperator, TCompareOperator>()
+            .put(WorkloadConditionOperator.EQUAL, TCompareOperator.EQUAL)
+            .put(WorkloadConditionOperator.GREATER, TCompareOperator.GREATER)
+            .put(WorkloadConditionOperator.GREATER_EQUAL, TCompareOperator.GREATER_EQUAL)
+            .put(WorkloadConditionOperator.LESS, TCompareOperator.LESS)
+            .put(WorkloadConditionOperator.LESS_EQUAl, TCompareOperator.LESS_EQUAL).build();
+
     @SerializedName(value = "id")
     long id;
     @SerializedName(value = "name")
@@ -173,6 +198,48 @@ public List<WorkloadActionMeta> getActionMetaList() {
         return actionMetaList;
     }
 
+    public TopicInfo toTopicInfo() {
+        TWorkloadSchedPolicy tPolicy = new TWorkloadSchedPolicy();
+        tPolicy.setId(id);
+        tPolicy.setName(name);
+        tPolicy.setVersion(version);
+        tPolicy.setPriority(priority);
+        tPolicy.setEnabled(enabled);
+
+        List<TWorkloadCondition> condList = new ArrayList();
+        for (WorkloadConditionMeta cond : conditionMetaList) {
+            TWorkloadCondition tCond = new TWorkloadCondition();
+            TWorkloadMetricType metricType = METRIC_MAP.get(cond.metricName);
+            if (metricType == null) {
+                return null;
+            }
+            tCond.setMetricName(metricType);
+            tCond.setOp(OP_MAP.get(cond.op));
+            tCond.setValue(cond.value);
+            condList.add(tCond);
+        }
+
+        List<TWorkloadAction> actionList = new ArrayList();
+        for (WorkloadActionMeta action : actionMetaList) {
+            TWorkloadAction tAction = new TWorkloadAction();
+            TWorkloadActionType tActionType = ACTION_MAP.get(action.action);
+            if (tActionType == null) {
+                return null;
+            }
+            tAction.setAction(tActionType);
+            tAction.setActionArgs(action.actionArgs);
+            actionList.add(tAction);
+        }
+
+        tPolicy.setConditionList(condList);
+        tPolicy.setActionList(actionList);
+
+        TopicInfo topicInfo = new TopicInfo();
+        topicInfo.setWorkloadSchedPolicy(tPolicy);
+
+        return topicInfo;
+    }
+
     @Override
     public void write(DataOutput out) throws IOException {
         String json = GsonUtils.GSON.toJson(this);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
index 346e34796c7aa2..45ba3a35de469b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
@@ -35,6 +35,7 @@
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.service.ExecuteEnv;
 import org.apache.doris.thrift.TUserIdentity;
+import org.apache.doris.thrift.TopicInfo;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -412,6 +413,22 @@ private void writeUnlock() {
         lock.writeLock().unlock();
     }
 
+    public List<TopicInfo> getPublishTopicInfoList() {
+        List<TopicInfo> topicInfoList = new ArrayList();
+        readLock();
+        try {
+            for (Map.Entry<Long, WorkloadSchedPolicy> entry : idToPolicy.entrySet()) {
+                TopicInfo tInfo = entry.getValue().toTopicInfo();
+                if (tInfo != null) {
+                    topicInfoList.add(tInfo);
+                }
+            }
+        } finally {
+            readUnlock();
+        }
+        return topicInfoList;
+    }
+
     public void replayCreateWorkloadSchedPolicy(WorkloadSchedPolicy policy) {
         insertWorkloadSchedPolicy(policy);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java
new file mode 100644
index 00000000000000..5083d183e6c957
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.resource.workloadschedpolicy;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.publish.TopicPublisher;
+import org.apache.doris.thrift.TPublishTopicRequest;
+import org.apache.doris.thrift.TTopicInfoType;
+import org.apache.doris.thrift.TopicInfo;
+
+import java.util.List;
+
+public class WorkloadSchedPolicyPublisher implements TopicPublisher {
+
+    private Env env;
+
+    public WorkloadSchedPolicyPublisher(Env env) {
+        this.env = env;
+    }
+
+    @Override
+    public void getTopicInfo(TPublishTopicRequest req) {
+        List<TopicInfo> list = env.getWorkloadSchedPolicyMgr().getPublishTopicInfoList();
+        if (list.size() > 0) {
+            req.putToTopicMap(TTopicInfoType.WORKLOAD_SCHED_POLICY, list);
+        }
+    }
+
+}
diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift
index 855b74897114ef..2c5b199fc1749c 100644
--- a/gensrc/thrift/BackendService.thrift
+++ b/gensrc/thrift/BackendService.thrift
@@ -163,6 +163,7 @@ struct TQueryIngestBinlogResult {
 enum TTopicInfoType {
     WORKLOAD_GROUP
     MOVE_QUERY_TO_GROUP
+    WORKLOAD_SCHED_POLICY
 }
 
 struct TWorkloadGroupInfo {
@@ -178,12 +179,53 @@ struct TWorkloadGroupInfo {
 
 struct TWorkloadMoveQueryToGroupAction {
     1: optional Types.TUniqueId query_id
-    2: optional i64 workload_group_id;
+    2: optional i64 workload_group_id
+}
+
+enum TWorkloadMetricType {
+    QUERY_TIME
+    SCAN_ROWS
+    SCAN_BYTES
+}
+
+enum TCompareOperator {
+    EQUAL
+    GREATER
+    GREATER_EQUAL
+    LESS
+    LESS_EQUAL
+}
+
+struct TWorkloadCondition {
+    1: optional TWorkloadMetricType metric_name
+    2: optional TCompareOperator op
+    3: optional string value
+}
+
+enum TWorkloadActionType {
+    MOVE_QUERY_TO_GROUP
+    CANCEL_QUERY
+}
+
+struct TWorkloadAction {
+    1: optional TWorkloadActionType action
+    2: optional string action_args
+}
+
+struct TWorkloadSchedPolicy {
+    1: optional i64 id
+    2: optional string name
+    3: optional i32 version
+    4: optional i32 priority
+    5: optional bool enabled
+    6: optional list<TWorkloadCondition> condition_list
+    7: optional list<TWorkloadAction> action_list
 }
 
 struct TopicInfo {
     1: optional TWorkloadGroupInfo workload_group_info
     2: optional TWorkloadMoveQueryToGroupAction move_action
+    3: optional TWorkloadSchedPolicy workload_sched_policy
 }
 
 struct TPublishTopicRequest {

From 14c902b50410c5ab0ff039df960abba36e3520c2 Mon Sep 17 00:00:00 2001
From: yujun <yu.jun.reach@gmail.com>
Date: Thu, 28 Dec 2023 15:09:21 +0800
Subject: [PATCH 114/299] [fix](regression test) fix test_alter_colocate_table
 (#29009)

---
 .../doris/catalog/ColocateGroupSchema.java    |  9 +++------
 .../org/apache/doris/common/ErrorCode.java    |  2 +-
 .../doris/catalog/ColocateTableTest.java      |  3 ++-
 .../test_alter_colocate_table.groovy          | 19 +++++++++++--------
 4 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java
index 32c48c5fba07cd..40b3405069d8b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColocateGroupSchema.java
@@ -116,18 +116,15 @@ public void checkDistribution(DistributionInfo distributionInfo) throws DdlExcep
     }
 
     private void checkReplicaAllocation(PartitionInfo partitionInfo) throws DdlException {
-        for (ReplicaAllocation replicaAlloc : partitionInfo.idToReplicaAllocation.values()) {
-            if (!replicaAlloc.equals(this.replicaAlloc)) {
-                ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_REPLICATION_ALLOCATION,
-                        this.replicaAlloc);
-            }
+        for (ReplicaAllocation alloc : partitionInfo.idToReplicaAllocation.values()) {
+            checkReplicaAllocation(alloc);
         }
     }
 
     public void checkReplicaAllocation(ReplicaAllocation replicaAlloc) throws DdlException {
         if (!replicaAlloc.equals(this.replicaAlloc)) {
             ErrorReport.reportDdlException(ErrorCode.ERR_COLOCATE_TABLE_MUST_HAS_SAME_REPLICATION_ALLOCATION,
-                    this.replicaAlloc);
+                    replicaAlloc, this.replicaAlloc);
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
index 95c65c7bbd947e..c36821fd996e95 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorCode.java
@@ -1110,7 +1110,7 @@ public enum ErrorCode {
     ERR_COLOCATE_TABLE_MUST_BE_OLAP_TABLE(5063, new byte[]{'4', '2', '0', '0', '0'},
             "Colocate table '%s' must be OLAP table"),
     ERR_COLOCATE_TABLE_MUST_HAS_SAME_REPLICATION_ALLOCATION(5063, new byte[]{'4', '2', '0', '0', '0'},
-            "Colocate tables must have same replication allocation: %s"),
+            "Colocate tables must have same replication allocation: { %s } should be { %s }"),
     ERR_COLOCATE_TABLE_MUST_HAS_SAME_BUCKET_NUM(5063, new byte[]{'4', '2', '0', '0', '0'},
             "Colocate tables must have same bucket num: %s"),
     ERR_COLOCATE_TABLE_MUST_HAS_SAME_DISTRIBUTION_COLUMN_SIZE(5063, new byte[]{'4', '2', '0', '0', '0'},
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java
index 4ea69f03945dcb..87b51808431a42 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ColocateTableTest.java
@@ -252,7 +252,8 @@ public void testReplicationNum() throws Exception {
                 + ");");
 
         expectedEx.expect(DdlException.class);
-        expectedEx.expectMessage("Colocate tables must have same replication allocation: tag.location.default: 1");
+        expectedEx.expectMessage("Colocate tables must have same replication allocation: { tag.location.default: 2 }"
+                + " should be { tag.location.default: 1 }");
         createTable("create table " + dbName + "." + tableName2 + " (\n"
                 + " `k1` int NULL COMMENT \"\",\n"
                 + " `k2` varchar(10) NULL COMMENT \"\"\n"
diff --git a/regression-test/suites/schema_change_p0/test_alter_colocate_table.groovy b/regression-test/suites/schema_change_p0/test_alter_colocate_table.groovy
index aac70b46e1c83e..ce23f70a3e1d0c 100644
--- a/regression-test/suites/schema_change_p0/test_alter_colocate_table.groovy
+++ b/regression-test/suites/schema_change_p0/test_alter_colocate_table.groovy
@@ -37,6 +37,9 @@ suite ("test_alter_colocate_table") {
     }
     assertFalse(existsGroup)
 
+    def forceReplicaNum = getFeConfig('force_olap_table_replication_num') as int
+    def replicaNum = forceReplicaNum > 0 ? forceReplicaNum : 1
+
     sql """
         CREATE TABLE IF NOT EXISTS col_tbl1
         (
@@ -48,7 +51,7 @@ suite ("test_alter_colocate_table") {
         DISTRIBUTED BY HASH(k2) BUCKETS 3
         PROPERTIES
         (
-            "replication_allocation" = "tag.location.default:1",
+            "replication_allocation" = "tag.location.default:${replicaNum}",
             "colocate_with" = 'x_group_1'
         )
     """
@@ -69,7 +72,7 @@ suite ("test_alter_colocate_table") {
         DISTRIBUTED BY HASH(k2) BUCKETS 3
         PROPERTIES
         (
-            "replication_allocation" = "tag.location.default:1",
+            "replication_allocation" = "tag.location.default:${replicaNum}",
             "colocate_with" = 'x_group_2'
         )
     """
@@ -85,7 +88,7 @@ suite ("test_alter_colocate_table") {
         DISTRIBUTED BY HASH(uuid) BUCKETS 3
         PROPERTIES
         (
-            "replication_allocation" = "tag.location.default:1",
+            "replication_allocation" = "tag.location.default:${replicaNum}",
             "colocate_with" = "x_group_3",
             "dynamic_partition.enable" = "true",
             "dynamic_partition.time_unit" = "DAY",
@@ -94,7 +97,7 @@ suite ("test_alter_colocate_table") {
             "dynamic_partition.buckets" = "3",
             "dynamic_partition.replication_num" = "1",
             "dynamic_partition.create_history_partition"= "true",
-            "dynamic_partition.replication_allocation" = "tag.location.default:1",
+            "dynamic_partition.replication_allocation" = "tag.location.default:${replicaNum}",
             "dynamic_partition.start" = "-3"
          );
     """
@@ -104,7 +107,7 @@ suite ("test_alter_colocate_table") {
     test {
         sql """
            ALTER TABLE col_tbl1 set (
-                "replication_allocation" = "tag.location.default:1"
+                "replication_allocation" = "tag.location.default:${replicaNum}"
            )
         """
         exception errMsg
@@ -113,7 +116,7 @@ suite ("test_alter_colocate_table") {
     test {
         sql """
            ALTER TABLE col_tbl3 set (
-                "dynamic_partition.replication_allocation" = "tag.location.default:1"
+                "dynamic_partition.replication_allocation" = "tag.location.default:${replicaNum}"
            )
         """
         exception errMsg
@@ -123,7 +126,7 @@ suite ("test_alter_colocate_table") {
         test {
             sql """
               ALTER TABLE ${tbl} set (
-                "default.replication_allocation" = "tag.location.default:1"
+                "default.replication_allocation" = "tag.location.default:${replicaNum}"
               )
             """
 
@@ -133,7 +136,7 @@ suite ("test_alter_colocate_table") {
         test {
             sql """
               ALTER TABLE ${tbl} MODIFY PARTITION (*) set (
-                "replication_allocation" = "tag.location.default:1"
+                "replication_allocation" = "tag.location.default:${replicaNum}"
               )
             """
 

From 5171a77f9e2e17056a59b4e876ab0ef28ae19920 Mon Sep 17 00:00:00 2001
From: jakevin <jakevingoo@gmail.com>
Date: Thu, 28 Dec 2023 15:32:45 +0800
Subject: [PATCH 115/299] [fix](Nereids): merge Offset in Limit Translator
 (#29100)

---
 .../translator/PhysicalPlanTranslator.java    |   5 +-
 .../nereids/rules/rewrite/MergeLimits.java    |  16 +-
 .../eliminate_not_null/eliminate_not_null.out |  63 +++----
 .../push_filter_inside_join.out               | 156 ++++++------------
 .../eliminate_not_null.groovy                 |   2 +
 .../filter_push_through_aggregate.groovy      |   4 +-
 .../push_filter_inside_join.groovy            |   3 +-
 7 files changed, 98 insertions(+), 151 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 75d886a1dc7d6d..6ad21d348905b8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -65,6 +65,7 @@
 import org.apache.doris.nereids.properties.OrderKey;
 import org.apache.doris.nereids.properties.PhysicalProperties;
 import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup;
+import org.apache.doris.nereids.rules.rewrite.MergeLimits;
 import org.apache.doris.nereids.stats.StatsErrorEstimator;
 import org.apache.doris.nereids.trees.UnaryNode;
 import org.apache.doris.nereids.trees.expressions.AggregateExpression;
@@ -1536,8 +1537,8 @@ public PlanFragment visitPhysicalNestedLoopJoin(
     public PlanFragment visitPhysicalLimit(PhysicalLimit<? extends Plan> physicalLimit, PlanTranslatorContext context) {
         PlanFragment inputFragment = physicalLimit.child(0).accept(this, context);
         PlanNode child = inputFragment.getPlanRoot();
-        child.setOffset(physicalLimit.getOffset());
-        child.setLimit(physicalLimit.getLimit());
+        child.setLimit(MergeLimits.mergeLimit(physicalLimit.getLimit(), physicalLimit.getOffset(), child.getLimit()));
+        child.setOffset(MergeLimits.mergeOffset(physicalLimit.getOffset(), child.getOffset()));
         updateLegacyPlanIdToPhysicalPlan(child, physicalLimit);
         return inputFragment;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java
index e1428983c159f7..e544eed07ce98d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/MergeLimits.java
@@ -47,11 +47,21 @@ public Rule build() {
                 .then(upperLimit -> {
                     LogicalLimit<? extends Plan> bottomLimit = upperLimit.child();
                     return new LogicalLimit<>(
-                            Math.min(upperLimit.getLimit(),
-                                    Math.max(bottomLimit.getLimit() - upperLimit.getOffset(), 0)),
-                            bottomLimit.getOffset() + upperLimit.getOffset(),
+                            mergeLimit(upperLimit.getLimit(), upperLimit.getOffset(), bottomLimit.getLimit()),
+                            mergeOffset(upperLimit.getOffset(), bottomLimit.getOffset()),
                             upperLimit.child().getPhase(), bottomLimit.child()
                     );
                 }).toRule(RuleType.MERGE_LIMITS);
     }
+
+    public static long mergeOffset(long upperOffset, long bottomOffset) {
+        return upperOffset + bottomOffset;
+    }
+
+    public static long mergeLimit(long upperLimit, long upperOffset, long bottomLimit) {
+        if (bottomLimit < 0) {
+            return upperLimit;
+        }
+        return Math.min(upperLimit, Math.max(bottomLimit - upperOffset, 0));
+    }
 }
diff --git a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out
index a02974e6ccb4e3..929e4ce0d67cd7 100644
--- a/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out
+++ b/regression-test/data/nereids_rules_p0/eliminate_not_null/eliminate_not_null.out
@@ -1,70 +1,55 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !eliminate_not_null_basic_comparison --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not score IS NULL) and (t.score > 13))
-------PhysicalOlapScan[t]
+--filter(( not score IS NULL) and (t.score > 13))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_in_clause --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not id IS NULL) and id IN (1, 2, 3))
-------PhysicalOlapScan[t]
+--filter(( not id IS NULL) and id IN (1, 2, 3))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_not_equal --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not (score = 13)) and ( not score IS NULL))
-------PhysicalOlapScan[t]
+--filter(( not (score = 13)) and ( not score IS NULL))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_string_function --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not name IS NULL) and (length(name) > 0))
-------PhysicalOlapScan[t]
+--filter(( not name IS NULL) and (length(name) > 0))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_aggregate --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashAgg[LOCAL]
---------PhysicalProject
-----------filter(( not score IS NULL) and (t.score > 0))
-------------PhysicalOlapScan[t]
+--hashAgg[LOCAL]
+----filter(( not score IS NULL) and (t.score > 0))
+------PhysicalOlapScan[t]
 
 -- !eliminate_not_null_between --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not score IS NULL) and (t.score <= 10) and (t.score >= 1))
-------PhysicalOlapScan[t]
+--filter(( not score IS NULL) and (t.score <= 10) and (t.score >= 1))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_math_function --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not score IS NULL) and (abs(score) = 5))
-------PhysicalOlapScan[t]
+--filter(( not score IS NULL) and (abs(score) = 5))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_complex_logic --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not score IS NULL) and ((t.score > 5) OR (t.id < 10)))
-------PhysicalOlapScan[t]
+--filter(( not score IS NULL) and ((t.score > 5) OR (t.id < 10)))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_date_function --
 PhysicalResultSink
---PhysicalDistribute
-----filter(( not name IS NULL) and (year(cast(name as DATEV2)) = 2022))
-------PhysicalOlapScan[t]
+--filter(( not name IS NULL) and (year(cast(name as DATEV2)) = 2022))
+----PhysicalOlapScan[t]
 
 -- !eliminate_not_null_with_subquery --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[LEFT_SEMI_JOIN] hashCondition=((t.score = t.score)) otherCondition=() build RFs:RF0 score->[score]
---------filter(( not score IS NULL) and (t.score > 0))
-----------PhysicalOlapScan[t] apply RFs: RF0
---------PhysicalDistribute
-----------PhysicalProject
-------------filter((t.score > 0))
---------------PhysicalOlapScan[t]
+--hashJoin[LEFT_SEMI_JOIN] hashCondition=((t.score = t.score)) otherCondition=() build RFs:RF0 score->[score]
+----filter(( not score IS NULL) and (t.score > 0))
+------PhysicalOlapScan[t] apply RFs: RF0
+----filter((t.score > 0))
+------PhysicalOlapScan[t]
 
diff --git a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out
index bbac000f08d52d..355b4b231fc7af 100644
--- a/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out
+++ b/regression-test/data/nereids_rules_p0/filter_push_down/push_filter_inside_join.out
@@ -1,157 +1,103 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !pushdown_cross_join --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------NestedLoopJoin[INNER_JOIN](t1.msg > t2.msg)
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--NestedLoopJoin[INNER_JOIN](t1.msg > t2.msg)
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_cross_join --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=()
---------PhysicalDistribute
-----------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=()
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_inner_join --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_left_join --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg > t2.msg))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_right_join --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t2.msg < t1.msg))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t2.msg < t1.msg))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_full_join --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg < t2.msg))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id)) otherCondition=((t1.msg < t2.msg))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_cross_join --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------NestedLoopJoin[INNER_JOIN](t1.msg < t2.msg)
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--NestedLoopJoin[INNER_JOIN](t1.msg < t2.msg)
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_inner_join_hash --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=()
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=()
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_left_join_hash --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=()
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=()
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_right_join_hash --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=()
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=()
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_full_join_hash --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=()
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=()
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_cross_join_combine --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
---------PhysicalDistribute
-----------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_inner_join_combine --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_left_join_combine --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_right_join_combine --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t2.msg = t1.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_full_join_combine --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
---------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.id = t2.id) and (t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
 -- !pushdown_cross_join_combine --
 PhysicalResultSink
---PhysicalDistribute
-----PhysicalProject
-------hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
---------PhysicalDistribute
-----------PhysicalOlapScan[t1]
---------PhysicalDistribute
-----------PhysicalOlapScan[t2]
+--hashJoin[INNER_JOIN] hashCondition=((t1.msg = t2.msg)) otherCondition=(((cast(msg as DOUBLE) + cast(msg as DOUBLE)) = cast('' as DOUBLE)))
+----PhysicalOlapScan[t1]
+----PhysicalOlapScan[t2]
 
diff --git a/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy b/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy
index 1b81108ada10e3..cc318a3e854f73 100644
--- a/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy
+++ b/regression-test/suites/nereids_rules_p0/eliminate_not_null/eliminate_not_null.groovy
@@ -18,6 +18,8 @@
 suite("eliminate_not_null") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "SET disable_join_reorder=true"
 
     sql """
         DROP TABLE IF EXISTS t
diff --git a/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy b/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy
index dfb65dd24669ba..735aaa939e6364 100644
--- a/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy
+++ b/regression-test/suites/nereids_rules_p0/filter_push_down/filter_push_through_aggregate.groovy
@@ -18,7 +18,9 @@
 suite("filter_push_through_aggregate") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
-
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "SET disable_join_reorder=true"
+  
     sql """
         DROP TABLE IF EXISTS filter_push_through_aggregate
        """
diff --git a/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy b/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy
index a19ddfd50639c8..9dfadb9630bacb 100644
--- a/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/filter_push_down/push_filter_inside_join.groovy
@@ -19,7 +19,8 @@ suite("push_filter_inside_join") {
     sql "SET enable_nereids_planner=true"
     sql "SET enable_fallback_to_original_planner=false"
     sql "use regression_test_nereids_rules_p0"
-    sql "set disable_join_reorder=true"
+    sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'"
+    sql "SET disable_join_reorder=true"
     sql 'set be_number_for_test=3'
 
     // Push down > condition to cross join

From ba7b7c1f60ad532dcad9a1fc8c3654856db4a342 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Thu, 28 Dec 2023 15:40:16 +0800
Subject: [PATCH 116/299] [Chore](Job)It is forbidden to change the status of
 internal JOB through PAUSE/RESUME (#29036)

---
 .../java/org/apache/doris/analysis/AlterJobStatusStmt.java  | 4 +++-
 .../main/java/org/apache/doris/analysis/CreateJobStmt.java  | 6 +++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java
index 09f135d2fb02b9..16077cc2609caf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AlterJobStatusStmt.java
@@ -58,6 +58,8 @@ public void analyze(Analyzer analyzer) throws UserException {
             throw new AnalysisException("Value can't is null");
         }
         this.jobName = inputValue;
-
+        if (CreateJobStmt.isInnerJob(jobName)) {
+            throw new AnalysisException("Can't alter inner job status");
+        }
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
index ef76aedba2b400..17e2185bf5e216 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateJobStmt.java
@@ -85,7 +85,7 @@ public class CreateJobStmt extends DdlStmt {
     private JobExecuteType executeType;
 
     // exclude job name prefix, which is used by inner job
-    private final String excludeJobNamePrefix = "inner_";
+    private static final String excludeJobNamePrefix = "inner_";
 
     private static final ImmutableSet<Class<? extends DdlStmt>> supportStmtSuperClass
             = new ImmutableSet.Builder<Class<? extends DdlStmt>>().add(InsertStmt.class)
@@ -208,4 +208,8 @@ private String parseExecuteSql(String sql) throws AnalysisException {
         }
         return executeSql;
     }
+
+    protected static boolean isInnerJob(String jobName) {
+        return jobName.startsWith(excludeJobNamePrefix);
+    }
 }

From 8becf053cb1597d28a906efe9b7abede23f36930 Mon Sep 17 00:00:00 2001
From: slothever <18522955+wsjz@users.noreply.github.com>
Date: Thu, 28 Dec 2023 15:43:28 +0800
Subject: [PATCH 117/299] [fix](multi-catalog)unsupported hive input format
 should throw an exception and remove useless method (#29087)

introduce from: #28644
---
 .../main/java/org/apache/doris/common/Config.java    |  6 ------
 .../doris/catalog/external/HMSExternalTable.java     | 12 +++++++++++-
 .../main/java/org/apache/doris/common/util/Util.java | 10 ----------
 3 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index eae58ddaafd4ab..0f4d0193470efe 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1679,12 +1679,6 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true)
     public static boolean enable_decimal_conversion = true;
 
-    /**
-     * List of S3 API compatible object storage systems.
-     */
-    @ConfField
-    public static String s3_compatible_object_storages = "s3,oss,cos,bos";
-
     /**
      * Support complex data type ARRAY.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index 8d86ea72afcfa6..0528194b64fff0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -203,12 +203,22 @@ public boolean isHoodieCowTable() {
      * Support managed_table and external_table.
      */
     private boolean supportedHiveTable() {
+        // we will return false if null, which means that the table type maybe unsupported.
+        if (remoteTable.getSd() == null) {
+            return false;
+        }
         String inputFileFormat = remoteTable.getSd().getInputFormat();
         if (inputFileFormat == null) {
             return false;
         }
+        boolean supportedFileFormat = SUPPORTED_HIVE_FILE_FORMATS.contains(inputFileFormat);
+        if (!supportedFileFormat) {
+            // for easier debugging, need return error message if unsupported input format is used.
+            // NotSupportedException is required by some operation.
+            throw new NotSupportedException("Unsupported hive input format: " + inputFileFormat);
+        }
         LOG.debug("hms table {} is {} with file format: {}", name, remoteTable.getTableType(), inputFileFormat);
-        return SUPPORTED_HIVE_FILE_FORMATS.contains(inputFileFormat);
+        return true;
     }
 
     /**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index 58de5133c6b5f3..87b67dfa28d5ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -20,7 +20,6 @@
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.Config;
 import org.apache.doris.common.FeNameFormat;
 import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.qe.ConnectContext;
@@ -513,15 +512,6 @@ public static void prohibitExternalCatalog(String catalog, String msg) throws An
         }
     }
 
-    public static boolean isS3CompatibleStorageSchema(String schema) {
-        for (String objectStorage : Config.s3_compatible_object_storages.split(",")) {
-            if (objectStorage.equalsIgnoreCase(schema)) {
-                return true;
-            }
-        }
-        return false;
-    }
-
     private static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
 
     public static String bytesToHex(byte[] bytes) {

From 03a6a2880a1204829845841511dd0c503fb5d19e Mon Sep 17 00:00:00 2001
From: walter <w41ter.l@gmail.com>
Date: Thu, 28 Dec 2023 15:57:51 +0800
Subject: [PATCH 118/299] [fix](journal) Fix infinite block due to initial BDB
 journal failed (#29205)

Opening a BDBJournal will acquire the max journal id, but it doesn't
need to check whether the replica txn is matched with the master.
---
 .../apache/doris/journal/bdbje/BDBJEJournal.java    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
index 134b609549e2a8..3698b8f76ca717 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
@@ -271,6 +271,15 @@ public JournalCursor read(long fromKey, long toKey) {
 
     @Override
     public long getMaxJournalId() {
+        return getMaxJournalIdInternal(true);
+    }
+
+    // get max journal id but do not check whether the txn is matched.
+    private long getMaxJournalIdWithoutCheck() {
+        return getMaxJournalIdInternal(false);
+    }
+
+    private long getMaxJournalIdInternal(boolean checkTxnMatched) {
         long ret = -1;
         if (bdbEnvironment == null) {
             return ret;
@@ -287,7 +296,7 @@ public long getMaxJournalId() {
         String dbName = dbNames.get(index).toString();
         long dbNumberName = dbNames.get(index);
         Database database = bdbEnvironment.openDatabase(dbName);
-        if (!isReplicaTxnAreMatched(database, dbNumberName)) {
+        if (checkTxnMatched && !isReplicaTxnAreMatched(database, dbNumberName)) {
             LOG.warn("The current replica hasn't synced up with the master, current db name: {}", dbNumberName);
             if (index != 0) {
                 // Because roll journal occurs after write, the previous write must have
@@ -419,7 +428,7 @@ public synchronized void open() {
                 }
 
                 // set next journal id
-                nextJournalId.set(getMaxJournalId() + 1);
+                nextJournalId.set(getMaxJournalIdWithoutCheck() + 1);
 
                 break;
             } catch (InsufficientLogException insufficientLogEx) {

From b31494b18c8dd0fff14e1841a14236db28ef6552 Mon Sep 17 00:00:00 2001
From: zhengyu <freeman.zhang1992@gmail.com>
Date: Thu, 28 Dec 2023 16:16:26 +0800
Subject: [PATCH 119/299] [test](regression) add fault injection cases for
 LoadStream (#29101)

Signed-off-by: freemandealer <freeman.zhang1992@gmail.com>
---
 be/src/io/fs/file_writer.h                    |   6 +-
 be/src/io/fs/local_file_system.cpp            |   5 +
 be/src/io/fs/local_file_writer.cpp            |   6 +
 be/src/io/fs/stream_sink_file_writer.cpp      |   1 +
 be/src/olap/rowset_builder.cpp                |   2 +
 be/src/runtime/load_stream.cpp                |  15 +++
 be/src/runtime/load_stream_writer.cpp         |  15 ++-
 be/src/runtime/load_stream_writer.h           |   2 -
 .../test_load_stream_fault_injection.groovy   | 119 ++++++++++++++++++
 9 files changed, 165 insertions(+), 6 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy

diff --git a/be/src/io/fs/file_writer.h b/be/src/io/fs/file_writer.h
index b8f44c2e339ca8..51339afebcd0b3 100644
--- a/be/src/io/fs/file_writer.h
+++ b/be/src/io/fs/file_writer.h
@@ -22,6 +22,7 @@
 #include "common/status.h"
 #include "gutil/macros.h"
 #include "io/fs/path.h"
+#include "util/debug_points.h"
 #include "util/slice.h"
 
 namespace doris {
@@ -63,7 +64,10 @@ class FileWriter {
 
     const Path& path() const { return _path; }
 
-    size_t bytes_appended() const { return _bytes_appended; }
+    size_t bytes_appended() const {
+        DBUG_EXECUTE_IF("FileWriter.bytes_appended.zero_bytes_appended", { return 0; });
+        return _bytes_appended;
+    }
 
     std::shared_ptr<FileSystem> fs() const { return _fs; }
 
diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp
index a71708ac72c824..d7e1372ffe9672 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -40,6 +40,7 @@
 #include "io/fs/local_file_writer.h"
 #include "runtime/thread_context.h"
 #include "util/async_io.h" // IWYU pragma: keep
+#include "util/debug_points.h"
 #include "util/defer_op.h"
 
 namespace doris {
@@ -57,6 +58,10 @@ LocalFileSystem::~LocalFileSystem() = default;
 Status LocalFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer,
                                          const FileWriterOptions* opts) {
     int fd = ::open(file.c_str(), O_TRUNC | O_WRONLY | O_CREAT | O_CLOEXEC, 0666);
+    DBUG_EXECUTE_IF("LocalFileSystem.create_file_impl.open_file_failed", {
+        ::close(fd);
+        fd = -1;
+    });
     if (-1 == fd) {
         return Status::IOError("failed to open {}: {}", file.native(), errno_to_str());
     }
diff --git a/be/src/io/fs/local_file_writer.cpp b/be/src/io/fs/local_file_writer.cpp
index af913d60a985c0..3915cf91b6d2d3 100644
--- a/be/src/io/fs/local_file_writer.cpp
+++ b/be/src/io/fs/local_file_writer.cpp
@@ -38,6 +38,7 @@
 #include "io/fs/file_writer.h"
 #include "io/fs/local_file_system.h"
 #include "io/fs/path.h"
+#include "util/debug_points.h"
 #include "util/doris_metrics.h"
 
 namespace doris {
@@ -203,6 +204,11 @@ Status LocalFileWriter::_close(bool sync) {
     if (0 != ::close(_fd)) {
         return Status::IOError("cannot close {}: {}", _path.native(), std::strerror(errno));
     }
+
+    DBUG_EXECUTE_IF("LocalFileWriter.close.failed", {
+        return Status::IOError("cannot close {}: {}", _path.native(), std::strerror(errno));
+    });
+
     return Status::OK();
 }
 
diff --git a/be/src/io/fs/stream_sink_file_writer.cpp b/be/src/io/fs/stream_sink_file_writer.cpp
index 484be9e07e9daf..25a4f5b27d5bb0 100644
--- a/be/src/io/fs/stream_sink_file_writer.cpp
+++ b/be/src/io/fs/stream_sink_file_writer.cpp
@@ -21,6 +21,7 @@
 
 #include "olap/olap_common.h"
 #include "olap/rowset/beta_rowset_writer.h"
+#include "util/debug_points.h"
 #include "util/uid_util.h"
 #include "vec/sink/load_stream_stub.h"
 
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index 5fda708cf4eb56..e7fb520c0bfd1a 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -149,6 +149,8 @@ Status RowsetBuilder::check_tablet_version_count() {
                      << st;
     }
     int version_count = tablet()->version_count();
+    DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
+                    { version_count = INT_MAX; });
     if (version_count > config::max_tablet_version_num) {
         return Status::Error<TOO_MANY_VERSION>(
                 "failed to init rowset builder. version count: {}, exceed limit: {}, "
diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index 836d4da147afbb..d2ea41f1c35762 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -37,10 +37,13 @@
 #include "runtime/load_channel.h"
 #include "runtime/load_stream_mgr.h"
 #include "runtime/load_stream_writer.h"
+#include "util/debug_points.h"
 #include "util/runtime_profile.h"
 #include "util/thrift_util.h"
 #include "util/uid_util.h"
 
+#define UNKNOWN_ID_FOR_TEST 0x7c00
+
 namespace doris {
 
 bvar::LatencyRecorder g_load_stream_flush_wait_ms("load_stream_flush_wait_ms");
@@ -169,6 +172,7 @@ Status TabletStream::add_segment(const PStreamHeader& header, butil::IOBuf* data
     int64_t src_id = header.src_id();
     uint32_t segid = header.segment_id();
     uint32_t new_segid;
+    DBUG_EXECUTE_IF("TabletStream.add_segment.unknown_segid", { segid = UNKNOWN_ID_FOR_TEST; });
     {
         std::lock_guard lock_guard(_lock);
         if (!_segids_mapping.contains(src_id)) {
@@ -440,6 +444,8 @@ Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data)
     IndexStreamSharedPtr index_stream;
 
     int64_t index_id = header.index_id();
+    DBUG_EXECUTE_IF("TabletStream.add_segment.unknown_indexid",
+                    { index_id = UNKNOWN_ID_FOR_TEST; });
     auto it = _index_streams_map.find(index_id);
     if (it == _index_streams_map.end()) {
         return Status::Error<ErrorCode::INVALID_ARGUMENT>("unknown index_id {}", index_id);
@@ -479,6 +485,15 @@ int LoadStream::on_received_messages(StreamId id, butil::IOBuf* const messages[]
 void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* data) {
     VLOG_DEBUG << PStreamHeader_Opcode_Name(hdr.opcode()) << " from " << hdr.src_id()
                << " with tablet " << hdr.tablet_id();
+    DBUG_EXECUTE_IF("LoadStream._dispatch.unknown_loadid", {
+        PUniqueId& load_id = const_cast<PUniqueId&>(hdr.load_id());
+        load_id.set_hi(UNKNOWN_ID_FOR_TEST);
+        load_id.set_lo(UNKNOWN_ID_FOR_TEST);
+    });
+    DBUG_EXECUTE_IF("LoadStream._dispatch.unknown_srcid", {
+        PStreamHeader& t_hdr = const_cast<PStreamHeader&>(hdr);
+        t_hdr.set_src_id(UNKNOWN_ID_FOR_TEST);
+    });
     if (UniqueId(hdr.load_id()) != UniqueId(_load_id)) {
         Status st = Status::Error<ErrorCode::INVALID_ARGUMENT>(
                 "invalid load id {}, expected {}", print_id(hdr.load_id()), print_id(_load_id));
diff --git a/be/src/runtime/load_stream_writer.cpp b/be/src/runtime/load_stream_writer.cpp
index 0a339e854aae32..27ca047d6875ab 100644
--- a/be/src/runtime/load_stream_writer.cpp
+++ b/be/src/runtime/load_stream_writer.cpp
@@ -59,6 +59,7 @@
 #include "runtime/memory/mem_tracker.h"
 #include "service/backend_options.h"
 #include "util/brpc_client_cache.h"
+#include "util/debug_points.h"
 #include "util/mem_info.h"
 #include "util/ref_count_closure.h"
 #include "util/stopwatch.hpp"
@@ -87,9 +88,7 @@ Status LoadStreamWriter::append_data(uint32_t segid, uint64_t offset, butil::IOB
     io::FileWriter* file_writer = nullptr;
     {
         std::lock_guard lock_guard(_lock);
-        if (!_is_init) {
-            RETURN_IF_ERROR(init());
-        }
+        DCHECK(_is_init);
         if (segid >= _segment_file_writers.size()) {
             for (size_t i = _segment_file_writers.size(); i <= segid; i++) {
                 Status st;
@@ -106,6 +105,7 @@ Status LoadStreamWriter::append_data(uint32_t segid, uint64_t offset, butil::IOB
         // TODO: IOBuf to Slice
         file_writer = _segment_file_writers[segid].get();
     }
+    DBUG_EXECUTE_IF("LoadStreamWriter.append_data.null_file_writer", { file_writer = nullptr; });
     VLOG_DEBUG << " file_writer " << file_writer << "seg id " << segid;
     if (file_writer == nullptr) {
         return Status::Corruption("append_data failed, file writer {} is destoryed", segid);
@@ -122,14 +122,18 @@ Status LoadStreamWriter::close_segment(uint32_t segid) {
     io::FileWriter* file_writer = nullptr;
     {
         std::lock_guard lock_guard(_lock);
+        DBUG_EXECUTE_IF("LoadStreamWriter.close_segment.uninited_writer", { _is_init = false; });
         if (!_is_init) {
             return Status::Corruption("close_segment failed, LoadStreamWriter is not inited");
         }
+        DBUG_EXECUTE_IF("LoadStreamWriter.close_segment.bad_segid",
+                        { segid = _segment_file_writers.size(); });
         if (segid >= _segment_file_writers.size()) {
             return Status::Corruption("close_segment failed, segment {} is never opened", segid);
         }
         file_writer = _segment_file_writers[segid].get();
     }
+    DBUG_EXECUTE_IF("LoadStreamWriter.close_segment.null_file_writer", { file_writer = nullptr; });
     if (file_writer == nullptr) {
         return Status::Corruption("close_segment failed, file writer {} is destoryed", segid);
     }
@@ -151,14 +155,18 @@ Status LoadStreamWriter::add_segment(uint32_t segid, const SegmentStatistics& st
     io::FileWriter* file_writer = nullptr;
     {
         std::lock_guard lock_guard(_lock);
+        DBUG_EXECUTE_IF("LoadStreamWriter.add_segment.uninited_writer", { _is_init = false; });
         if (!_is_init) {
             return Status::Corruption("add_segment failed, LoadStreamWriter is not inited");
         }
+        DBUG_EXECUTE_IF("LoadStreamWriter.add_segment.bad_segid",
+                        { segid = _segment_file_writers.size(); });
         if (segid >= _segment_file_writers.size()) {
             return Status::Corruption("add_segment failed, segment {} is never opened", segid);
         }
         file_writer = _segment_file_writers[segid].get();
     }
+    DBUG_EXECUTE_IF("LoadStreamWriter.add_segment.null_file_writer", { file_writer = nullptr; });
     if (file_writer == nullptr) {
         return Status::Corruption("add_segment failed, file writer {} is destoryed", segid);
     }
@@ -177,6 +185,7 @@ Status LoadStreamWriter::add_segment(uint32_t segid, const SegmentStatistics& st
 
 Status LoadStreamWriter::close() {
     std::lock_guard<std::mutex> l(_lock);
+    DBUG_EXECUTE_IF("LoadStreamWriter.close.uninited_writer", { _is_init = false; });
     if (!_is_init) {
         // if this delta writer is not initialized, but close() is called.
         // which means this tablet has no data loaded, but at least one tablet
diff --git a/be/src/runtime/load_stream_writer.h b/be/src/runtime/load_stream_writer.h
index ab6530bf60df6d..4f35950a93f166 100644
--- a/be/src/runtime/load_stream_writer.h
+++ b/be/src/runtime/load_stream_writer.h
@@ -70,8 +70,6 @@ class LoadStreamWriter {
     // wait for all memtables to be flushed.
     Status close();
 
-    int64_t tablet_id() const { return _req.tablet_id; }
-
 private:
     bool _is_init = false;
     bool _is_canceled = false;
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
new file mode 100644
index 00000000000000..09f271fead5331
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("load_stream_fault_injection", "nonConcurrent") {
+    // init query case data
+    sql """
+        CREATE TABLE IF NOT EXISTS `baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+    sql """
+        CREATE TABLE IF NOT EXISTS `test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    GetDebugPoint().clearDebugPointsForAllBEs()
+    streamLoad {
+        table "baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    def load_with_injection = { injection, expect_errmsg ->
+        try {
+            GetDebugPoint().enableDebugPointForAllBEs(injection)
+            sql "insert into test select * from baseall where k1 <= 3"
+        } catch(Exception e) {
+            // assertTrue(e.getMessage().contains("Process has no memory available"))  // the msg should contain the root cause
+            logger.info(e.getMessage())
+        } finally {
+            GetDebugPoint().disableDebugPointForAllBEs(injection)
+        }
+    }
+
+    // LoadStreamWriter create file failed
+    load_with_injection("LocalFileSystem.create_file_impl.open_file_failed", "")
+    // LoadStreamWriter append_data meet null file writer error
+    load_with_injection("LoadStreamWriter.append_data.null_file_writer", "")
+    // LoadStreamWriter append_data meet bytes_appended and real file size not match error
+    load_with_injection("FileWriter.bytes_appended.zero_bytes_appended", "")
+    // LoadStreamWriter close_segment meet not inited error
+    load_with_injection("LoadStreamWriter.close_segment.uninited_writer", "")
+    // LoadStreamWriter close_segment meet not bad segid error
+    load_with_injection("LoadStreamWriter.close_segment.bad_segid", "")
+    // LoadStreamWriter close_segment meet null file writer error
+    load_with_injection("LoadStreamWriter.close_segment.null_file_writer", "")
+    // LoadStreamWriter close_segment meet file writer failed to close error
+    load_with_injection("LocalFileWriter.close.failed", "")
+    // LoadStreamWriter close_segment meet bytes_appended and real file size not match error
+    load_with_injection("FileWriter.close_segment.zero_bytes_appended", "")
+    // LoadStreamWriter add_segment meet not inited error
+    load_with_injection("LoadStreamWriter.add_segment.uninited_writer", "")
+    // LoadStreamWriter add_segment meet not bad segid error
+    load_with_injection("LoadStreamWriter.add_segment.bad_segid", "")
+    // LoadStreamWriter add_segment meet null file writer error
+    load_with_injection("LoadStreamWriter.add_segment.null_file_writer", "")
+    // LoadStreamWriter add_segment meet bytes_appended and real file size not match error
+    load_with_injection("FileWriter.add_segment.zero_bytes_appended", "")
+    // LoadStreamWriter close meet not inited error
+    load_with_injection("LoadStreamWriter.close.uninited_writer", "")
+    // LoadStream init failed coz LoadStreamWriter init failed
+    load_with_injection("RowsetBuilder.check_tablet_version_count.too_many_version", "")
+    // LoadStream add_segment meet unknown segid in request header
+    load_with_injection("TabletStream.add_segment.unknown_segid", "")
+    // LoadStream append_data meet unknown index id in request header
+    load_with_injection("abletStream.add_segment.unknown_indexid", "")
+    // LoadStream dispatch meet unknown load id
+    load_with_injection("LoadStream._dispatch.unknown_loadid", "")
+    // LoadStream dispatch meet unknown src id
+    load_with_injection("LoadStream._dispatch.unknown_srcid", "")
+}
+

From 6323c17ad540e627fd0ded8dde77d3fabab20f86 Mon Sep 17 00:00:00 2001
From: zclllyybb <zhaochangle@selectdb.com>
Date: Thu, 28 Dec 2023 16:51:48 +0800
Subject: [PATCH 120/299] [fix](test) fix wrong DDL in test pipeline load
 #29211

---
 .../suites/load_p0/stream_load/test_pipeline_load.groovy        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy b/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
index ad0aa89b17daa6..27940760bfdfdc 100644
--- a/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
@@ -128,7 +128,7 @@ suite("test_pipeline_load", "nonConcurrent") {
                 `c47` varchar(48) NULL,
                 `c48` varchar(48) NULL,
                 `c49` varchar(48) NULL,
-                `c50` varchar(48) NULL,
+                `c50` varchar(48) NULL
             ) ENGINE=OLAP
             DUPLICATE KEY(`c1`)
             COMMENT 'OLAP'

From e610044bae59d64894e8482175c1ac49cfdb69a8 Mon Sep 17 00:00:00 2001
From: Luwei <814383175@qq.com>
Date: Thu, 28 Dec 2023 17:11:24 +0800
Subject: [PATCH 121/299] [Enhancement] (schema) add column type check (#28718)

---
 be/src/common/config.cpp                      |  1 +
 be/src/common/config.h                        |  2 ++
 be/src/olap/rowset/segment_v2/column_reader.h |  2 ++
 be/src/olap/rowset/segment_v2/segment.cpp     | 10 ++++++++++
 4 files changed, 15 insertions(+)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 358db99e592b24..42a95302122661 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1144,6 +1144,7 @@ DEFINE_Bool(enable_snapshot_action, "false");
 
 DEFINE_mInt32(variant_max_merged_tablet_schema_size, "2048");
 
+DEFINE_mBool(enable_column_type_check, "true");
 // 128 MB
 DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");
 
diff --git a/be/src/common/config.h b/be/src/common/config.h
index a41a3d06141182..d38e059c41650e 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1223,6 +1223,8 @@ DECLARE_mInt32(variant_max_merged_tablet_schema_size);
 
 DECLARE_mInt64(local_exchange_buffer_mem_limit);
 
+DECLARE_mBool(enable_column_type_check);
+
 #ifdef BE_TEST
 // test s3
 DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 99ff23108083a2..78cd01645335e9 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -182,6 +182,8 @@ class ColumnReader {
 
     void disable_index_meta_cache() { _use_index_page_cache = false; }
 
+    FieldType get_meta_type() { return _meta_type; }
+
 private:
     ColumnReader(const ColumnReaderOptions& opts, const ColumnMetaPB& meta, uint64_t num_rows,
                  io::FileReaderSPtr file_reader);
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index 1f9956e5e692fe..b7d3a4ca0f961e 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -524,6 +524,16 @@ Status Segment::new_column_iterator(const TabletColumn& tablet_column,
     ColumnIterator* it;
     RETURN_IF_ERROR(_column_readers.at(tablet_column.unique_id())->new_iterator(&it));
     iter->reset(it);
+
+    if (config::enable_column_type_check &&
+        tablet_column.type() != _column_readers.at(tablet_column.unique_id())->get_meta_type()) {
+        LOG(WARNING) << "different type between schema and column reader,"
+                     << " column schema name: " << tablet_column.name()
+                     << " column schema type: " << int(tablet_column.type())
+                     << " column reader meta type"
+                     << int(_column_readers.at(tablet_column.unique_id())->get_meta_type());
+        return Status::InternalError("different type between schema and column reader");
+    }
     return Status::OK();
 }
 

From a14daca7bacd02122d73078ada4ff2677d6d0351 Mon Sep 17 00:00:00 2001
From: qiye <jianliang5669@gmail.com>
Date: Thu, 28 Dec 2023 17:18:59 +0800
Subject: [PATCH 122/299] [feature](inverted index)write separated index files
 in RAM directory to reduce IO(#28810)

Normally we write the separate index files to disk before we merge the index files into an idx compound file.
In high-frequency load scenarios, disk IO can become a bottleneck.
In order to reduce the pressure on the disk, we write the standalone index file to the RAM directory for the first time, and then write it to the disk when merging it into a composite file.

Add config `index_inverted_index_by_ram_dir_enable`, default is `false`.
---
 be/src/clucene                                |   2 +-
 be/src/common/config.cpp                      |   2 +
 be/src/common/config.h                        |   2 +
 be/src/index-tools/index_tool.cpp             |   8 +-
 be/src/olap/compaction.cpp                    |   3 +-
 .../segment_v2/inverted_index_cache.cpp       |  10 +-
 .../segment_v2/inverted_index_compaction.cpp  |  18 +-
 .../inverted_index_compound_directory.cpp     | 315 +++++++++++++-----
 .../inverted_index_compound_directory.h       | 123 +++++--
 .../segment_v2/inverted_index_reader.cpp      |   2 +-
 .../segment_v2/inverted_index_writer.cpp      |  12 +-
 11 files changed, 358 insertions(+), 139 deletions(-)

diff --git a/be/src/clucene b/be/src/clucene
index d95d6be91ecd4e..df3ab39ca636e5 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit d95d6be91ecd4e471306caa57b580ba548605962
+Subproject commit df3ab39ca636e58ed1ee640921444f7ef6d6438d
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 42a95302122661..b5a16ef0166af7 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1013,6 +1013,8 @@ DEFINE_Int32(inverted_index_read_buffer_size, "4096");
 DEFINE_Int32(max_depth_in_bkd_tree, "32");
 // index compaction
 DEFINE_Bool(inverted_index_compaction_enable, "false");
+// index by RAM directory
+DEFINE_mBool(inverted_index_ram_dir_enable, "false");
 // use num_broadcast_buffer blocks as buffer to do broadcast
 DEFINE_Int32(num_broadcast_buffer, "32");
 
diff --git a/be/src/common/config.h b/be/src/common/config.h
index d38e059c41650e..53dc0a2a0d8bc9 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1050,6 +1050,8 @@ DECLARE_Int32(inverted_index_read_buffer_size);
 DECLARE_Int32(max_depth_in_bkd_tree);
 // index compaction
 DECLARE_Bool(inverted_index_compaction_enable);
+// index by RAM directory
+DECLARE_mBool(inverted_index_ram_dir_enable);
 // use num_broadcast_buffer blocks as buffer to do broadcast
 DECLARE_Int32(num_broadcast_buffer);
 
diff --git a/be/src/index-tools/index_tool.cpp b/be/src/index-tools/index_tool.cpp
index c356449175994f..cb84e2b9846e72 100644
--- a/be/src/index-tools/index_tool.cpp
+++ b/be/src/index-tools/index_tool.cpp
@@ -33,7 +33,7 @@
 #include "olap/rowset/segment_v2/inverted_index_compound_reader.h"
 
 using doris::segment_v2::DorisCompoundReader;
-using doris::segment_v2::DorisCompoundDirectory;
+using doris::segment_v2::DorisCompoundDirectoryFactory;
 using doris::io::FileInfo;
 using namespace lucene::analysis;
 using namespace lucene::index;
@@ -186,7 +186,7 @@ int main(int argc, char** argv) {
         auto fs = doris::io::global_local_filesystem();
         try {
             lucene::store::Directory* dir =
-                    DorisCompoundDirectory::getDirectory(fs, dir_str.c_str());
+                    DorisCompoundDirectoryFactory::getDirectory(fs, dir_str.c_str());
             auto reader = new DorisCompoundReader(dir, file_str.c_str(), 4096);
             std::vector<std::string> files;
             std::cout << "Nested files for " << file_str << std::endl;
@@ -209,7 +209,7 @@ int main(int argc, char** argv) {
         auto fs = doris::io::global_local_filesystem();
         try {
             lucene::store::Directory* dir =
-                    DorisCompoundDirectory::getDirectory(fs, dir_str.c_str());
+                    DorisCompoundDirectoryFactory::getDirectory(fs, dir_str.c_str());
             auto reader = new DorisCompoundReader(dir, file_str.c_str(), 4096);
             std::cout << "Term statistics for " << file_str << std::endl;
             std::cout << "==================================" << std::endl;
@@ -226,7 +226,7 @@ int main(int argc, char** argv) {
         auto fs = doris::io::global_local_filesystem();
         try {
             lucene::store::Directory* dir =
-                    DorisCompoundDirectory::getDirectory(fs, FLAGS_directory.c_str());
+                    DorisCompoundDirectoryFactory::getDirectory(fs, FLAGS_directory.c_str());
             if (FLAGS_idx_file_name == "") {
                 //try to search from directory's all files
                 std::vector<FileInfo> files;
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index a6ad5f6ee529c8..4ad78367d28014 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -603,7 +603,8 @@ Status Compaction::construct_output_rowset_writer(RowsetWriterContext& ctx, bool
                                 std::string dir_str = p.parent_path().string();
                                 std::string file_str = p.filename().string();
                                 lucene::store::Directory* dir =
-                                        DorisCompoundDirectory::getDirectory(fs, dir_str.c_str());
+                                        DorisCompoundDirectoryFactory::getDirectory(
+                                                fs, dir_str.c_str());
                                 DorisCompoundReader reader(dir, file_str.c_str());
                                 std::vector<std::string> files;
                                 reader.list(&files);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
index 0440f6865a4a53..e88573c1be84ac 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
@@ -178,8 +178,8 @@ Status InvertedIndexSearcherCache::get_index_searcher(
         // During the process of opening the index, write the file information read to the idx file cache.
         bool open_idx_file_cache = true;
         auto* directory = new DorisCompoundReader(
-                DorisCompoundDirectory::getDirectory(fs, index_dir.c_str()), file_name.c_str(),
-                config::inverted_index_read_buffer_size, open_idx_file_cache);
+                DorisCompoundDirectoryFactory::getDirectory(fs, index_dir.c_str()),
+                file_name.c_str(), config::inverted_index_read_buffer_size, open_idx_file_cache);
         auto null_bitmap_file_name = InvertedIndexDescriptor::get_temporary_null_bitmap_file_name();
         if (!directory->fileExists(null_bitmap_file_name.c_str())) {
             has_null = false;
@@ -261,9 +261,9 @@ Status InvertedIndexSearcherCache::insert(const io::FileSystemSPtr& fs,
             return Status::Error<ErrorCode::INVERTED_INDEX_NOT_SUPPORTED>(
                     "InvertedIndexSearcherCache do not support reader type.");
         }
-        auto* directory =
-                new DorisCompoundReader(DorisCompoundDirectory::getDirectory(fs, index_dir.c_str()),
-                                        file_name.c_str(), config::inverted_index_read_buffer_size);
+        auto* directory = new DorisCompoundReader(
+                DorisCompoundDirectoryFactory::getDirectory(fs, index_dir.c_str()),
+                file_name.c_str(), config::inverted_index_read_buffer_size);
         OptionalIndexSearcherPtr result;
         RETURN_IF_ERROR(builder->build(directory, result));
         if (!result.has_value()) {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
index ca9d1c8f9a4d1c..677d359f1a4172 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
@@ -22,8 +22,7 @@
 #include "inverted_index_compound_directory.h"
 #include "inverted_index_compound_reader.h"
 
-namespace doris {
-namespace segment_v2 {
+namespace doris::segment_v2 {
 Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_num,
                       std::vector<std::string> src_index_files,
                       std::vector<std::string> dest_index_files, const io::FileSystemSPtr& fs,
@@ -31,7 +30,7 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu
                       std::vector<std::vector<std::pair<uint32_t, uint32_t>>> trans_vec,
                       std::vector<uint32_t> dest_segment_num_rows) {
     lucene::store::Directory* dir =
-            DorisCompoundDirectory::getDirectory(fs, index_writer_path.c_str(), false);
+            DorisCompoundDirectoryFactory::getDirectory(fs, index_writer_path.c_str());
     lucene::analysis::SimpleAnalyzer<char> analyzer;
     auto index_writer = _CLNEW lucene::index::IndexWriter(dir, &analyzer, true /* create */,
                                                           true /* closeDirOnShutdown */);
@@ -42,8 +41,8 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu
         // format: rowsetId_segmentId_indexId.idx
         std::string src_idx_full_name =
                 src_index_files[i] + "_" + std::to_string(index_id) + ".idx";
-        DorisCompoundReader* reader = new DorisCompoundReader(
-                DorisCompoundDirectory::getDirectory(fs, tablet_path.c_str()),
+        auto* reader = new DorisCompoundReader(
+                DorisCompoundDirectoryFactory::getDirectory(fs, tablet_path.c_str()),
                 src_idx_full_name.c_str());
         src_index_dirs[i] = reader;
     }
@@ -53,7 +52,7 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu
     for (int i = 0; i < dest_segment_num; ++i) {
         // format: rowsetId_segmentId_columnId
         auto path = tablet_path + "/" + dest_index_files[i] + "_" + std::to_string(index_id);
-        dest_index_dirs[i] = DorisCompoundDirectory::getDirectory(fs, path.c_str(), true);
+        dest_index_dirs[i] = DorisCompoundDirectoryFactory::getDirectory(fs, path.c_str(), true);
     }
 
     index_writer->indexCompaction(src_index_dirs, dest_index_dirs, trans_vec,
@@ -65,13 +64,13 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu
     // when index_writer is destroyed, if closeDir is set, dir will be close
     // _CLDECDELETE(dir) will try to ref_cnt--, when it decreases to 1, dir will be destroyed.
     _CLDECDELETE(dir)
-    for (auto d : src_index_dirs) {
+    for (auto* d : src_index_dirs) {
         if (d != nullptr) {
             d->close();
             _CLDELETE(d);
         }
     }
-    for (auto d : dest_index_dirs) {
+    for (auto* d : dest_index_dirs) {
         if (d != nullptr) {
             // NOTE: DO NOT close dest dir here, because it will be closed when dest index writer finalize.
             //d->close();
@@ -83,5 +82,4 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu
     static_cast<void>(fs->delete_directory(index_writer_path.c_str()));
     return Status::OK();
 }
-} // namespace segment_v2
-} // namespace doris
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
index 6b1cacee454350..d43afd4fc5e055 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.cpp
@@ -18,10 +18,10 @@
 #include "olap/rowset/segment_v2/inverted_index_compound_directory.h"
 
 #include "CLucene/SharedHeader.h"
+#include "CLucene/_SharedHeader.h"
 #include "common/status.h"
 #include "io/fs/file_reader.h"
 #include "io/fs/file_writer.h"
-#include "io/fs/path.h"
 #include "util/debug_points.h"
 #include "util/slice.h"
 
@@ -47,17 +47,14 @@
 #include <CLucene/store/RAMDirectory.h>
 #include <CLucene/util/Misc.h>
 #include <assert.h>
-// IWYU pragma: no_include <bthread/errno.h>
 #include <errno.h> // IWYU pragma: keep
 #include <glog/logging.h>
 #include <stdio.h>
 #include <string.h>
 #include <wchar.h>
 
-#include <algorithm>
 #include <filesystem>
 #include <iostream>
-#include <memory>
 #include <mutex>
 #include <utility>
 
@@ -76,15 +73,12 @@
         LOG(WARNING) << err;                                     \
         _CLTHROWA(CL_ERR_IO, err.c_str());                       \
     }
-namespace doris {
-namespace segment_v2 {
+namespace doris::segment_v2 {
 
 const char* WRITE_LOCK_FILE = "write.lock";
 const char* COMPOUND_FILE_EXTENSION = ".idx";
 const int64_t MAX_HEADER_DATA_SIZE = 1024 * 128; // 128k
 
-bool DorisCompoundDirectory::disableLocks = false;
-
 DorisCompoundFileWriter::DorisCompoundFileWriter(CL_NS(store)::Directory* dir) {
     if (dir == nullptr) {
         _CLTHROWA(CL_ERR_NullPointer, "directory cannot be null");
@@ -127,7 +121,7 @@ void DorisCompoundFileWriter::writeCompoundFile() {
     for (auto file : files) {
         FileInfo file_info;
         file_info.filename = file;
-        file_info.filesize = ((DorisCompoundDirectory*)directory)->fileLength(file.c_str());
+        file_info.filesize = directory->fileLength(file.c_str());
         sorted_files.emplace_back(std::move(file_info));
     }
     sort_files(sorted_files);
@@ -139,7 +133,7 @@ void DorisCompoundFileWriter::writeCompoundFile() {
     std::string idx_name = std::string(cfs_path.stem().c_str()) + COMPOUND_FILE_EXTENSION;
     // write file entries to ram directory to get header length
     lucene::store::RAMDirectory ram_dir;
-    auto out_idx = ram_dir.createOutput(idx_name.c_str());
+    auto* out_idx = ram_dir.createOutput(idx_name.c_str());
     if (out_idx == nullptr) {
         LOG(WARNING) << "Write compound file error: RAMDirectory output is nullptr.";
         _CLTHROWA(CL_ERR_IO, "Create RAMDirectory output error");
@@ -169,9 +163,9 @@ void DorisCompoundFileWriter::writeCompoundFile() {
     ram_dir.close();
 
     auto compound_fs = ((DorisCompoundDirectory*)directory)->getCompoundFileSystem();
-    auto out_dir = DorisCompoundDirectory::getDirectory(compound_fs, idx_path.c_str(), false);
+    auto* out_dir = DorisCompoundDirectoryFactory::getDirectory(compound_fs, idx_path.c_str());
 
-    auto out = out_dir->createOutput(idx_name.c_str());
+    auto* out = out_dir->createOutput(idx_name.c_str());
     if (out == nullptr) {
         LOG(WARNING) << "Write compound file error: CompoundDirectory output is nullptr.";
         _CLTHROWA(CL_ERR_IO, "Create CompoundDirectory output error");
@@ -277,7 +271,7 @@ bool DorisCompoundDirectory::FSIndexInput::open(const io::FileSystemSPtr& fs, co
     if (buffer_size == -1) {
         buffer_size = CL_NS(store)::BufferedIndexOutput::BUFFER_SIZE;
     }
-    SharedHandle* h = _CLNEW SharedHandle(path);
+    auto* h = _CLNEW SharedHandle(path);
 
     io::FileReaderOptions reader_options;
     reader_options.cache_type = config::enable_file_cache ? io::FileCachePolicy::FILE_BLOCK_CACHE
@@ -519,10 +513,12 @@ DorisCompoundDirectory::DorisCompoundDirectory() {
 }
 
 void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _path,
+                                  bool use_compound_file_writer,
                                   lucene::store::LockFactory* lock_factory,
                                   const io::FileSystemSPtr& cfs, const char* cfs_path) {
     fs = _fs;
     directory = _path;
+    useCompoundFileWriter = use_compound_file_writer;
 
     if (cfs == nullptr) {
         compound_fs = fs;
@@ -534,17 +530,12 @@ void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _pa
     } else {
         cfs_directory = _path;
     }
-    bool doClearLockID = false;
 
     if (lock_factory == nullptr) {
         lock_factory = _CLNEW lucene::store::NoLockFactory();
     }
 
-    setLockFactory(lock_factory);
-
-    if (doClearLockID) {
-        lockFactory->setLockPrefix(nullptr);
-    }
+    lucene::store::Directory::setLockFactory(lock_factory);
 
     // It's fail checking directory existence in S3.
     if (fs->type() == io::FileSystemType::S3) {
@@ -559,24 +550,6 @@ void DorisCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _pa
     }
 }
 
-void DorisCompoundDirectory::create() {
-    std::lock_guard<std::mutex> wlock(_this_lock);
-
-    //clear old files
-    std::vector<std::string> files;
-    lucene::util::Misc::listFiles(directory.c_str(), files, false);
-    std::vector<std::string>::iterator itr = files.begin();
-    while (itr != files.end()) {
-        if (CL_NS(index)::IndexReader::isLuceneFile(itr->c_str())) {
-            if (unlink((directory + PATH_DELIMITERA + *itr).c_str()) == -1) {
-                _CLTHROWA(CL_ERR_IO, "Couldn't delete file ");
-            }
-        }
-        itr++;
-    }
-    lockFactory->clearLock(CL_NS(index)::IndexWriter::WRITE_LOCK_NAME);
-}
-
 void DorisCompoundDirectory::priv_getFN(char* buffer, const char* name) const {
     buffer[0] = 0;
     strcpy(buffer, directory.c_str());
@@ -619,45 +592,6 @@ const char* DorisCompoundDirectory::getCfsDirName() const {
     return cfs_directory.c_str();
 }
 
-DorisCompoundDirectory* DorisCompoundDirectory::getDirectory(const io::FileSystemSPtr& fs,
-                                                             const char* file,
-                                                             bool use_compound_file_writer,
-                                                             const io::FileSystemSPtr& cfs_fs,
-                                                             const char* cfs_file) {
-    DorisCompoundDirectory* dir =
-            getDirectory(fs, file, (lucene::store::LockFactory*)nullptr, cfs_fs, cfs_file);
-    dir->useCompoundFileWriter = use_compound_file_writer;
-    return dir;
-}
-
-//static
-DorisCompoundDirectory* DorisCompoundDirectory::getDirectory(
-        const io::FileSystemSPtr& _fs, const char* _file, lucene::store::LockFactory* lock_factory,
-        const io::FileSystemSPtr& _cfs, const char* _cfs_file) {
-    const char* cfs_file = _cfs_file;
-    if (cfs_file == nullptr) {
-        cfs_file = _file;
-    }
-    DorisCompoundDirectory* dir = nullptr;
-    if (!_file || !*_file) {
-        _CLTHROWA(CL_ERR_IO, "Invalid directory");
-    }
-
-    const char* file = _file;
-
-    bool exists = false;
-    LOG_AND_THROW_IF_ERROR(_fs->exists(file, &exists), "Get directory exists IO error")
-    if (!exists) {
-        LOG_AND_THROW_IF_ERROR(_fs->create_directory(file),
-                               "Get directory create directory IO error")
-    }
-
-    dir = _CLNEW DorisCompoundDirectory();
-    dir->init(_fs, file, lock_factory, _cfs, cfs_file);
-
-    return dir;
-}
-
 int64_t DorisCompoundDirectory::fileModified(const char* name) const {
     CND_PRECONDITION(directory[0] != 0, "directory is not open");
     struct stat buf;
@@ -698,7 +632,7 @@ bool DorisCompoundDirectory::openInput(const char* name, lucene::store::IndexInp
 
 void DorisCompoundDirectory::close() {
     if (useCompoundFileWriter) {
-        DorisCompoundFileWriter* cfsWriter = _CLNEW DorisCompoundFileWriter(this);
+        auto* cfsWriter = _CLNEW DorisCompoundFileWriter(this);
         // write compound file
         cfsWriter->writeCompoundFile();
         // delete index path, which contains separated inverted index files
@@ -754,7 +688,7 @@ lucene::store::IndexOutput* DorisCompoundDirectory::createOutput(const char* nam
         LOG_AND_THROW_IF_ERROR(fs->exists(fl, &exists), "Create output file exists IO error")
         assert(!exists);
     }
-    auto ret = _CLNEW FSIndexOutput();
+    auto* ret = _CLNEW FSIndexOutput();
     try {
         ret->init(fs, fl);
     } catch (CLuceneError& err) {
@@ -768,5 +702,226 @@ std::string DorisCompoundDirectory::toString() const {
     return std::string("DorisCompoundDirectory@") + this->directory;
 }
 
-} // namespace segment_v2
-} // namespace doris
+/**
+ * DorisRAMCompoundDirectory
+ */
+DorisRAMCompoundDirectory::DorisRAMCompoundDirectory() {
+    filesMap = _CLNEW FileMap(true, true);
+    this->sizeInBytes = 0;
+}
+
+DorisRAMCompoundDirectory::~DorisRAMCompoundDirectory() {
+    _CLDELETE(lockFactory);
+    _CLDELETE(filesMap);
+}
+
+void DorisRAMCompoundDirectory::init(const io::FileSystemSPtr& _fs, const char* _path,
+                                     bool use_compound_file_writer,
+                                     lucene::store::LockFactory* lock_factory,
+                                     const io::FileSystemSPtr& cfs, const char* cfs_path) {
+    fs = _fs;
+    directory = _path;
+    useCompoundFileWriter = use_compound_file_writer;
+
+    if (cfs == nullptr) {
+        compound_fs = fs;
+    } else {
+        compound_fs = cfs;
+    }
+    if (cfs_path != nullptr) {
+        cfs_directory = cfs_path;
+    } else {
+        cfs_directory = _path;
+    }
+
+    lucene::store::Directory::setLockFactory(_CLNEW lucene::store::SingleInstanceLockFactory());
+}
+
+bool DorisRAMCompoundDirectory::list(std::vector<std::string>* names) const {
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    auto itr = filesMap->begin();
+    while (itr != filesMap->end()) {
+        names->emplace_back(itr->first);
+        ++itr;
+    }
+    return true;
+}
+
+bool DorisRAMCompoundDirectory::fileExists(const char* name) const {
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    return filesMap->exists((char*)name);
+}
+
+int64_t DorisRAMCompoundDirectory::fileModified(const char* name) const {
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    auto* f = filesMap->get((char*)name);
+    return f->getLastModified();
+}
+
+void DorisRAMCompoundDirectory::touchFile(const char* name) {
+    lucene::store::RAMFile* file = nullptr;
+    {
+        std::lock_guard<std::mutex> wlock(_this_lock);
+        file = filesMap->get((char*)name);
+    }
+    const uint64_t ts1 = file->getLastModified();
+    uint64_t ts2 = lucene::util::Misc::currentTimeMillis();
+
+    //make sure that the time has actually changed
+    while (ts1 == ts2) {
+        _LUCENE_SLEEP(1);
+        ts2 = lucene::util::Misc::currentTimeMillis();
+    };
+
+    file->setLastModified(ts2);
+}
+
+int64_t DorisRAMCompoundDirectory::fileLength(const char* name) const {
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    auto* f = filesMap->get((char*)name);
+    return f->getLength();
+}
+
+bool DorisRAMCompoundDirectory::openInput(const char* name, lucene::store::IndexInput*& ret,
+                                          CLuceneError& error, int32_t bufferSize) {
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    auto* file = filesMap->get((char*)name);
+    if (file == nullptr) {
+        error.set(CL_ERR_IO,
+                  "[DorisRAMCompoundDirectory::open] The requested file does not exist.");
+        return false;
+    }
+    ret = _CLNEW lucene::store::RAMInputStream(file);
+    return true;
+}
+
+void DorisRAMCompoundDirectory::close() {
+    // write compound file
+    DorisCompoundDirectory::close();
+
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    filesMap->clear();
+    _CLDELETE(filesMap);
+}
+
+bool DorisRAMCompoundDirectory::doDeleteFile(const char* name) {
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    auto itr = filesMap->find((char*)name);
+    if (itr != filesMap->end()) {
+        SCOPED_LOCK_MUTEX(this->THIS_LOCK);
+        sizeInBytes -= itr->second->sizeInBytes;
+        filesMap->removeitr(itr);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+bool DorisRAMCompoundDirectory::deleteDirectory() {
+    // do nothing, RAM dir do not have actual files
+    return true;
+}
+
+void DorisRAMCompoundDirectory::renameFile(const char* from, const char* to) {
+    std::lock_guard<std::mutex> wlock(_this_lock);
+    auto itr = filesMap->find((char*)from);
+
+    /* DSR:CL_BUG_LEAK:
+    ** If a file named $to already existed, its old value was leaked.
+    ** My inclination would be to prevent this implicit deletion with an
+    ** exception, but it happens routinely in CLucene's internals (e.g., during
+    ** IndexWriter.addIndexes with the file named 'segments'). */
+    if (filesMap->exists((char*)to)) {
+        auto itr1 = filesMap->find((char*)to);
+        SCOPED_LOCK_MUTEX(this->THIS_LOCK);
+        sizeInBytes -= itr1->second->sizeInBytes;
+        filesMap->removeitr(itr1);
+    }
+    if (itr == filesMap->end()) {
+        char tmp[1024];
+        snprintf(tmp, 1024, "cannot rename %s, file does not exist", from);
+        _CLTHROWT(CL_ERR_IO, tmp);
+    }
+    DCHECK(itr != filesMap->end());
+    auto* file = itr->second;
+    filesMap->removeitr(itr, false, true);
+    filesMap->put(strdup(to), file);
+}
+
+lucene::store::IndexOutput* DorisRAMCompoundDirectory::createOutput(const char* name) {
+    /* Check the $filesMap VoidMap to see if there was a previous file named
+    ** $name.  If so, delete the old RAMFile object, but reuse the existing
+    ** char buffer ($n) that holds the filename.  If not, duplicate the
+    ** supplied filename buffer ($name) and pass ownership of that memory ($n)
+    ** to $files. */
+    std::lock_guard<std::mutex> wlock(_this_lock);
+
+    // get the actual pointer to the output name
+    char* n = nullptr;
+    auto itr = filesMap->find(const_cast<char*>(name));
+    if (itr != filesMap->end()) {
+        n = itr->first;
+        lucene::store::RAMFile* rf = itr->second;
+        SCOPED_LOCK_MUTEX(this->THIS_LOCK);
+        sizeInBytes -= rf->sizeInBytes;
+        _CLDELETE(rf);
+    } else {
+        n = STRDUP_AtoA(name);
+    }
+
+    auto* file = _CLNEW lucene::store::RAMFile();
+    (*filesMap)[n] = file;
+
+    return _CLNEW lucene::store::RAMOutputStream(file);
+}
+
+std::string DorisRAMCompoundDirectory::toString() const {
+    return std::string("DorisRAMCompoundDirectory@") + this->directory;
+}
+
+const char* DorisRAMCompoundDirectory::getClassName() {
+    return "DorisRAMCompoundDirectory";
+}
+
+const char* DorisRAMCompoundDirectory::getObjectName() const {
+    return getClassName();
+}
+
+/**
+ * DorisCompoundDirectoryFactory
+ */
+DorisCompoundDirectory* DorisCompoundDirectoryFactory::getDirectory(
+        const io::FileSystemSPtr& _fs, const char* _file, bool use_compound_file_writer,
+        bool can_use_ram_dir, lucene::store::LockFactory* lock_factory,
+        const io::FileSystemSPtr& _cfs, const char* _cfs_file) {
+    const char* cfs_file = _cfs_file;
+    if (cfs_file == nullptr) {
+        cfs_file = _file;
+    }
+    DorisCompoundDirectory* dir = nullptr;
+    if (!_file || !*_file) {
+        _CLTHROWA(CL_ERR_IO, "Invalid directory");
+    }
+
+    const char* file = _file;
+
+    // Write by RAM directory
+    // 1. only write separated index files, which is can_use_ram_dir = true.
+    // 2. config::inverted_index_ram_dir_enable = true
+    if (config::inverted_index_ram_dir_enable && can_use_ram_dir) {
+        dir = _CLNEW DorisRAMCompoundDirectory();
+    } else {
+        bool exists = false;
+        LOG_AND_THROW_IF_ERROR(_fs->exists(file, &exists), "Get directory exists IO error")
+        if (!exists) {
+            LOG_AND_THROW_IF_ERROR(_fs->create_directory(file),
+                                   "Get directory create directory IO error")
+        }
+        dir = _CLNEW DorisCompoundDirectory();
+    }
+    dir->init(_fs, file, use_compound_file_writer, lock_factory, _cfs, cfs_file);
+
+    return dir;
+}
+
+} // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h
index 9b88f46645c18f..a6def0fd2f0a4e 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compound_directory.h
@@ -21,6 +21,7 @@
 #include <CLucene/store/Directory.h>
 #include <CLucene/store/IndexInput.h>
 #include <CLucene/store/IndexOutput.h>
+#include <CLucene/store/_RAMDirectory.h>
 #include <stdint.h>
 
 #include <string>
@@ -33,15 +34,11 @@
 
 class CLuceneError;
 
-namespace lucene {
-namespace store {
+namespace lucene::store {
 class LockFactory;
-} // namespace store
-} // namespace lucene
+} // namespace lucene::store
 
-namespace doris {
-
-namespace segment_v2 {
+namespace doris::segment_v2 {
 
 class DorisCompoundFileWriter : LUCENE_BASE {
 public:
@@ -69,26 +66,15 @@ class CLUCENE_EXPORT DorisCompoundDirectory : public lucene::store::Directory {
 private:
     int filemode;
 
-    std::mutex _this_lock;
-
 protected:
-    DorisCompoundDirectory();
-    virtual void init(const io::FileSystemSPtr& fs, const char* path,
-                      lucene::store::LockFactory* lock_factory = nullptr,
-                      const io::FileSystemSPtr& compound_fs = nullptr,
-                      const char* cfs_path = nullptr);
-    void priv_getFN(char* buffer, const char* name) const;
-
-private:
+    mutable std::mutex _this_lock;
     io::FileSystemSPtr fs;
     io::FileSystemSPtr compound_fs;
     std::string directory;
     std::string cfs_directory;
-    void create();
-    static bool disableLocks;
     bool useCompoundFileWriter {false};
 
-protected:
+    void priv_getFN(char* buffer, const char* name) const;
     /// Removes an existing file in the directory.
     bool doDeleteFile(const char* name) override;
 
@@ -106,16 +92,6 @@ class CLUCENE_EXPORT DorisCompoundDirectory : public lucene::store::Directory {
     bool list(std::vector<std::string>* names) const override;
     bool fileExists(const char* name) const override;
     const char* getCfsDirName() const;
-    static DorisCompoundDirectory* getDirectory(const io::FileSystemSPtr& fs, const char* file,
-                                                lucene::store::LockFactory* lock_factory = nullptr,
-                                                const io::FileSystemSPtr& cfs_fs = nullptr,
-                                                const char* cfs_file = nullptr);
-
-    static DorisCompoundDirectory* getDirectory(const io::FileSystemSPtr& fs, const char* file,
-                                                bool use_compound_file_writer,
-                                                const io::FileSystemSPtr& cfs_fs = nullptr,
-                                                const char* cfs_file = nullptr);
-
     int64_t fileModified(const char* name) const override;
     int64_t fileLength(const char* name) const override;
     bool openInput(const char* name, lucene::store::IndexInput*& ret, CLuceneError& err,
@@ -127,7 +103,77 @@ class CLUCENE_EXPORT DorisCompoundDirectory : public lucene::store::Directory {
     std::string toString() const override;
     static const char* getClassName();
     const char* getObjectName() const override;
-    bool deleteDirectory();
+    virtual bool deleteDirectory();
+
+    DorisCompoundDirectory();
+
+    virtual void init(const io::FileSystemSPtr& fs, const char* path, bool use_compound_file_writer,
+                      lucene::store::LockFactory* lock_factory = nullptr,
+                      const io::FileSystemSPtr& compound_fs = nullptr,
+                      const char* cfs_path = nullptr);
+};
+
+class CLUCENE_EXPORT DorisRAMCompoundDirectory : public DorisCompoundDirectory {
+protected:
+    using FileMap =
+            lucene::util::CLHashMap<char*, lucene::store::RAMFile*, lucene::util::Compare::Char,
+                                    lucene::util::Equals::Char, lucene::util::Deletor::acArray,
+                                    lucene::util::Deletor::Object<lucene::store::RAMFile>>;
+
+    // unlike the java Hashtable, FileMap is not synchronized, and all access must be protected by a lock
+    FileMap* filesMap;
+    void init(const io::FileSystemSPtr& fs, const char* path, bool use_compound_file_writer,
+              lucene::store::LockFactory* lock_factory = nullptr,
+              const io::FileSystemSPtr& compound_fs = nullptr,
+              const char* cfs_path = nullptr) override;
+
+public:
+    int64_t sizeInBytes;
+
+    /// Returns a null terminated array of strings, one for each file in the directory.
+    bool list(std::vector<std::string>* names) const override;
+
+    /** Constructs an empty {@link Directory}. */
+    DorisRAMCompoundDirectory();
+
+    ///Destructor - only call this if you are sure the directory
+    ///is not being used anymore. Otherwise use the ref-counting
+    ///facilities of dir->close
+    ~DorisRAMCompoundDirectory() override;
+
+    bool doDeleteFile(const char* name) override;
+
+    bool deleteDirectory() override;
+
+    /// Returns true iff the named file exists in this directory.
+    bool fileExists(const char* name) const override;
+
+    /// Returns the time the named file was last modified.
+    int64_t fileModified(const char* name) const override;
+
+    /// Returns the length in bytes of a file in the directory.
+    int64_t fileLength(const char* name) const override;
+
+    /// Removes an existing file in the directory.
+    void renameFile(const char* from, const char* to) override;
+
+    /** Set the modified time of an existing file to now. */
+    void touchFile(const char* name) override;
+
+    /// Creates a new, empty file in the directory with the given name.
+    ///	Returns a stream writing this file.
+    lucene::store::IndexOutput* createOutput(const char* name) override;
+
+    /// Returns a stream reading an existing file.
+    bool openInput(const char* name, lucene::store::IndexInput*& ret, CLuceneError& error,
+                   int32_t bufferSize = -1) override;
+
+    void close() override;
+
+    std::string toString() const override;
+
+    static const char* getClassName();
+    const char* getObjectName() const override;
 };
 
 class DorisCompoundDirectory::FSIndexInput : public lucene::store::BufferedIndexInput {
@@ -180,5 +226,16 @@ class DorisCompoundDirectory::FSIndexInput : public lucene::store::BufferedIndex
     void readInternal(uint8_t* b, const int32_t len) override;
 };
 
-} // namespace segment_v2
-} // namespace doris
+/**
+ * Factory function to create DorisCompoundDirectory
+ */
+class DorisCompoundDirectoryFactory {
+public:
+    static DorisCompoundDirectory* getDirectory(const io::FileSystemSPtr& fs, const char* file,
+                                                bool use_compound_file_writer = false,
+                                                bool can_use_ram_dir = false,
+                                                lucene::store::LockFactory* lock_factory = nullptr,
+                                                const io::FileSystemSPtr& cfs_fs = nullptr,
+                                                const char* cfs_file = nullptr);
+};
+} // namespace doris::segment_v2
\ No newline at end of file
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
index 6928a976fa8e7b..b8406156f82059 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
@@ -187,7 +187,7 @@ Status InvertedIndexReader::read_null_bitmap(InvertedIndexQueryCacheHandle* cach
 
         if (!dir) {
             dir = new DorisCompoundReader(
-                    DorisCompoundDirectory::getDirectory(_fs, index_dir.c_str()),
+                    DorisCompoundDirectoryFactory::getDirectory(_fs, index_dir.c_str()),
                     index_file_name.c_str(), config::inverted_index_read_buffer_size);
             owned_dir = true;
         }
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index 13a31e24768fc6..a43049aaa2bc34 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -169,7 +169,8 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
     }
 
     Status create_index_directory(std::unique_ptr<DorisCompoundDirectory>& dir) {
-        bool create = true;
+        bool use_compound_file_writer = true;
+        bool can_use_ram_dir = true;
         auto index_path = InvertedIndexDescriptor::get_temporary_index_path(
                 _directory + "/" + _segment_file_name, _index_meta->index_id(),
                 _index_meta->get_index_suffix());
@@ -185,8 +186,8 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
             return Status::InternalError("init_fulltext_index directory already exists");
         }
 
-        dir = std::unique_ptr<DorisCompoundDirectory>(
-                DorisCompoundDirectory::getDirectory(_fs, index_path.c_str(), create));
+        dir = std::unique_ptr<DorisCompoundDirectory>(DorisCompoundDirectoryFactory::getDirectory(
+                _fs, index_path.c_str(), use_compound_file_writer, can_use_ram_dir));
         return Status::OK();
     }
 
@@ -536,7 +537,10 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
                 auto index_path = InvertedIndexDescriptor::get_temporary_index_path(
                         _directory + "/" + _segment_file_name, _index_meta->index_id(),
                         _index_meta->get_index_suffix());
-                dir = DorisCompoundDirectory::getDirectory(_fs, index_path.c_str(), true);
+                bool use_compound_file_writer = true;
+                bool can_use_ram_dir = true;
+                dir = DorisCompoundDirectoryFactory::getDirectory(
+                        _fs, index_path.c_str(), use_compound_file_writer, can_use_ram_dir);
                 write_null_bitmap(null_bitmap_out, dir);
                 _bkd_writer->max_doc_ = _rid;
                 _bkd_writer->docs_seen_ = _row_ids_seen_for_bkd;

From c8a0d3e03ccd13af5f18fa2af1a2fb41ecae2790 Mon Sep 17 00:00:00 2001
From: zzzxl <33418555+zzzxl1993@users.noreply.github.com>
Date: Thu, 28 Dec 2023 19:58:41 +0800
Subject: [PATCH 123/299] [fix](invert index) fix error handling for
 match_regexp resulting in an empty match. (#29233)

---
 .../segment_v2/inverted_index/query/regexp_query.cpp   | 10 ++++++++--
 .../data/inverted_index_p0/test_index_match_regexp.out |  3 +++
 .../inverted_index_p0/test_index_match_regexp.groovy   |  2 ++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp b/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp
index 83c5401bac0e5b..ee959e1da7367e 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index/query/regexp_query.cpp
@@ -70,10 +70,12 @@ void RegexpQuery::add(const std::wstring& field_name, const std::string& pattern
             }
 
             if (is_match) {
-                terms.emplace_back(std::move(input));
-                if (++count >= _max_expansions) {
+                if (_max_expansions > 0 && count >= _max_expansions) {
                     break;
                 }
+
+                terms.emplace_back(std::move(input));
+                count++;
             }
 
             _CLDECDELETE(term);
@@ -88,6 +90,10 @@ void RegexpQuery::add(const std::wstring& field_name, const std::string& pattern
         hs_free_database(database);
     })
 
+    if (terms.empty()) {
+        return;
+    }
+
     query.add(field_name, terms);
 }
 
diff --git a/regression-test/data/inverted_index_p0/test_index_match_regexp.out b/regression-test/data/inverted_index_p0/test_index_match_regexp.out
index eab27de65ee45f..f9a9caf6d74a5b 100644
--- a/regression-test/data/inverted_index_p0/test_index_match_regexp.out
+++ b/regression-test/data/inverted_index_p0/test_index_match_regexp.out
@@ -14,3 +14,6 @@
 -- !sql --
 38
 
+-- !sql --
+0
+
diff --git a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy
index 7567102d596ee8..3458f086740e48 100644
--- a/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy
+++ b/regression-test/suites/inverted_index_p0/test_index_match_regexp.groovy
@@ -85,6 +85,8 @@ suite("test_index_match_regexp", "p0"){
         qt_sql """ select count() from test_index_match_regexp where request match_regexp 's\$'; """
         qt_sql """ select count() from test_index_match_regexp where request match_regexp 'er\$'; """
         qt_sql """ select count() from test_index_match_regexp where request match_regexp '.*tickets.*'; """
+        qt_sql """ select count() from test_index_match_regexp where request match_regexp 'nonexistence'; """
+
     } finally {
         //try_sql("DROP TABLE IF EXISTS ${testTable}")
     }

From 5129ab5738c3da0eb7c9c165c022051b771a3cf4 Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Thu, 28 Dec 2023 21:17:16 +0800
Subject: [PATCH 124/299] [fix](decimalv2) fix decimalv2 agg errors (#29246)

---
 be/src/common/consts.h                        |  4 +-
 be/src/runtime/type_limit.h                   |  5 +-
 be/src/util/string_parser.hpp                 | 31 +++++----
 be/src/vec/data_types/data_type_decimal.cpp   |  7 +-
 be/src/vec/data_types/data_type_decimal.h     |  2 +-
 .../decimalv2/test_decimalv2_agg.out          | 19 ++++++
 .../decimalv2/test_decimalv2_load.out         |  8 +++
 .../data/datatype_p0/decimalv3/test_load.out  |  6 ++
 .../decimalv2/test_decimalv2_agg.groovy       | 64 +++++++++++++++++++
 .../decimalv2/test_decimalv2_load.groovy      | 36 +++++++++++
 .../datatype_p0/decimalv3/test_load.groovy    | 43 +++++++++++++
 11 files changed, 199 insertions(+), 26 deletions(-)
 create mode 100644 regression-test/data/datatype_p0/decimalv2/test_decimalv2_agg.out
 create mode 100644 regression-test/suites/datatype_p0/decimalv2/test_decimalv2_agg.groovy

diff --git a/be/src/common/consts.h b/be/src/common/consts.h
index 7548f9a2026900..dfee235e37bf79 100644
--- a/be/src/common/consts.h
+++ b/be/src/common/consts.h
@@ -40,8 +40,8 @@ constexpr int MAX_DECIMAL128_PRECISION = 38;
 constexpr int MAX_DECIMAL256_PRECISION = 76;
 
 /// Must be kept in sync with FE's max precision/scale.
-static constexpr int MAX_DECIMALV2_PRECISION = MAX_DECIMAL128_PRECISION;
-static constexpr int MAX_DECIMALV2_SCALE = MAX_DECIMALV2_PRECISION;
+static constexpr int MAX_DECIMALV2_PRECISION = 27;
+static constexpr int MAX_DECIMALV2_SCALE = 9;
 
 static constexpr int MAX_DECIMALV3_PRECISION = MAX_DECIMAL256_PRECISION;
 static constexpr int MAX_DECIMALV3_SCALE = MAX_DECIMALV3_PRECISION;
diff --git a/be/src/runtime/type_limit.h b/be/src/runtime/type_limit.h
index ebfc1d92ec7d98..be3b2186d12024 100644
--- a/be/src/runtime/type_limit.h
+++ b/be/src/runtime/type_limit.h
@@ -64,10 +64,7 @@ struct type_limit<vectorized::Decimal128I> {
 
 template <>
 struct type_limit<vectorized::Decimal128> {
-    static vectorized::Decimal128 max() {
-        return (static_cast<int128_t>(999999999999999999ll) * 100000000000000000ll * 1000ll +
-                static_cast<int128_t>(99999999999999999ll) * 1000ll + 999ll);
-    }
+    static vectorized::Decimal128 max() { return DecimalV2Value::get_max_decimal().value(); }
     static vectorized::Decimal128 min() { return -max(); }
 };
 static wide::Int256 MAX_DECIMAL256_INT({18446744073709551615ul, 8607968719199866879ul,
diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp
index 83dcc4fd958500..3a72250d2fa0b5 100644
--- a/be/src/util/string_parser.hpp
+++ b/be/src/util/string_parser.hpp
@@ -154,7 +154,8 @@ class StringParser {
         return string_to_bool_internal(s + i, len - i, result);
     }
 
-    template <PrimitiveType P, typename T = PrimitiveTypeTraits<P>::CppType::NativeType>
+    template <PrimitiveType P, typename T = PrimitiveTypeTraits<P>::CppType::NativeType,
+              typename DecimalType = PrimitiveTypeTraits<P>::ColumnType::value_type>
     static inline T string_to_decimal(const char* s, int len, int type_precision, int type_scale,
                                       ParseResult* result);
 
@@ -519,7 +520,7 @@ inline bool StringParser::string_to_bool_internal(const char* s, int len, ParseR
     return false;
 }
 
-template <PrimitiveType P, typename T>
+template <PrimitiveType P, typename T, typename DecimalType>
 T StringParser::string_to_decimal(const char* s, int len, int type_precision, int type_scale,
                                   ParseResult* result) {
     static_assert(std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> ||
@@ -598,8 +599,9 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in
                     value = (value * 10) + (c - '0'); // Benchmarks are faster with parenthesis...
                 } else {
                     *result = StringParser::PARSE_OVERFLOW;
-                    value = is_negative ? type_limit<DecimalV2Value>::min()
-                                        : type_limit<DecimalV2Value>::max();
+                    value = is_negative
+                                    ? vectorized::min_decimal_value<DecimalType>(type_precision)
+                                    : vectorized::max_decimal_value<DecimalType>(type_precision);
                     return value;
                 }
                 DCHECK(value >= 0); // For some reason //DCHECK_GE doesn't work with __int128.
@@ -646,10 +648,9 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in
                     cur_digit = precision - scale;
                 } else if (!found_dot && max_digit < (precision - scale)) {
                     *result = StringParser::PARSE_OVERFLOW;
-                    value = is_negative ? vectorized::min_decimal_value<vectorized::Decimal<T>>(
-                                                  type_precision)
-                                        : vectorized::max_decimal_value<vectorized::Decimal<T>>(
-                                                  type_precision);
+                    value = is_negative
+                                    ? vectorized::min_decimal_value<DecimalType>(type_precision)
+                                    : vectorized::max_decimal_value<DecimalType>(type_precision);
                     return value;
                 } else if (found_dot && scale >= type_scale && !has_round) {
                     // make rounding cases
@@ -689,11 +690,10 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in
                     if (!is_numeric_ascii(c)) {
                         if (cur_digit > type_precision) {
                             *result = StringParser::PARSE_OVERFLOW;
-                            value = is_negative
-                                            ? vectorized::min_decimal_value<vectorized::Decimal<T>>(
-                                                      type_precision)
-                                            : vectorized::max_decimal_value<vectorized::Decimal<T>>(
-                                                      type_precision);
+                            value = is_negative ? vectorized::min_decimal_value<DecimalType>(
+                                                          type_precision)
+                                                : vectorized::max_decimal_value<DecimalType>(
+                                                          type_precision);
                             return value;
                         }
                         return is_negative ? T(-value) : T(value);
@@ -732,9 +732,8 @@ T StringParser::string_to_decimal(const char* s, int len, int type_precision, in
         *result = StringParser::PARSE_OVERFLOW;
         if constexpr (TYPE_DECIMALV2 != P) {
             // decimalv3 overflow will return max min value for type precision
-            value = is_negative
-                            ? vectorized::min_decimal_value<vectorized::Decimal<T>>(type_precision)
-                            : vectorized::max_decimal_value<vectorized::Decimal<T>>(type_precision);
+            value = is_negative ? vectorized::min_decimal_value<DecimalType>(type_precision)
+                                : vectorized::max_decimal_value<DecimalType>(type_precision);
             return value;
         }
     } else if (UNLIKELY(scale > type_scale)) {
diff --git a/be/src/vec/data_types/data_type_decimal.cpp b/be/src/vec/data_types/data_type_decimal.cpp
index 38c3468c5dfdb2..20e444fb0e2106 100644
--- a/be/src/vec/data_types/data_type_decimal.cpp
+++ b/be/src/vec/data_types/data_type_decimal.cpp
@@ -166,11 +166,12 @@ bool DataTypeDecimal<T>::parse_from_string(const std::string& str, T* res) const
 }
 
 DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value, bool use_v2) {
-    if (precision_value < min_decimal_precision() ||
-        precision_value > max_decimal_precision<Decimal256>()) {
+    auto max_precision =
+            use_v2 ? max_decimal_precision<Decimal128>() : max_decimal_precision<Decimal256>();
+    if (precision_value < min_decimal_precision() || precision_value > max_precision) {
         throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
                                "Wrong precision {}, min: {}, max: {}", precision_value,
-                               min_decimal_precision(), max_decimal_precision<Decimal256>());
+                               min_decimal_precision(), max_precision);
     }
 
     if (static_cast<UInt64>(scale_value) > precision_value) {
diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h
index 7bb78265738a17..68d48e393b06af 100644
--- a/be/src/vec/data_types/data_type_decimal.h
+++ b/be/src/vec/data_types/data_type_decimal.h
@@ -87,7 +87,7 @@ constexpr size_t max_decimal_precision<Decimal64>() {
 }
 template <>
 constexpr size_t max_decimal_precision<Decimal128>() {
-    return BeConsts::MAX_DECIMAL128_PRECISION;
+    return BeConsts::MAX_DECIMALV2_PRECISION;
 }
 template <>
 constexpr size_t max_decimal_precision<Decimal128I>() {
diff --git a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_agg.out b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_agg.out
new file mode 100644
index 00000000000000..19e5ec004b6af0
--- /dev/null
+++ b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_agg.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !decimalv2_min --
+-123456789012345678.901234567
+
+-- !decimalv2_max --
+123456789012345678.901234567
+
+-- !decimalv2_count --
+3
+
+-- !decimalv2_sum --
+0.012345679
+
+-- !decimalv2_avg --
+0.004115226
+
+-- !decimalv2_sum2 --
+1000000000000000000.000000000
+
diff --git a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out
index 8156a9144aa4c2..ae9921a0f37704 100644
--- a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out
+++ b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out
@@ -15,3 +15,11 @@
 11.99990
 837.43444
 
+-- !decimalv2_insert --
+999999999999999999.999999999	1.000000000
+-999999999999999999.999999999	2.000000000
+999999999999999999.999999999	3.000000000
+-999999999999999999.999999999	4.000000000
+999999999999999999.999999999	5.000000000
+-999999999999999999.999999999	6.000000000
+
diff --git a/regression-test/data/datatype_p0/decimalv3/test_load.out b/regression-test/data/datatype_p0/decimalv3/test_load.out
index 35c355781e68d9..204304a2a30cca 100644
--- a/regression-test/data/datatype_p0/decimalv3/test_load.out
+++ b/regression-test/data/datatype_p0/decimalv3/test_load.out
@@ -4,3 +4,9 @@
 0.000135039891190100
 0.000390160098269360
 
+-- !decimalv3_insert --
+-99999999999999999999999999999999.999999	2.000000
+-99999999999999999999999999999999.999999	4.000000
+99999999999999999999999999999999.999999	1.000000
+99999999999999999999999999999999.999999	3.000000
+
diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_agg.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_agg.groovy
new file mode 100644
index 00000000000000..36fc347c0c4607
--- /dev/null
+++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_agg.groovy
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_decimalv2_agg", "nonConcurrent") {
+
+    sql """
+        admin set frontend config("enable_decimal_conversion" = "false");
+    """
+
+    sql """
+        drop table if exists test_decimalv2_agg;
+    """
+    sql """
+        create table test_decimalv2_agg (
+            k1 decimalv2(27,9)
+        )distributed by hash(k1)
+        properties("replication_num"="1");
+    """
+    sql """
+        insert into test_decimalv2_agg values
+            (123456789012345678.901234567),
+            (-123456789012345678.901234567),
+            (0.012345679),
+            (NULL);
+    """
+    qt_decimalv2_min " select min(k1) from test_decimalv2_agg; "
+    qt_decimalv2_max " select max(k1) from test_decimalv2_agg; "
+    qt_decimalv2_count " select count(k1) from test_decimalv2_agg; "
+    qt_decimalv2_sum " select sum(k1) from test_decimalv2_agg; "
+    qt_decimalv2_avg " select avg(k1) from test_decimalv2_agg; "
+
+    // test overflow
+    sql """
+        drop table if exists test_decimalv2_agg;
+    """
+    sql """
+        create table test_decimalv2_agg (
+            k1 decimalv2(27,9)
+        )distributed by hash(k1)
+        properties("replication_num"="1");
+    """
+    sql """
+        insert into test_decimalv2_agg values
+            (999999999999999999.999999999),
+            (0.000000001),
+            (NULL);
+    """
+    qt_decimalv2_sum2 " select sum(k1) from test_decimalv2_agg; "
+
+}
\ No newline at end of file
diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy
index 5c065a921a0b29..08027c96d1d4a1 100644
--- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy
+++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy
@@ -84,6 +84,42 @@ suite("test_decimalv2_load", "nonConcurrent") {
         select * from ${tableName2} order by 1;
     """
 
+    sql """
+        drop table if exists test_decimalv2_insert;
+    """
+    sql """
+        CREATE TABLE `test_decimalv2_insert` (
+            `k1` decimalv2(27, 9) null,
+            `k2` decimalv2(27, 9) null
+        )
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 10
+        PROPERTIES (
+        "replication_num" = "1"
+        );
+    """
+    sql "set enable_insert_strict=true;"
+    // overflow, max is inserted
+    sql """
+        insert into test_decimalv2_insert values("999999999999999999999999999999",1);
+    """
+    // underflow, min is inserted
+    sql """
+        insert into test_decimalv2_insert values("-999999999999999999999999999999",2);
+    """
+    sql """
+        insert into test_decimalv2_insert values("999999999999999999.9999999991",3);
+    """
+    sql """
+        insert into test_decimalv2_insert values("-999999999999999999.9999999991",4);
+    """
+    sql """
+        insert into test_decimalv2_insert values("999999999999999999.9999999995",5);
+    """
+    sql """
+        insert into test_decimalv2_insert values("-999999999999999999.9999999995",6);
+    """
+    qt_decimalv2_insert "select * from test_decimalv2_insert order by 2; "
+
     sql """
         admin set frontend config("enable_decimal_conversion" = "true");
     """
diff --git a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy
index c2651b3c908a6d..726b1ef048561e 100644
--- a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy
+++ b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy
@@ -57,4 +57,47 @@ suite("test_load") {
     } finally {
         try_sql("DROP TABLE IF EXISTS ${tableName}")
     }
+
+    sql """
+        drop table if exists test_decimalv3_insert;
+    """
+    sql """
+        CREATE TABLE `test_decimalv3_insert` (
+            `k1` decimalv3(38, 6) null,
+            `k2` decimalv3(38, 6) null
+        )
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 10
+        PROPERTIES (
+        "replication_num" = "1"
+        );
+    """
+    sql "set enable_insert_strict=true;"
+    // overflow, max is inserted
+    sql """
+        insert into test_decimalv3_insert values("9999999999999999999999999999999999999999",1);
+    """
+    // underflow, min is inserted
+    sql """
+        insert into test_decimalv3_insert values("-9999999999999999999999999999999999999999",2);
+    """
+    sql """
+        insert into test_decimalv3_insert values("99999999999999999999999999999999.9999991",3);
+    """
+    sql """
+        insert into test_decimalv3_insert values("-99999999999999999999999999999999.9999991",4);
+    """
+
+    test {
+        sql """
+        insert into test_decimalv3_insert values("99999999999999999999999999999999.9999999",5);
+        """
+        exception "error"
+    }
+    test {
+        sql """
+        insert into test_decimalv3_insert values("-99999999999999999999999999999999.9999999",6);
+        """
+        exception "error"
+    }
+    qt_decimalv3_insert "select * from test_decimalv3_insert order by 1;"
 }

From b093097bc310d64434b7c4a24172d924f6501112 Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Thu, 28 Dec 2023 21:21:17 +0800
Subject: [PATCH 125/299] [improvement](statistic)Improve auto analyze
 visibility. (#29046)

Show auto analyze can show the running jobs, not only the finished/failed jobs.
Show analyze task status could show auto tasks as well.
Remove some useless code.
Auto analyze execute catalog/db/table in the order of id, small id first.
---
 .../org/apache/doris/qe/ShowExecutor.java     |  12 +-
 .../doris/statistics/AnalysisManager.java     | 228 +++++-------------
 .../statistics/StatisticsAutoCollector.java   |  22 +-
 .../doris/statistics/StatisticsCollector.java |   4 -
 .../doris/statistics/AnalysisManagerTest.java | 114 +--------
 5 files changed, 88 insertions(+), 292 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 0a895276909b04..cfb8962e095cba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -2654,8 +2654,7 @@ private void handleShowCreateCatalog() throws AnalysisException {
 
     private void handleShowAnalyze() {
         ShowAnalyzeStmt showStmt = (ShowAnalyzeStmt) stmt;
-        List<AnalysisInfo> results = Env.getCurrentEnv().getAnalysisManager()
-                .showAnalysisJob(showStmt);
+        List<AnalysisInfo> results = Env.getCurrentEnv().getAnalysisManager().showAnalysisJob(showStmt);
         List<List<String>> resultRows = Lists.newArrayList();
         DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
         for (AnalysisInfo analysisInfo : results) {
@@ -2681,14 +2680,7 @@ private void handleShowAnalyze() {
                         LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.lastExecTimeInMs),
                         ZoneId.systemDefault())));
                 row.add(analysisInfo.state.toString());
-                try {
-                    row.add(showStmt.isAuto()
-                            ? analysisInfo.progress
-                            : Env.getCurrentEnv().getAnalysisManager().getJobProgress(analysisInfo.jobId));
-                } catch (Exception e) {
-                    row.add("N/A");
-                    LOG.warn("Failed to get progress for job: {}", analysisInfo, e);
-                }
+                row.add(Env.getCurrentEnv().getAnalysisManager().getJobProgress(analysisInfo.jobId));
                 row.add(analysisInfo.scheduleType.toString());
                 LocalDateTime startTime =
                         LocalDateTime.ofInstant(Instant.ofEpochMilli(analysisInfo.startTime),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 39ae191d45a83c..0a52cc8faeaa9a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -78,7 +78,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -96,8 +95,6 @@
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
-import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
 public class AnalysisManager implements Writable {
@@ -119,9 +116,6 @@ public class AnalysisManager implements Writable {
     protected final NavigableMap<Long, AnalysisInfo> analysisJobInfoMap =
             Collections.synchronizedNavigableMap(new TreeMap<>());
 
-    // Tracking system submitted job, keep in mem only
-    protected final Map<Long, AnalysisInfo> systemJobInfoMap = new ConcurrentHashMap<>();
-
     // Tracking and control sync analyze tasks, keep in mem only
     private final ConcurrentMap<ConnectContext, SyncTaskCollection> ctxToSyncTask = new ConcurrentHashMap<>();
 
@@ -129,123 +123,11 @@ public class AnalysisManager implements Writable {
 
     private final Map<Long, AnalysisJob> idToAnalysisJob = new ConcurrentHashMap<>();
 
+    // To be deprecated, keep it for meta compatibility now, will remove later.
     protected SimpleQueue<AnalysisInfo> autoJobs = createSimpleQueue(null, this);
 
-    private final Function<TaskStatusWrapper, Void> userJobStatusUpdater = w -> {
-        AnalysisInfo info = w.info;
-        AnalysisState taskState = w.taskState;
-        String message = w.message;
-        long time = w.time;
-        if (analysisJobIdToTaskMap.get(info.jobId) == null) {
-            return null;
-        }
-        info.state = taskState;
-        info.message = message;
-        // Update the task cost time when task finished or failed. And only log the final state.
-        if (taskState.equals(AnalysisState.FINISHED) || taskState.equals(AnalysisState.FAILED)) {
-            info.timeCostInMs = time - info.lastExecTimeInMs;
-            info.lastExecTimeInMs = time;
-            logCreateAnalysisTask(info);
-        }
-        info.lastExecTimeInMs = time;
-        AnalysisInfo job = analysisJobInfoMap.get(info.jobId);
-        // Job may get deleted during execution.
-        if (job == null) {
-            return null;
-        }
-        // Synchronize the job state change in job level.
-        synchronized (job) {
-            job.lastExecTimeInMs = time;
-            // Set the job state to RUNNING when its first task becomes RUNNING.
-            if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) {
-                job.state = AnalysisState.RUNNING;
-                job.markStartTime(System.currentTimeMillis());
-                replayCreateAnalysisJob(job);
-            }
-            boolean allFinished = true;
-            boolean hasFailure = false;
-            for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) {
-                AnalysisInfo taskInfo = task.info;
-                if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) {
-                    allFinished = false;
-                    break;
-                }
-                if (taskInfo.state.equals(AnalysisState.FAILED)) {
-                    hasFailure = true;
-                }
-            }
-            if (allFinished) {
-                if (hasFailure) {
-                    job.markFailed();
-                } else {
-                    job.markFinished();
-                    try {
-                        updateTableStats(job);
-                    } catch (Throwable e) {
-                        LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e);
-                    }
-                }
-                logCreateAnalysisJob(job);
-                analysisJobIdToTaskMap.remove(job.jobId);
-            }
-        }
-        return null;
-    };
-
     private final String progressDisplayTemplate = "%d Finished  |  %d Failed  |  %d In Progress  |  %d Total";
 
-    protected final Function<TaskStatusWrapper, Void> systemJobStatusUpdater = w -> {
-        AnalysisInfo info = w.info;
-        info.state = w.taskState;
-        info.message = w.message;
-        AnalysisInfo job = systemJobInfoMap.get(info.jobId);
-        if (job == null) {
-            return null;
-        }
-        synchronized (job) {
-            // Set the job state to RUNNING when its first task becomes RUNNING.
-            if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) {
-                job.state = AnalysisState.RUNNING;
-                job.markStartTime(System.currentTimeMillis());
-            }
-        }
-        int failedCount = 0;
-        StringJoiner reason = new StringJoiner(", ");
-        Map<Long, BaseAnalysisTask> taskMap = analysisJobIdToTaskMap.get(info.jobId);
-        for (BaseAnalysisTask task : taskMap.values()) {
-            if (task.info.state.equals(AnalysisState.RUNNING) || task.info.state.equals(AnalysisState.PENDING)) {
-                return null;
-            }
-            if (task.info.state.equals(AnalysisState.FAILED)) {
-                failedCount++;
-                reason.add(task.info.message);
-            }
-        }
-        try {
-            updateTableStats(job);
-        } catch (Throwable e) {
-            LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e);
-        } finally {
-            job.lastExecTimeInMs = System.currentTimeMillis();
-            job.message = reason.toString();
-            job.progress = String.format(progressDisplayTemplate,
-                    taskMap.size() - failedCount, failedCount, 0, taskMap.size());
-            if (failedCount > 0) {
-                job.message = reason.toString();
-                job.markFailed();
-            } else {
-                job.markFinished();
-            }
-            autoJobs.offer(job);
-            systemJobInfoMap.remove(info.jobId);
-            analysisJobIdToTaskMap.remove(info.jobId);
-        }
-        return null;
-    };
-
-    private final Function<TaskStatusWrapper, Void>[] updaters =
-            new Function[] {userJobStatusUpdater, systemJobStatusUpdater};
-
     public AnalysisManager() {
         if (!Env.isCheckpointThread()) {
             this.taskExecutor = new AnalysisTaskExecutor(Config.statistics_simultaneously_running_task_num,
@@ -524,7 +406,7 @@ public AnalysisInfo buildAnalysisJobInfo(AnalyzeTblStmt stmt) throws DdlExceptio
     }
 
     @VisibleForTesting
-    public void recordAnalysisJob(AnalysisInfo jobInfo) throws DdlException {
+    public void recordAnalysisJob(AnalysisInfo jobInfo) {
         if (jobInfo.scheduleType == ScheduleType.PERIOD && jobInfo.lastExecTimeInMs > 0) {
             return;
         }
@@ -554,13 +436,7 @@ public void createTaskForEachColumns(AnalysisInfo jobInfo, Map<Long, BaseAnalysi
             if (isSync) {
                 continue;
             }
-            try {
-                if (!jobInfo.jobType.equals(JobType.SYSTEM)) {
-                    replayCreateAnalysisTask(analysisInfo);
-                }
-            } catch (Exception e) {
-                throw new DdlException("Failed to create analysis task", e);
-            }
+            replayCreateAnalysisTask(analysisInfo);
         }
     }
 
@@ -594,16 +470,63 @@ public void createTableLevelTaskForExternalTable(AnalysisInfo jobInfo,
             // For sync job, don't need to persist, return here and execute it immediately.
             return;
         }
-        try {
-            replayCreateAnalysisTask(analysisInfo);
-        } catch (Exception e) {
-            throw new DdlException("Failed to create analysis task", e);
-        }
+        replayCreateAnalysisTask(analysisInfo);
     }
 
     public void updateTaskStatus(AnalysisInfo info, AnalysisState taskState, String message, long time) {
-        TaskStatusWrapper taskStatusWrapper = new TaskStatusWrapper(info, taskState, message, time);
-        updaters[info.jobType.ordinal()].apply(taskStatusWrapper);
+        if (analysisJobIdToTaskMap.get(info.jobId) == null) {
+            return;
+        }
+        info.state = taskState;
+        info.message = message;
+        // Update the task cost time when task finished or failed. And only log the final state.
+        if (taskState.equals(AnalysisState.FINISHED) || taskState.equals(AnalysisState.FAILED)) {
+            info.timeCostInMs = time - info.lastExecTimeInMs;
+            info.lastExecTimeInMs = time;
+            logCreateAnalysisTask(info);
+        }
+        info.lastExecTimeInMs = time;
+        AnalysisInfo job = analysisJobInfoMap.get(info.jobId);
+        // Job may get deleted during execution.
+        if (job == null) {
+            return;
+        }
+        // Synchronize the job state change in job level.
+        synchronized (job) {
+            job.lastExecTimeInMs = time;
+            // Set the job state to RUNNING when its first task becomes RUNNING.
+            if (info.state.equals(AnalysisState.RUNNING) && job.state.equals(AnalysisState.PENDING)) {
+                job.state = AnalysisState.RUNNING;
+                job.markStartTime(System.currentTimeMillis());
+                replayCreateAnalysisJob(job);
+            }
+            boolean allFinished = true;
+            boolean hasFailure = false;
+            for (BaseAnalysisTask task : analysisJobIdToTaskMap.get(info.jobId).values()) {
+                AnalysisInfo taskInfo = task.info;
+                if (taskInfo.state.equals(AnalysisState.RUNNING) || taskInfo.state.equals(AnalysisState.PENDING)) {
+                    allFinished = false;
+                    break;
+                }
+                if (taskInfo.state.equals(AnalysisState.FAILED)) {
+                    hasFailure = true;
+                }
+            }
+            if (allFinished) {
+                if (hasFailure) {
+                    job.markFailed();
+                } else {
+                    job.markFinished();
+                    try {
+                        updateTableStats(job);
+                    } catch (Throwable e) {
+                        LOG.warn("Failed to update Table statistics in job: {}", info.toString(), e);
+                    }
+                }
+                logCreateAnalysisJob(job);
+                analysisJobIdToTaskMap.remove(job.jobId);
+            }
+        }
     }
 
     @VisibleForTesting
@@ -629,12 +552,6 @@ public void updateTableStats(AnalysisInfo jobInfo) {
     }
 
     public List<AnalysisInfo> showAnalysisJob(ShowAnalyzeStmt stmt) {
-        if (stmt.isAuto()) {
-            // It's ok to sync on this field, it would only be assigned when instance init or do checkpoint
-            synchronized (autoJobs) {
-                return findShowAnalyzeResult(autoJobs, stmt);
-            }
-        }
         return findShowAnalyzeResult(analysisJobInfoMap.values(), stmt);
     }
 
@@ -650,6 +567,8 @@ protected List<AnalysisInfo> findShowAnalyzeResult(Collection<AnalysisInfo> anal
                 .filter(a -> stmt.getJobId() == 0 || a.jobId == stmt.getJobId())
                 .filter(a -> state == null || a.state.equals(AnalysisState.valueOf(state)))
                 .filter(a -> tblName == null || a.tblId == tblId)
+                .filter(a -> stmt.isAuto() && a.jobType.equals(JobType.SYSTEM)
+                             || !stmt.isAuto() && a.jobType.equals(JobType.MANUAL))
                 .sorted(Comparator.comparingLong(a -> a.jobId))
                 .collect(Collectors.toList());
     }
@@ -884,24 +803,6 @@ public void execute(ThreadPoolExecutor executor) {
         }
     }
 
-    public List<AnalysisInfo> findPeriodicJobs() {
-        synchronized (analysisJobInfoMap) {
-            Predicate<AnalysisInfo> p = a -> {
-                if (a.state.equals(AnalysisState.RUNNING)) {
-                    return false;
-                }
-                if (a.cronExpression == null) {
-                    return a.scheduleType.equals(ScheduleType.PERIOD)
-                            && System.currentTimeMillis() - a.lastExecTimeInMs > a.periodTimeInMs;
-                }
-                return a.cronExpression.getTimeAfter(new Date(a.lastExecTimeInMs)).before(new Date());
-            };
-            return analysisJobInfoMap.values().stream()
-                    .filter(p)
-                    .collect(Collectors.toList());
-        }
-    }
-
     public List<AnalysisInfo> findTasks(long jobId) {
         synchronized (analysisTaskInfoMap) {
             return analysisTaskInfoMap.values().stream().filter(i -> i.jobId == jobId).collect(Collectors.toList());
@@ -979,10 +880,11 @@ private static void readIdToTblStats(DataInput in, Map<Long, TableStatsMeta> map
         }
     }
 
+    // To be deprecated, keep it for meta compatibility now, will remove later.
     private static void readAutoJobs(DataInput in, AnalysisManager analysisManager) throws IOException {
         Type type = new TypeToken<LinkedList<AnalysisInfo>>() {}.getType();
-        Collection<AnalysisInfo> autoJobs = GsonUtils.GSON.fromJson(Text.readString(in), type);
-        analysisManager.autoJobs = analysisManager.createSimpleQueue(autoJobs, analysisManager);
+        GsonUtils.GSON.fromJson(Text.readString(in), type);
+        analysisManager.autoJobs = analysisManager.createSimpleQueue(null, null);
     }
 
     @Override
@@ -1045,7 +947,7 @@ public void logCreateTableStats(TableStatsMeta tableStats) {
     }
 
     public void registerSysJob(AnalysisInfo jobInfo, Map<Long, BaseAnalysisTask> taskInfos) {
-        systemJobInfoMap.put(jobInfo.jobId, jobInfo);
+        recordAnalysisJob(jobInfo);
         analysisJobIdToTaskMap.put(jobInfo.jobId, taskInfos);
     }
 
@@ -1117,10 +1019,6 @@ public void removeJob(long id) {
         idToAnalysisJob.remove(id);
     }
 
-    public boolean hasUnFinished() {
-        return !analysisJobIdToTaskMap.isEmpty();
-    }
-
     /**
      * Only OlapTable and Hive HMSExternalTable can sample for now.
      * @param table
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index f799da56206095..17d666b7ea4a2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -38,7 +38,6 @@
 
 import java.time.LocalTime;
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -69,12 +68,12 @@ protected void collect() {
 
     @SuppressWarnings({"rawtypes", "unchecked"})
     private void analyzeAll() {
-        Set<CatalogIf> catalogs = Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog();
+        List<CatalogIf> catalogs = getCatalogsInOrder();
         for (CatalogIf ctl : catalogs) {
             if (!ctl.enableAutoAnalyze()) {
                 continue;
             }
-            Collection<DatabaseIf> dbs = ctl.getAllDbs();
+            List<DatabaseIf> dbs = getDatabasesInOrder(ctl);
             for (DatabaseIf<TableIf> databaseIf : dbs) {
                 if (StatisticConstants.SYSTEM_DBS.contains(databaseIf.getFullName())) {
                     continue;
@@ -89,6 +88,21 @@ private void analyzeAll() {
         }
     }
 
+    public List<CatalogIf> getCatalogsInOrder() {
+        return Env.getCurrentEnv().getCatalogMgr().getCopyOfCatalog().stream()
+            .sorted((c1, c2) -> (int) (c1.getId() - c2.getId())).collect(Collectors.toList());
+    }
+
+    public List<DatabaseIf<? extends TableIf>> getDatabasesInOrder(CatalogIf<DatabaseIf> catalog) {
+        return catalog.getAllDbs().stream()
+            .sorted((d1, d2) -> (int) (d1.getId() - d2.getId())).collect(Collectors.toList());
+    }
+
+    public List<TableIf> getTablesInOrder(DatabaseIf<? extends TableIf> db) {
+        return db.getTables().stream()
+            .sorted((t1, t2) -> (int) (t1.getId() - t2.getId())).collect(Collectors.toList());
+    }
+
     public void analyzeDb(DatabaseIf<TableIf> databaseIf) throws DdlException {
         List<AnalysisInfo> analysisInfos = constructAnalysisInfo(databaseIf);
         for (AnalysisInfo analysisInfo : analysisInfos) {
@@ -109,7 +123,7 @@ public void analyzeDb(DatabaseIf<TableIf> databaseIf) throws DdlException {
 
     protected List<AnalysisInfo> constructAnalysisInfo(DatabaseIf<? extends TableIf> db) {
         List<AnalysisInfo> analysisInfos = new ArrayList<>();
-        for (TableIf table : db.getTables()) {
+        for (TableIf table : getTablesInOrder(db)) {
             try {
                 if (skip(table)) {
                     continue;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
index 9d4c311523b59c..0985b9b2b9539e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCollector.java
@@ -52,10 +52,6 @@ protected void runAfterCatalogReady() {
         if (Env.isCheckpointThread()) {
             return;
         }
-        if (Env.getCurrentEnv().getAnalysisManager().hasUnFinished()) {
-            LOG.info("Analyze tasks those submitted in last time is not finished, skip");
-            return;
-        }
         collect();
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index 41ace52b8ba67c..27edb960e67575 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -57,12 +57,13 @@ public void testUpdateTaskStatus(@Mocked BaseAnalysisTask task1,
 
         new MockUp<AnalysisManager>() {
             @Mock
-            public void logCreateAnalysisTask(AnalysisInfo job) {
-            }
+            public void logCreateAnalysisTask(AnalysisInfo job) {}
 
             @Mock
-            public void logCreateAnalysisJob(AnalysisInfo job) {
-            }
+            public void logCreateAnalysisJob(AnalysisInfo job) {}
+
+            @Mock
+            public void updateTableStats(AnalysisInfo jobInfo) {}
 
         };
 
@@ -269,111 +270,6 @@ public void logCreateAnalysisJob(AnalysisInfo analysisJob) {
         };
     }
 
-    @Test
-    public void testSystemJobStatusUpdater() {
-        new MockUp<BaseAnalysisTask>() {
-
-            @Mock
-            protected void init(AnalysisInfo info) {
-
-            }
-        };
-
-        new MockUp<AnalysisManager>() {
-            @Mock
-            public void updateTableStats(AnalysisInfo jobInfo) {}
-
-            @Mock
-            protected void logAutoJob(AnalysisInfo autoJob) {
-
-            }
-        };
-
-        AnalysisManager analysisManager = new AnalysisManager();
-        AnalysisInfo job = new AnalysisInfoBuilder()
-                .setJobId(0)
-                .setColName("col1, col2").build();
-        analysisManager.systemJobInfoMap.put(job.jobId, job);
-        AnalysisInfo task1 = new AnalysisInfoBuilder()
-                .setJobId(0)
-                .setTaskId(1)
-                .setState(AnalysisState.RUNNING)
-                .setColName("col1").build();
-        AnalysisInfo task2 = new AnalysisInfoBuilder()
-                .setJobId(0)
-                .setTaskId(1)
-                .setState(AnalysisState.FINISHED)
-                .setColName("col2").build();
-        OlapAnalysisTask ot1 = new OlapAnalysisTask(task1);
-        OlapAnalysisTask ot2 = new OlapAnalysisTask(task2);
-        Map<Long, BaseAnalysisTask> taskMap = new HashMap<>();
-        taskMap.put(ot1.info.taskId, ot1);
-        taskMap.put(ot2.info.taskId, ot2);
-        analysisManager.analysisJobIdToTaskMap.put(job.jobId, taskMap);
-
-        // test invalid job
-        AnalysisInfo invalidJob = new AnalysisInfoBuilder().setJobId(-1).build();
-        analysisManager.systemJobStatusUpdater.apply(new TaskStatusWrapper(invalidJob,
-                AnalysisState.FAILED, "", 0));
-
-        // test finished
-        analysisManager.systemJobStatusUpdater.apply(new TaskStatusWrapper(task1, AnalysisState.FAILED, "", 0));
-        analysisManager.systemJobStatusUpdater.apply(new TaskStatusWrapper(task1, AnalysisState.FINISHED, "", 0));
-        Assertions.assertEquals(1, analysisManager.autoJobs.size());
-        Assertions.assertTrue(analysisManager.systemJobInfoMap.isEmpty());
-    }
-
-    @Test
-    public void testSystemJobStartTime() {
-        new MockUp<BaseAnalysisTask>() {
-
-            @Mock
-            protected void init(AnalysisInfo info) {
-
-            }
-        };
-
-        new MockUp<AnalysisManager>() {
-            @Mock
-            public void updateTableStats(AnalysisInfo jobInfo) {
-            }
-
-            @Mock
-            protected void logAutoJob(AnalysisInfo autoJob) {
-
-            }
-        };
-
-        AnalysisManager analysisManager = new AnalysisManager();
-        AnalysisInfo job = new AnalysisInfoBuilder()
-            .setJobId(0)
-            .setColName("col1, col2").build();
-        analysisManager.systemJobInfoMap.put(job.jobId, job);
-        AnalysisInfo task1 = new AnalysisInfoBuilder()
-            .setJobId(0)
-            .setTaskId(1)
-            .setState(AnalysisState.PENDING)
-            .setColName("col1").build();
-        AnalysisInfo task2 = new AnalysisInfoBuilder()
-            .setJobId(0)
-            .setTaskId(1)
-            .setState(AnalysisState.PENDING)
-            .setColName("col2").build();
-        OlapAnalysisTask ot1 = new OlapAnalysisTask(task1);
-        OlapAnalysisTask ot2 = new OlapAnalysisTask(task2);
-        Map<Long, BaseAnalysisTask> taskMap = new HashMap<>();
-        taskMap.put(ot1.info.taskId, ot1);
-        taskMap.put(ot2.info.taskId, ot2);
-        analysisManager.analysisJobIdToTaskMap.put(job.jobId, taskMap);
-
-        job.state = AnalysisState.PENDING;
-        long l = System.currentTimeMillis();
-        analysisManager.systemJobInfoMap.put(job.jobId, job);
-        analysisManager.systemJobStatusUpdater.apply(new TaskStatusWrapper(task1,
-            AnalysisState.RUNNING, "", 0));
-        Assertions.assertTrue(job.startTime >= l);
-    }
-
     @Test
     public void testReAnalyze() {
         new MockUp<OlapTable>() {

From 0912b137e6398857579694ce1d0cad55c3797d35 Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Thu, 28 Dec 2023 21:22:29 +0800
Subject: [PATCH 126/299] [Improvement](pipelineX) optimize local exchange sink
 (#29250)

---
 .../pipeline_x_fragment_context.cpp           | 55 +++++++++++++------
 .../pipeline_x/pipeline_x_fragment_context.h  |  5 ++
 2 files changed, 43 insertions(+), 17 deletions(-)

diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
index b628e998f89189..a8a576889d21e1 100644
--- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
@@ -718,27 +718,15 @@ void PipelineXFragmentContext::_inherit_pipeline_properties(
     pipe_with_source->set_data_distribution(data_distribution);
 }
 
-Status PipelineXFragmentContext::_add_local_exchange(
-        int pip_idx, int idx, int node_id, ObjectPool* pool, PipelinePtr cur_pipe,
+Status PipelineXFragmentContext::_add_local_exchange_impl(
+        int idx, ObjectPool* pool, PipelinePtr cur_pipe, PipelinePtr new_pip,
         DataDistribution data_distribution, bool* do_local_exchange, int num_buckets,
         const std::map<int, int>& bucket_seq_to_instance_idx,
         const bool ignore_data_hash_distribution) {
-    DCHECK(_enable_local_shuffle());
-    if (_num_instances <= 1) {
-        return Status::OK();
-    }
-
-    if (!cur_pipe->need_to_local_exchange(data_distribution)) {
-        return Status::OK();
-    }
-    *do_local_exchange = true;
-
     auto& operator_xs = cur_pipe->operator_xs();
-    auto total_op_num = operator_xs.size();
     const auto downstream_pipeline_id = cur_pipe->id();
     auto local_exchange_id = next_operator_id();
     // 1. Create a new pipeline with local exchange sink.
-    auto new_pip = add_pipeline(cur_pipe, pip_idx + 1);
     DataSinkOperatorXPtr sink;
     auto sink_id = next_sink_operator_id();
     sink.reset(new LocalExchangeSinkOperatorX(sink_id, local_exchange_id, _num_instances,
@@ -751,12 +739,13 @@ Status PipelineXFragmentContext::_add_local_exchange(
     auto shared_state = LocalExchangeSharedState::create_shared();
     switch (data_distribution.distribution_type) {
     case ExchangeType::HASH_SHUFFLE:
-        shared_state->exchanger =
-                ShuffleExchanger::create_unique(cur_pipe->num_tasks(), _num_instances);
+        shared_state->exchanger = ShuffleExchanger::create_unique(
+                std::max(cur_pipe->num_tasks(), _num_instances), _num_instances);
         break;
     case ExchangeType::BUCKET_HASH_SHUFFLE:
         shared_state->exchanger = BucketShuffleExchanger::create_unique(
-                cur_pipe->num_tasks(), _num_instances, num_buckets, ignore_data_hash_distribution);
+                std::max(cur_pipe->num_tasks(), _num_instances), _num_instances, num_buckets,
+                ignore_data_hash_distribution);
         break;
     case ExchangeType::PASSTHROUGH:
         shared_state->exchanger =
@@ -840,11 +829,43 @@ Status PipelineXFragmentContext::_add_local_exchange(
 
     // 7. Inherit properties from current pipeline.
     _inherit_pipeline_properties(data_distribution, cur_pipe, new_pip);
+    return Status::OK();
+}
+
+Status PipelineXFragmentContext::_add_local_exchange(
+        int pip_idx, int idx, int node_id, ObjectPool* pool, PipelinePtr cur_pipe,
+        DataDistribution data_distribution, bool* do_local_exchange, int num_buckets,
+        const std::map<int, int>& bucket_seq_to_instance_idx, const bool ignore_data_distribution) {
+    DCHECK(_enable_local_shuffle());
+    if (_num_instances <= 1) {
+        return Status::OK();
+    }
+
+    if (!cur_pipe->need_to_local_exchange(data_distribution)) {
+        return Status::OK();
+    }
+    *do_local_exchange = true;
+
+    auto& operator_xs = cur_pipe->operator_xs();
+    auto total_op_num = operator_xs.size();
+    auto new_pip = add_pipeline(cur_pipe, pip_idx + 1);
+    RETURN_IF_ERROR(_add_local_exchange_impl(idx, pool, cur_pipe, new_pip, data_distribution,
+                                             do_local_exchange, num_buckets,
+                                             bucket_seq_to_instance_idx, ignore_data_distribution));
 
     CHECK(total_op_num + 1 == cur_pipe->operator_xs().size() + new_pip->operator_xs().size())
             << "total_op_num: " << total_op_num
             << " cur_pipe->operator_xs().size(): " << cur_pipe->operator_xs().size()
             << " new_pip->operator_xs().size(): " << new_pip->operator_xs().size();
+
+    // Add passthrough local exchanger if necessary
+    if (cur_pipe->num_tasks() > 1 && new_pip->num_tasks() == 1 &&
+        Pipeline::is_hash_exchange(data_distribution.distribution_type)) {
+        RETURN_IF_ERROR(_add_local_exchange_impl(
+                new_pip->operator_xs().size(), pool, new_pip, add_pipeline(new_pip, pip_idx + 2),
+                DataDistribution(ExchangeType::PASSTHROUGH), do_local_exchange, num_buckets,
+                bucket_seq_to_instance_idx, ignore_data_distribution));
+    }
     return Status::OK();
 }
 
diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h
index 6fe3488096431d..c8f9ef1edc6cf8 100644
--- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h
+++ b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.h
@@ -129,6 +129,11 @@ class PipelineXFragmentContext : public PipelineFragmentContext {
                                const bool ignore_data_distribution);
     void _inherit_pipeline_properties(const DataDistribution& data_distribution,
                                       PipelinePtr pipe_with_source, PipelinePtr pipe_with_sink);
+    Status _add_local_exchange_impl(int idx, ObjectPool* pool, PipelinePtr cur_pipe,
+                                    PipelinePtr new_pipe, DataDistribution data_distribution,
+                                    bool* do_local_exchange, int num_buckets,
+                                    const std::map<int, int>& bucket_seq_to_instance_idx,
+                                    const bool ignore_data_distribution);
 
     [[nodiscard]] Status _build_pipelines(ObjectPool* pool,
                                           const doris::TPipelineFragmentParams& request,

From ffd178f5ff16370e4d7440821a1927648019e672 Mon Sep 17 00:00:00 2001
From: Jerry Hu <mrhhsg@gmail.com>
Date: Thu, 28 Dec 2023 21:29:07 +0800
Subject: [PATCH 127/299] [feat](pipelinex) support parallel scan on pipeline x
 engine (#29070)

* [feat](pipelinex) support parallel scan on pipeline x engine

* make parallel scan be independent of shared scan
---
 be/src/olap/parallel_scanner_builder.cpp      |  2 +
 be/src/pipeline/exec/olap_scan_operator.cpp   | 67 ++++++++++++++++++-
 be/src/vec/exec/scan/new_olap_scan_node.cpp   | 43 ++++++------
 .../java/org/apache/doris/qe/Coordinator.java |  4 ++
 4 files changed, 94 insertions(+), 22 deletions(-)

diff --git a/be/src/olap/parallel_scanner_builder.cpp b/be/src/olap/parallel_scanner_builder.cpp
index 32b163b7a1a29a..4e64551dc3b669 100644
--- a/be/src/olap/parallel_scanner_builder.cpp
+++ b/be/src/olap/parallel_scanner_builder.cpp
@@ -18,6 +18,7 @@
 #include "parallel_scanner_builder.h"
 
 #include "olap/rowset/beta_rowset.h"
+#include "pipeline/exec/olap_scan_operator.h"
 #include "vec/exec/scan/new_olap_scanner.h"
 
 namespace doris {
@@ -197,5 +198,6 @@ std::shared_ptr<NewOlapScanner> ParallelScannerBuilder<ParentType>::_build_scann
 }
 
 template class ParallelScannerBuilder<NewOlapScanNode>;
+template class ParallelScannerBuilder<pipeline::OlapScanLocalState>;
 
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp b/be/src/pipeline/exec/olap_scan_operator.cpp
index 3e40e173585a08..d1575075b4cea0 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -21,6 +21,7 @@
 
 #include <memory>
 
+#include "olap/parallel_scanner_builder.h"
 #include "olap/storage_engine.h"
 #include "olap/tablet_manager.h"
 #include "pipeline/exec/scan_operator.h"
@@ -28,8 +29,6 @@
 #include "util/to_string.h"
 #include "vec/exec/runtime_filter_consumer.h"
 #include "vec/exec/scan/new_olap_scanner.h"
-#include "vec/exec/scan/pip_scanner_context.h"
-#include "vec/exec/scan/scanner_context.h"
 #include "vec/exec/scan/vscan_node.h"
 #include "vec/exprs/vcompound_pred.h"
 #include "vec/exprs/vectorized_fn_call.h"
@@ -256,6 +255,70 @@ Status OlapScanLocalState::_init_scanners(std::list<vectorized::VScannerSPtr>* s
     if (_cond_ranges.empty()) {
         _cond_ranges.emplace_back(new doris::OlapScanRange());
     }
+
+    bool enable_parallel_scan = state()->enable_parallel_scan();
+    bool has_cpu_limit = state()->query_options().__isset.resource_limit &&
+                         state()->query_options().resource_limit.__isset.cpu_limit;
+
+    if (enable_parallel_scan && !p._should_run_serial && !has_cpu_limit &&
+        p._push_down_agg_type == TPushAggOp::NONE) {
+        std::vector<TabletWithVersion> tablets;
+        bool is_dup_mow_key = true;
+        for (auto&& scan_range : _scan_ranges) {
+            auto tablet = DORIS_TRY(ExecEnv::get_tablet(scan_range->tablet_id));
+            is_dup_mow_key =
+                    tablet->keys_type() == DUP_KEYS || (tablet->keys_type() == UNIQUE_KEYS &&
+                                                        tablet->enable_unique_key_merge_on_write());
+            if (!is_dup_mow_key) {
+                break;
+            }
+
+            int64_t version = 0;
+            std::from_chars(scan_range->version.data(),
+                            scan_range->version.data() + scan_range->version.size(), version);
+            tablets.emplace_back(
+                    TabletWithVersion {std::dynamic_pointer_cast<Tablet>(tablet), version});
+        }
+
+        if (is_dup_mow_key) {
+            std::vector<OlapScanRange*> key_ranges;
+            for (auto& range : _cond_ranges) {
+                if (range->begin_scan_range.size() == 1 &&
+                    range->begin_scan_range.get_value(0) == NEGATIVE_INFINITY) {
+                    continue;
+                }
+                key_ranges.emplace_back(range.get());
+            }
+
+            ParallelScannerBuilder<OlapScanLocalState> scanner_builder(
+                    this, tablets, _scanner_profile, key_ranges, state(), p._limit_per_scanner,
+                    is_dup_mow_key, p._olap_scan_node.is_preaggregation);
+
+            int max_scanners_count = state()->parallel_scan_max_scanners_count();
+
+            // If the `max_scanners_count` was not set,
+            // use `config::doris_scanner_thread_pool_thread_num` as the default value.
+            if (max_scanners_count <= 0) {
+                max_scanners_count = config::doris_scanner_thread_pool_thread_num;
+            }
+
+            // Too small value of `min_rows_per_scanner` is meaningless.
+            auto min_rows_per_scanner =
+                    std::max<int64_t>(1024, state()->parallel_scan_min_rows_per_scanner());
+            scanner_builder.set_max_scanners_count(max_scanners_count);
+            scanner_builder.set_min_rows_per_scanner(min_rows_per_scanner);
+
+            RETURN_IF_ERROR(scanner_builder.build_scanners(*scanners));
+            for (auto& scanner : *scanners) {
+                auto* olap_scanner = assert_cast<vectorized::NewOlapScanner*>(scanner.get());
+                RETURN_IF_ERROR(olap_scanner->prepare(state(), _conjuncts));
+                olap_scanner->set_compound_filters(_compound_filters);
+            }
+            LOG(INFO) << "parallel scanners count: " << scanners->size();
+            return Status::OK();
+        }
+    }
+
     int scanners_per_tablet = std::max(1, 64 / (int)_scan_ranges.size());
 
     auto build_new_scanner = [&](BaseTabletSPtr tablet, int64_t version,
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp b/be/src/vec/exec/scan/new_olap_scan_node.cpp
index b33c656a8df409..8ac1af004d90a4 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp
@@ -532,7 +532,7 @@ Status NewOlapScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
     // Split tablet segment by scanner, only use in pipeline in duplicate key
     // 1. if tablet count lower than scanner thread num, count segment num of all tablet ready for scan
     // TODO: some tablet may do not have segment, may need split segment all case
-    if (_shared_scan_opt && _scan_ranges.size() < config::doris_scanner_thread_pool_thread_num) {
+    if (_scan_ranges.size() < config::doris_scanner_thread_pool_thread_num) {
         for (auto&& [tablet, version] : tablets_to_scan) {
             is_dup_mow_key =
                     tablet->keys_type() == DUP_KEYS || (tablet->keys_type() == UNIQUE_KEYS &&
@@ -541,26 +541,29 @@ Status NewOlapScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
                 break;
             }
 
-            auto& read_source = tablets_read_source.emplace_back();
-            {
-                std::shared_lock rdlock(tablet->get_header_lock());
-                auto st = tablet->capture_rs_readers({0, version}, &read_source.rs_splits, false);
-                if (!st.ok()) {
-                    LOG(WARNING) << "fail to init reader.res=" << st;
-                    return Status::InternalError(
-                            "failed to initialize storage reader. tablet_id={} : {}",
-                            tablet->tablet_id(), st.to_string());
+            if (_shared_scan_opt) {
+                auto& read_source = tablets_read_source.emplace_back();
+                {
+                    std::shared_lock rdlock(tablet->get_header_lock());
+                    auto st =
+                            tablet->capture_rs_readers({0, version}, &read_source.rs_splits, false);
+                    if (!st.ok()) {
+                        LOG(WARNING) << "fail to init reader.res=" << st;
+                        return Status::InternalError(
+                                "failed to initialize storage reader. tablet_id={} : {}",
+                                tablet->tablet_id(), st.to_string());
+                    }
+                }
+                if (!_state->skip_delete_predicate()) {
+                    read_source.fill_delete_predicates();
                 }
-            }
-            if (!_state->skip_delete_predicate()) {
-                read_source.fill_delete_predicates();
-            }
 
-            auto& rs_seg_count = tablet_rs_seg_count.emplace_back();
-            for (const auto& rowset_splits : read_source.rs_splits) {
-                auto num_segments = rowset_splits.rs_reader->rowset()->num_segments();
-                rs_seg_count.emplace_back(num_segments);
-                segment_count += num_segments;
+                auto& rs_seg_count = tablet_rs_seg_count.emplace_back();
+                for (const auto& rowset_splits : read_source.rs_splits) {
+                    auto num_segments = rowset_splits.rs_reader->rowset()->num_segments();
+                    rs_seg_count.emplace_back(num_segments);
+                    segment_count += num_segments;
+                }
             }
         }
     }
@@ -627,7 +630,7 @@ Status NewOlapScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
         return Status::OK();
     };
 
-    if (is_dup_mow_key) {
+    if (_shared_scan_opt && is_dup_mow_key) {
         // 2. Split segment evenly to each scanner (e.g. each scanner need to scan `avg_segment_count_per_scanner` segments)
         const auto avg_segment_count_by_scanner =
                 std::max(segment_count / config::doris_scanner_thread_pool_thread_num, (size_t)1);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 2d4e5764aad93f..92bb329b98c970 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -1099,6 +1099,10 @@ private void waitPipelineRpc(List<Triple<PipelineExecContexts, BackendServicePro
             try {
                 PExecPlanFragmentResult result = triple.getRight().get(timeoutMs, TimeUnit.MILLISECONDS);
                 code = TStatusCode.findByValue(result.getStatus().getStatusCode());
+                if (code == null) {
+                    code = TStatusCode.INTERNAL_ERROR;
+                }
+
                 if (code != TStatusCode.OK) {
                     if (!result.getStatus().getErrorMsgsList().isEmpty()) {
                         errMsg = result.getStatus().getErrorMsgsList().get(0);

From e64c5687f21c2c2775d838cbf847c81480d54944 Mon Sep 17 00:00:00 2001
From: qiye <jianliang5669@gmail.com>
Date: Thu, 28 Dec 2023 21:33:21 +0800
Subject: [PATCH 128/299] [fix](index compaction)support compact multi segments
 in one index (#28889)

---
 be/src/clucene                                |   2 +-
 be/src/common/config.cpp                      |   5 +-
 be/src/common/config.h                        |   3 +-
 be/src/olap/compaction.cpp                    |  38 ++-
 .../segment_v2/inverted_index_compaction.cpp  |   1 +
 .../segment_v2/inverted_index_writer.cpp      |   3 +-
 ...x_compaction_with_multi_index_segments.out | 214 +++++++++++++
 ...ompaction_with_multi_index_segments.groovy | 287 ++++++++++++++++++
 8 files changed, 543 insertions(+), 10 deletions(-)
 create mode 100644 regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out
 create mode 100644 regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy

diff --git a/be/src/clucene b/be/src/clucene
index df3ab39ca636e5..d05cb8154ef436 160000
--- a/be/src/clucene
+++ b/be/src/clucene
@@ -1 +1 @@
-Subproject commit df3ab39ca636e58ed1ee640921444f7ef6d6438d
+Subproject commit d05cb8154ef4368bd40c43c94e8e3c679e13c490
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index b5a16ef0166af7..6292b191f65a69 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1006,13 +1006,16 @@ DEFINE_String(inverted_index_query_cache_limit, "10%");
 
 // inverted index
 DEFINE_mDouble(inverted_index_ram_buffer_size, "512");
+// -1 indicates not working.
+// Normally we should not change this, it's useful for testing.
+DEFINE_mInt32(inverted_index_max_buffered_docs, "-1");
 // dict path for chinese analyzer
 DEFINE_String(inverted_index_dict_path, "${DORIS_HOME}/dict");
 DEFINE_Int32(inverted_index_read_buffer_size, "4096");
 // tree depth for bkd index
 DEFINE_Int32(max_depth_in_bkd_tree, "32");
 // index compaction
-DEFINE_Bool(inverted_index_compaction_enable, "false");
+DEFINE_mBool(inverted_index_compaction_enable, "false");
 // index by RAM directory
 DEFINE_mBool(inverted_index_ram_dir_enable, "false");
 // use num_broadcast_buffer blocks as buffer to do broadcast
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 53dc0a2a0d8bc9..e580b38e483a4c 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1043,13 +1043,14 @@ DECLARE_String(inverted_index_query_cache_limit);
 
 // inverted index
 DECLARE_mDouble(inverted_index_ram_buffer_size);
+DECLARE_mInt32(inverted_index_max_buffered_docs);
 // dict path for chinese analyzer
 DECLARE_String(inverted_index_dict_path);
 DECLARE_Int32(inverted_index_read_buffer_size);
 // tree depth for bkd index
 DECLARE_Int32(max_depth_in_bkd_tree);
 // index compaction
-DECLARE_Bool(inverted_index_compaction_enable);
+DECLARE_mBool(inverted_index_compaction_enable);
 // index by RAM directory
 DECLARE_mBool(inverted_index_ram_dir_enable);
 // use num_broadcast_buffer blocks as buffer to do broadcast
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 4ad78367d28014..e3fddb990cdfed 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -400,6 +400,34 @@ Status Compaction::do_compaction_impl(int64_t permits) {
 
     if (_input_row_num > 0 && stats.rowid_conversion && config::inverted_index_compaction_enable) {
         OlapStopWatch inverted_watch;
+
+        // check rowid_conversion correctness
+        Version version = _tablet->max_version();
+        DeleteBitmap output_rowset_delete_bitmap(_tablet->tablet_id());
+        std::set<RowLocation> missed_rows;
+        std::map<RowsetSharedPtr, std::list<std::pair<RowLocation, RowLocation>>> location_map;
+        // Convert the delete bitmap of the input rowsets to output rowset.
+        std::size_t missed_rows_size = 0;
+        _tablet->calc_compaction_output_rowset_delete_bitmap(
+                _input_rowsets, _rowid_conversion, 0, version.second + 1, &missed_rows,
+                &location_map, _tablet->tablet_meta()->delete_bitmap(),
+                &output_rowset_delete_bitmap);
+        if (!allow_delete_in_cumu_compaction()) {
+            missed_rows_size = missed_rows.size();
+            if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION &&
+                stats.merged_rows != missed_rows_size) {
+                std::string err_msg = fmt::format(
+                        "cumulative compaction: the merged rows({}) is not equal to missed "
+                        "rows({}) in rowid conversion, tablet_id: {}, table_id:{}",
+                        stats.merged_rows, missed_rows_size, _tablet->tablet_id(),
+                        _tablet->table_id());
+                DCHECK(false) << err_msg;
+                LOG(WARNING) << err_msg;
+            }
+        }
+
+        RETURN_IF_ERROR(_tablet->check_rowid_conversion(_output_rowset, location_map));
+
         // translation vec
         // <<dest_idx_num, dest_docId>>
         // the first level vector: index indicates src segment.
@@ -425,7 +453,7 @@ Status Compaction::do_compaction_impl(int64_t permits) {
             // src index files
             // format: rowsetId_segmentId
             std::vector<std::string> src_index_files(src_segment_num);
-            for (auto m : src_seg_to_id_map) {
+            for (const auto& m : src_seg_to_id_map) {
                 std::pair<RowsetId, uint32_t> p = m.first;
                 src_index_files[m.second] = p.first.to_string() + "_" + std::to_string(p.second);
             }
@@ -676,11 +704,11 @@ Status Compaction::modify_rowsets(const Merger::Statistics* stats) {
         // of incremental data later.
         // TODO(LiaoXin): check if there are duplicate keys
         std::size_t missed_rows_size = 0;
+        _tablet->calc_compaction_output_rowset_delete_bitmap(
+                _input_rowsets, _rowid_conversion, 0, version.second + 1, &missed_rows,
+                &location_map, _tablet->tablet_meta()->delete_bitmap(),
+                &output_rowset_delete_bitmap);
         if (!allow_delete_in_cumu_compaction()) {
-            _tablet->calc_compaction_output_rowset_delete_bitmap(
-                    _input_rowsets, _rowid_conversion, 0, version.second + 1, &missed_rows,
-                    &location_map, _tablet->tablet_meta()->delete_bitmap(),
-                    &output_rowset_delete_bitmap);
             missed_rows_size = missed_rows.size();
             if (compaction_type() == ReaderType::READER_CUMULATIVE_COMPACTION && stats != nullptr &&
                 stats->merged_rows != missed_rows_size) {
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
index 677d359f1a4172..f34a36c7baebec 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_compaction.cpp
@@ -55,6 +55,7 @@ Status compact_column(int32_t index_id, int src_segment_num, int dest_segment_nu
         dest_index_dirs[i] = DorisCompoundDirectoryFactory::getDirectory(fs, path.c_str(), true);
     }
 
+    DCHECK_EQ(src_index_dirs.size(), trans_vec.size());
     index_writer->indexCompaction(src_index_dirs, dest_index_dirs, trans_vec,
                                   dest_segment_num_rows);
 
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
index a43049aaa2bc34..e6e519b9d2cb75 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_writer.cpp
@@ -67,7 +67,6 @@
 
 namespace doris::segment_v2 {
 const int32_t MAX_FIELD_LEN = 0x7FFFFFFFL;
-const int32_t MAX_BUFFER_DOCS = 100000000;
 const int32_t MERGE_FACTOR = 100000000;
 const int32_t MAX_LEAF_COUNT = 1024;
 const float MAXMBSortInHeap = 512.0 * 8;
@@ -196,8 +195,8 @@ class InvertedIndexColumnWriterImpl : public InvertedIndexColumnWriter {
         bool close_dir_on_shutdown = true;
         index_writer = std::make_unique<lucene::index::IndexWriter>(
                 _dir.get(), _analyzer.get(), create_index, close_dir_on_shutdown);
-        index_writer->setMaxBufferedDocs(MAX_BUFFER_DOCS);
         index_writer->setRAMBufferSizeMB(config::inverted_index_ram_buffer_size);
+        _index_writer->setMaxBufferedDocs(config::inverted_index_max_buffered_docs);
         index_writer->setMaxFieldLength(MAX_FIELD_LEN);
         index_writer->setMergeFactor(MERGE_FACTOR);
         index_writer->setUseCompoundFile(false);
diff --git a/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out
new file mode 100644
index 00000000000000..57ad3c1080c604
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out
@@ -0,0 +1,214 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+
+-- !sql --
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+
+-- !sql --
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	8	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	9	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+2018-02-21T12:00	10	I'm using the builds
+
+-- !sql --
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	2	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	3	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	4	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	5	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	6	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+2018-02-21T12:00	7	I'm using the builds
+
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
new file mode 100644
index 00000000000000..765ee55515a420
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
@@ -0,0 +1,287 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_index_compaction_with_multi_index_segments", "p0") {
+    def tableName = "test_index_compaction_with_multi_index_segments"
+  
+    def set_be_config = { key, value ->
+        def backendId_to_backendIP = [:]
+        def backendId_to_backendHttpPort = [:]
+        getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+        }
+    }
+
+    boolean disableAutoCompaction = true
+    boolean invertedIndexCompactionEnable = false
+    int invertedIndexMaxBufferedDocs = -1;
+    boolean has_update_be_config = false
+
+    try {
+        String backend_id;
+        def backendId_to_backendIP = [:]
+        def backendId_to_backendHttpPort = [:]
+        getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+        backend_id = backendId_to_backendIP.keySet()[0]
+        def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+        
+        logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def configList = parseJson(out.trim())
+        assert configList instanceof List
+
+        for (Object ele in (List) configList) {
+            assert ele instanceof List<String>
+            if (((List<String>) ele)[0] == "inverted_index_compaction_enable") {
+                invertedIndexCompactionEnable = Boolean.parseBoolean(((List<String>) ele)[2])
+                logger.info("inverted_index_compaction_enable: ${((List<String>) ele)[2]}")
+            }
+            if (((List<String>) ele)[0] == "inverted_index_max_buffered_docs") {
+                invertedIndexMaxBufferedDocs = Integer.parseInt(((List<String>) ele)[2])
+                logger.info("inverted_index_max_buffered_docs: ${((List<String>) ele)[2]}")
+            }
+        }
+        set_be_config.call("inverted_index_compaction_enable", "true")
+        set_be_config.call("inverted_index_max_buffered_docs", "5")
+        has_update_be_config = true
+
+        sql """ DROP TABLE IF EXISTS ${tableName}; """
+        sql """
+            CREATE TABLE ${tableName} (
+                `file_time` DATETIME NOT NULL,
+                `comment_id` int(11)  NULL,
+                `body` TEXT NULL DEFAULT "",
+                INDEX idx_comment_id (`comment_id`) USING INVERTED COMMENT '''',
+                INDEX idx_body (`body`) USING INVERTED PROPERTIES("parser" = "unicode") COMMENT ''''
+            ) ENGINE=OLAP
+            DUPLICATE KEY(`file_time`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY RANDOM BUCKETS 1
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "disable_auto_compaction" = "true"
+            );
+        """
+
+        // insert 10 rows
+        sql """ INSERT INTO ${tableName} VALUES ("2018-02-21 12:00:00", 1, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 2, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 3, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 4, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 5, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 6, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 7, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 8, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 9, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 10, "I\'m using the builds"); """
+        // insert another 10 rows
+        sql """ INSERT INTO ${tableName} VALUES ("2018-02-21 12:00:00", 1, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 2, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 3, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 4, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 5, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 6, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 7, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 8, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 9, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 10, "I\'m using the builds"); """
+
+        qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "using" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "the" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where comment_id < 8 order by file_time, comment_id, body """
+
+        //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+        String[][] tablets = sql """ show tablets from ${tableName}; """
+
+        def replicaNum = get_table_replica_num(tableName)
+        logger.info("get table replica num: " + replicaNum)
+        // before full compaction, there are 3 rowsets.
+        int rowsetCount = 0
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            def compactionStatusUrlIndex = 18
+            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        assert (rowsetCount == 3 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            backend_id = tablet[2]
+            times = 1
+
+            do{
+                (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+                ++times
+                sleep(2000)
+            } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10)
+
+            def compactJson = parseJson(out.trim())
+            if (compactJson.status.toLowerCase() == "fail") {
+                assertEquals(disableAutoCompaction, false)
+                logger.info("Compaction was done automatically!")
+            }
+            if (disableAutoCompaction) {
+                assertEquals("success", compactJson.status.toLowerCase())
+            }
+        }
+
+        // wait for full compaction done
+        for (String[] tablet in tablets) {
+            boolean running = true
+            do {
+                Thread.sleep(1000)
+                String tablet_id = tablet[0]
+                backend_id = tablet[2]
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("success", compactionStatus.status.toLowerCase())
+                running = compactionStatus.run_status
+            } while (running)
+        }
+
+        // after full compaction, there is only 1 rowset.
+        
+        rowsetCount = 0
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            def compactionStatusUrlIndex = 18
+            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "using" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "the" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where comment_id < 8 order by file_time, comment_id, body """
+
+        // insert 10 rows, again
+        sql """ INSERT INTO ${tableName} VALUES ("2018-02-21 12:00:00", 1, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 2, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 3, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 4, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 5, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 6, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 7, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 8, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 9, "I\'m using the builds"),
+                                                ("2018-02-21 12:00:00", 10, "I\'m using the builds"); """
+
+        tablets = sql """ show tablets from ${tableName}; """
+
+        replicaNum = get_table_replica_num(tableName)
+        logger.info("get table replica num: " + replicaNum)
+        // before full compaction, there are 2 rowsets.
+        rowsetCount = 0
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            def compactionStatusUrlIndex = 18
+            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        assert (rowsetCount == 2 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            backend_id = tablet[2]
+            times = 1
+
+            do{
+                (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+                ++times
+                sleep(2000)
+            } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10)
+
+            def compactJson = parseJson(out.trim())
+            if (compactJson.status.toLowerCase() == "fail") {
+                assertEquals(disableAutoCompaction, false)
+                logger.info("Compaction was done automatically!")
+            }
+            if (disableAutoCompaction) {
+                assertEquals("success", compactJson.status.toLowerCase())
+            }
+        }
+
+        // wait for full compaction done
+        for (String[] tablet in tablets) {
+            boolean running = true
+            do {
+                Thread.sleep(1000)
+                String tablet_id = tablet[0]
+                backend_id = tablet[2]
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("success", compactionStatus.status.toLowerCase())
+                running = compactionStatus.run_status
+            } while (running)
+        }
+
+        // after full compaction, there is only 1 rowset.
+        
+        rowsetCount = 0
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            def compactionStatusUrlIndex = 18
+            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "using" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "the" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where comment_id < 8 order by file_time, comment_id, body """
+
+    } finally {
+        if (has_update_be_config) {
+            set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString())
+            set_be_config.call("inverted_index_max_buffered_docs", invertedIndexMaxBufferedDocs.toString())
+        }
+    }
+}
\ No newline at end of file

From 8a491e7b1d7d3b2c966ab332c226128dc07fc522 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@apache.org>
Date: Thu, 28 Dec 2023 22:41:42 +0800
Subject: [PATCH 129/299] Fix workload scheduler start too early may cause npe
 (#29258)

---
 be/src/runtime/exec_env_init.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 48d229b9e21961..c204ead069d120 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -196,9 +196,6 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
                               .set_max_queue_size(1000000)
                               .build(&_lazy_release_obj_pool));
 
-    _workload_sched_mgr = new WorkloadSchedPolicyMgr();
-    _workload_sched_mgr->start(this);
-
     init_file_cache_factory();
     RETURN_IF_ERROR(init_pipeline_task_scheduler());
     _task_group_manager = new taskgroup::TaskGroupManager();
@@ -275,6 +272,9 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
         return st;
     }
 
+    _workload_sched_mgr = new WorkloadSchedPolicyMgr();
+    _workload_sched_mgr->start(this);
+
     _s_ready = true;
 
     return Status::OK();

From a90304c208cf6a5d982e7db950b2654dbeda7b30 Mon Sep 17 00:00:00 2001
From: Ashin Gau <AshinGau@users.noreply.github.com>
Date: Thu, 28 Dec 2023 22:43:11 +0800
Subject: [PATCH 130/299] [fix](parquet) complex type in parquet is case
 sensitive (#29245)

Change name of complex type in parquet to case-insensitive. Otherwise, uppercase column names of complex types will return null.
---
 be/src/vec/exec/format/parquet/schema_desc.cpp | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/be/src/vec/exec/format/parquet/schema_desc.cpp b/be/src/vec/exec/format/parquet/schema_desc.cpp
index c9283c628890b6..08692de8743c06 100644
--- a/be/src/vec/exec/format/parquet/schema_desc.cpp
+++ b/be/src/vec/exec/format/parquet/schema_desc.cpp
@@ -167,9 +167,7 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
         auto child = &node_field->children[0];
         parse_physical_field(t_schema, false, child);
 
-        std::string lower_case_name;
-        transform(t_schema.name.begin(), t_schema.name.end(), lower_case_name.begin(), ::tolower);
-        node_field->name = lower_case_name;
+        node_field->name = to_lower(t_schema.name);
         node_field->type.type = TYPE_ARRAY;
         node_field->type.add_sub_type(child->type);
         node_field->is_nullable = false;
@@ -187,9 +185,7 @@ Status FieldDescriptor::parse_node_field(const std::vector<tparquet::SchemaEleme
 
 void FieldDescriptor::parse_physical_field(const tparquet::SchemaElement& physical_schema,
                                            bool is_nullable, FieldSchema* physical_field) {
-    std::string lower_case_name = physical_schema.name;
-    transform(lower_case_name.begin(), lower_case_name.end(), lower_case_name.begin(), ::tolower);
-    physical_field->name = lower_case_name;
+    physical_field->name = to_lower(physical_schema.name);
     physical_field->parquet_schema = physical_schema;
     physical_field->is_nullable = is_nullable;
     physical_field->physical_type = physical_schema.type;
@@ -443,7 +439,7 @@ Status FieldDescriptor::parse_group_field(const std::vector<tparquet::SchemaElem
         // produce a non-null list<struct>
         RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos, struct_field));
 
-        group_field->name = group_schema.name;
+        group_field->name = to_lower(group_schema.name);
         group_field->type.type = TYPE_ARRAY;
         group_field->type.add_sub_type(struct_field->type);
         group_field->is_nullable = false;
@@ -511,7 +507,7 @@ Status FieldDescriptor::parse_list_field(const std::vector<tparquet::SchemaEleme
         _next_schema_pos = curr_pos + 2;
     }
 
-    list_field->name = first_level.name;
+    list_field->name = to_lower(first_level.name);
     list_field->type.type = TYPE_ARRAY;
     list_field->type.add_sub_type(list_field->children[0].type);
     list_field->is_nullable = is_optional;
@@ -574,7 +570,7 @@ Status FieldDescriptor::parse_map_field(const std::vector<tparquet::SchemaElemen
     // produce MAP<STRUCT<KEY, VALUE>>
     RETURN_IF_ERROR(parse_struct_field(t_schemas, curr_pos + 1, map_kv_field));
 
-    map_field->name = map_schema.name;
+    map_field->name = to_lower(map_schema.name);
     map_field->type.type = TYPE_MAP;
     map_field->type.add_sub_type(map_kv_field->type.children[0]);
     map_field->type.add_sub_type(map_kv_field->type.children[1]);
@@ -598,7 +594,7 @@ Status FieldDescriptor::parse_struct_field(const std::vector<tparquet::SchemaEle
     for (int i = 0; i < num_children; ++i) {
         RETURN_IF_ERROR(parse_node_field(t_schemas, _next_schema_pos, &struct_field->children[i]));
     }
-    struct_field->name = struct_schema.name;
+    struct_field->name = to_lower(struct_schema.name);
     struct_field->is_nullable = is_optional;
     struct_field->type.type = TYPE_STRUCT;
     for (int i = 0; i < num_children; ++i) {

From feebe3e6fbe05a57d38535efbafca825bfc68475 Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Thu, 28 Dec 2023 23:08:01 +0800
Subject: [PATCH 131/299] [FIX](literal) fix expression literal error #29157

---
 .../doris/analysis/ExpressionFunctions.java   |  4 +++-
 .../literal_view/lietral_largeint.out         | 10 +++++++++
 .../literal_view/lietral_largeint.groovy      | 22 +++++++++++++++++++
 3 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/query_p0/literal_view/lietral_largeint.out
 create mode 100644 regression-test/suites/query_p0/literal_view/lietral_largeint.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java
index ea63a8ffabeac3..0ba9122373eed5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExpressionFunctions.java
@@ -278,8 +278,10 @@ private LiteralExpr[] createVariableLengthArgs(List<Expr> args, int typeIndex) t
             LiteralExpr[] exprs;
             if (argType.isStringType()) {
                 exprs = new StringLiteral[args.size()];
-            } else if (argType.isFixedPointType()) {
+            } else if (argType.isIntegerType()) {
                 exprs = new IntLiteral[args.size()];
+            } else if (argType.isLargeIntType()) {
+                exprs = new LargeIntLiteral[args.size()];
             } else if (argType.isDateType()) {
                 exprs = new DateLiteral[args.size()];
             } else if (argType.isDecimalV2() || argType.isDecimalV3()) {
diff --git a/regression-test/data/query_p0/literal_view/lietral_largeint.out b/regression-test/data/query_p0/literal_view/lietral_largeint.out
new file mode 100644
index 00000000000000..438ff77153965a
--- /dev/null
+++ b/regression-test/data/query_p0/literal_view/lietral_largeint.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+[1]
+
+-- !sql --
+[1, null]
+
+-- !sql --
+[1, null, 18446744073709551615]
+
diff --git a/regression-test/suites/query_p0/literal_view/lietral_largeint.groovy b/regression-test/suites/query_p0/literal_view/lietral_largeint.groovy
new file mode 100644
index 00000000000000..7ba7e048b8fdcc
--- /dev/null
+++ b/regression-test/suites/query_p0/literal_view/lietral_largeint.groovy
@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("literal_largeint") {
+    qt_sql "select array(cast(1 as LargeInt))"
+    qt_sql "select array(cast(1 as LargeInt), NULL)"
+    qt_sql "select array(cast(1 as LargeInt), NULL, cast(18446744073709551615 as LargeInt))"
+}

From 9a277a6f11daeb612012babd95fbc0aa73ee5720 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Fri, 29 Dec 2023 00:03:28 +0800
Subject: [PATCH 132/299] [fix](move-memtable) don't abort in replica write
 layer unless all replica fails (#29257)

---
 be/src/io/fs/stream_sink_file_writer.cpp | 33 +++++++++++++++++++++---
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/be/src/io/fs/stream_sink_file_writer.cpp b/be/src/io/fs/stream_sink_file_writer.cpp
index 25a4f5b27d5bb0..2fce7c6baa9c64 100644
--- a/be/src/io/fs/stream_sink_file_writer.cpp
+++ b/be/src/io/fs/stream_sink_file_writer.cpp
@@ -51,9 +51,22 @@ Status StreamSinkFileWriter::appendv(const Slice* data, size_t data_cnt) {
                << ", data_length: " << bytes_req;
 
     std::span<const Slice> slices {data, data_cnt};
+    bool ok = false;
     for (auto& stream : _streams) {
-        RETURN_IF_ERROR(stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id,
-                                            _bytes_appended, slices));
+        auto st = stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id,
+                                      _bytes_appended, slices);
+        ok = ok || st.ok();
+    }
+    if (!ok) {
+        std::stringstream ss;
+        for (auto& stream : _streams) {
+            ss << " " << stream->dst_id();
+        }
+        LOG(WARNING) << "failed to write any replicas, load_id: " << print_id(_load_id)
+                     << ", index_id: " << _index_id << ", tablet_id: " << _tablet_id
+                     << ", segment_id: " << _segment_id << ", data_length: " << bytes_req
+                     << ", backends:" << ss.str();
+        return Status::InternalError("failed to write any replicas");
     }
     _bytes_appended += bytes_req;
     return Status::OK();
@@ -63,9 +76,21 @@ Status StreamSinkFileWriter::finalize() {
     VLOG_DEBUG << "writer finalize, load_id: " << print_id(_load_id) << ", index_id: " << _index_id
                << ", tablet_id: " << _tablet_id << ", segment_id: " << _segment_id;
     // TODO(zhengyu): update get_inverted_index_file_size into stat
+    bool ok = false;
     for (auto& stream : _streams) {
-        RETURN_IF_ERROR(stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id,
-                                            _bytes_appended, {}, true));
+        auto st = stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id,
+                                      _bytes_appended, {}, true);
+        ok = ok || st.ok();
+    }
+    if (!ok) {
+        std::stringstream ss;
+        for (auto& stream : _streams) {
+            ss << " " << stream->dst_id();
+        }
+        LOG(WARNING) << "failed to finalize any replicas, load_id: " << print_id(_load_id)
+                     << ", index_id: " << _index_id << ", tablet_id: " << _tablet_id
+                     << ", segment_id: " << _segment_id << ", backends:" << ss.str();
+        return Status::InternalError("failed to finalize any replicas");
     }
     return Status::OK();
 }

From 9be0f0450662835d00674fc2c090ecc0af63cbed Mon Sep 17 00:00:00 2001
From: meiyi <myimeiyi@gmail.com>
Date: Fri, 29 Dec 2023 00:26:10 +0800
Subject: [PATCH 133/299] (improv)[group commit] refactor some group commit
 code (#29180)

---
 be/src/common/config.cpp                      |  17 ++-
 be/src/common/config.h                        |  17 ++-
 be/src/http/action/http_stream.cpp            |  91 +++++++++-------
 be/src/http/action/http_stream.h              |   5 +-
 be/src/http/action/stream_load.cpp            | 100 ++++++++++--------
 be/src/http/action/stream_load.h              |   2 +-
 be/src/runtime/group_commit_mgr.h             |   1 -
 be/src/vec/sink/group_commit_block_sink.cpp   |  30 +++---
 be/src/vec/sink/group_commit_block_sink.h     |   4 +-
 .../doris/analysis/NativeInsertStmt.java      |   7 ++
 .../doris/planner/GroupCommitBlockSink.java   |   2 +
 .../org/apache/doris/qe/SessionVariable.java  |   5 +-
 .../org/apache/doris/qe/StmtExecutor.java     |   5 +
 .../insert_p0/insert_group_commit_into.groovy |   5 +-
 ...sert_group_commit_with_prepare_stmt.groovy |  51 ++++++---
 15 files changed, 192 insertions(+), 150 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 6292b191f65a69..6ebfafce53ce50 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1103,16 +1103,20 @@ DEFINE_Int32(grace_shutdown_wait_seconds, "120");
 
 DEFINE_Int16(bitmap_serialize_version, "1");
 
-// group commit insert config
+// group commit config
 DEFINE_String(group_commit_wal_path, "");
 DEFINE_Int32(group_commit_replay_wal_retry_num, "10");
 DEFINE_Int32(group_commit_replay_wal_retry_interval_seconds, "5");
 DEFINE_Int32(group_commit_relay_wal_threads, "10");
-
-// the count of thread to group commit insert
+// This config can be set to limit thread number in group commit request fragment thread pool.
 DEFINE_Int32(group_commit_insert_threads, "10");
 DEFINE_Int32(group_commit_memory_rows_for_max_filter_ratio, "10000");
 DEFINE_Bool(wait_internal_group_commit_finish, "false");
+// Max size(bytes) of group commit queues, used for mem back pressure, defult 64M.
+DEFINE_Int32(group_commit_max_queue_size, "67108864");
+// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
+// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
+DEFINE_String(group_commit_wal_max_disk_limit, "10%");
 
 DEFINE_mInt32(scan_thread_nice_value, "0");
 DEFINE_mInt32(tablet_schema_cache_recycle_interval, "3600");
@@ -1130,13 +1134,6 @@ DEFINE_Bool(ignore_always_true_predicate_for_segment, "true");
 // Dir of default timezone files
 DEFINE_String(default_tzfiles_path, "${DORIS_HOME}/zoneinfo");
 
-// Max size(bytes) of group commit queues, used for mem back pressure, defult 64M.
-DEFINE_Int32(group_commit_max_queue_size, "67108864");
-
-// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
-// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
-DEFINE_String(group_commit_wal_max_disk_limit, "10%");
-
 // Ingest binlog work pool size, -1 is disable, 0 is hardware concurrency
 DEFINE_Int32(ingest_binlog_work_pool_size, "-1");
 
diff --git a/be/src/common/config.h b/be/src/common/config.h
index e580b38e483a4c..4e09bb2b08bfab 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1168,16 +1168,20 @@ DECLARE_Int32(grace_shutdown_wait_seconds);
 // BitmapValue serialize version.
 DECLARE_Int16(bitmap_serialize_version);
 
-// group commit insert config
+// group commit config
 DECLARE_String(group_commit_wal_path);
 DECLARE_Int32(group_commit_replay_wal_retry_num);
 DECLARE_Int32(group_commit_replay_wal_retry_interval_seconds);
 DECLARE_mInt32(group_commit_relay_wal_threads);
-
-// This config can be set to limit thread number in group commit insert thread pool.
+// This config can be set to limit thread number in group commit request fragment thread pool.
 DECLARE_mInt32(group_commit_insert_threads);
 DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio);
 DECLARE_Bool(wait_internal_group_commit_finish);
+// Max size(bytes) of group commit queues, used for mem back pressure.
+DECLARE_Int32(group_commit_max_queue_size);
+// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
+// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
+DECLARE_mString(group_commit_wal_max_disk_limit);
 
 // The configuration item is used to lower the priority of the scanner thread,
 // typically employed to ensure CPU scheduling for write operations.
@@ -1203,13 +1207,6 @@ DECLARE_Bool(ignore_always_true_predicate_for_segment);
 // Dir of default timezone files
 DECLARE_String(default_tzfiles_path);
 
-// Max size(bytes) of group commit queues, used for mem back pressure.
-DECLARE_Int32(group_commit_max_queue_size);
-
-// Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
-// group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
-DECLARE_mString(group_commit_wal_max_disk_limit);
-
 // Ingest binlog work pool size
 DECLARE_Int32(ingest_binlog_work_pool_size);
 
diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp
index 18256237a81107..a18b10f491fc1a 100644
--- a/be/src/http/action/http_stream.cpp
+++ b/be/src/http/action/http_stream.cpp
@@ -167,45 +167,8 @@ int HttpStreamAction::on_header(HttpRequest* req) {
 
     ctx->load_type = TLoadType::MANUL_LOAD;
     ctx->load_src_type = TLoadSourceType::RAW;
-
-    Status st = Status::OK();
-    std::string group_commit_mode = req->header(HTTP_GROUP_COMMIT);
-    if (!group_commit_mode.empty() && !iequal(group_commit_mode, "sync_mode") &&
-        !iequal(group_commit_mode, "async_mode") && !iequal(group_commit_mode, "off_mode")) {
-        st = Status::InternalError("group_commit can only be [async_mode, sync_mode, off_mode]");
-    } else if (group_commit_mode.empty() || iequal(group_commit_mode, "off_mode")) {
-        // off_mode and empty
-        group_commit_mode = "off_mode";
-        ctx->group_commit = false;
-    } else {
-        // sync_mode and async_mode
-        ctx->group_commit = true;
-    }
     ctx->two_phase_commit = req->header(HTTP_TWO_PHASE_COMMIT) == "true";
-    auto temp_partitions = !req->header(HTTP_TEMP_PARTITIONS).empty();
-    auto partitions = !req->header(HTTP_PARTITIONS).empty();
-    if (!temp_partitions && !partitions && !ctx->two_phase_commit &&
-        (!group_commit_mode.empty() || config::wait_internal_group_commit_finish)) {
-        if (iequal(group_commit_mode, "async_mode") || config::wait_internal_group_commit_finish) {
-            ctx->group_commit = true;
-            group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
-            if (iequal(group_commit_mode, "sync_mode")) {
-                size_t max_available_size =
-                        ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
-                LOG(INFO) << "When enable group commit, the data size can't be too large. The data "
-                             "size "
-                             "for this http load("
-                          << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
-                                      ? 0
-                                      : std::stol(req->header(HttpHeaders::CONTENT_LENGTH)))
-                          << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
-                          << max_available_size
-                          << " Bytes). So we set this load to \"group commit\"=sync_mode "
-                             "automatically.";
-                st = Status::Error<EXCEEDED_LIMIT>("Http load size too large.");
-            }
-        }
-    }
+    Status st = _handle_group_commit(req, ctx);
 
     LOG(INFO) << "new income streaming load request." << ctx->brief()
               << " sql : " << req->header(HTTP_SQL) << ", group_commit=" << ctx->group_commit;
@@ -338,6 +301,7 @@ Status HttpStreamAction::_process_put(HttpRequest* http_req,
         if (!http_req->header(HTTP_GROUP_COMMIT).empty()) {
             request.__set_group_commit_mode(http_req->header(HTTP_GROUP_COMMIT));
         } else {
+            // used for wait_internal_group_commit_finish
             request.__set_group_commit_mode("sync_mode");
         }
     }
@@ -366,10 +330,9 @@ Status HttpStreamAction::_process_put(HttpRequest* http_req,
     ctx->txn_id = ctx->put_result.params.txn_conf.txn_id;
     ctx->label = ctx->put_result.params.import_label;
     ctx->put_result.params.__set_wal_id(ctx->wal_id);
-    if (http_req->header(HTTP_GROUP_COMMIT) == "async mode") {
+    if (http_req->header(HTTP_GROUP_COMMIT) == "async_mode") {
         if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
-            size_t content_length = 0;
-            content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
+            size_t content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
             if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
                 ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
                 ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
@@ -403,4 +366,50 @@ void HttpStreamAction::_save_stream_load_record(std::shared_ptr<StreamLoadContex
     }
 }
 
+Status HttpStreamAction::_handle_group_commit(HttpRequest* req,
+                                              std::shared_ptr<StreamLoadContext> ctx) {
+    std::string group_commit_mode = req->header(HTTP_GROUP_COMMIT);
+    if (!group_commit_mode.empty() && !iequal(group_commit_mode, "sync_mode") &&
+        !iequal(group_commit_mode, "async_mode") && !iequal(group_commit_mode, "off_mode")) {
+        return Status::InternalError("group_commit can only be [async_mode, sync_mode, off_mode]");
+    }
+    if (config::wait_internal_group_commit_finish) {
+        group_commit_mode = "sync_mode";
+    }
+    if (group_commit_mode.empty() || iequal(group_commit_mode, "off_mode")) {
+        // off_mode and empty
+        ctx->group_commit = false;
+        return Status::OK();
+    }
+
+    auto partial_columns = !req->header(HTTP_PARTIAL_COLUMNS).empty() &&
+                           iequal(req->header(HTTP_PARTIAL_COLUMNS), "true");
+    auto temp_partitions = !req->header(HTTP_TEMP_PARTITIONS).empty();
+    auto partitions = !req->header(HTTP_PARTITIONS).empty();
+    if (!partial_columns && !partitions && !temp_partitions && !ctx->two_phase_commit) {
+        if (!config::wait_internal_group_commit_finish && !ctx->label.empty()) {
+            return Status::InternalError("label and group_commit can't be set at the same time");
+        }
+        ctx->group_commit = true;
+        if (iequal(group_commit_mode, "async_mode")) {
+            group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
+            if (iequal(group_commit_mode, "sync_mode")) {
+                size_t max_available_size =
+                        ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
+                LOG(INFO) << "When enable group commit, the data size can't be too large or "
+                             "unknown. The data size for this stream load("
+                          << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
+                                      ? 0
+                                      : req->header(HttpHeaders::CONTENT_LENGTH))
+                          << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
+                          << max_available_size
+                          << " Bytes). So we set this load to \"group commit\"=sync_mode\" "
+                             "automatically.";
+                return Status::Error<EXCEEDED_LIMIT>("Http load size too large.");
+            }
+        }
+    }
+    return Status::OK();
+}
+
 } // namespace doris
diff --git a/be/src/http/action/http_stream.h b/be/src/http/action/http_stream.h
index 192e68257eaf18..d4140a118d61e3 100644
--- a/be/src/http/action/http_stream.h
+++ b/be/src/http/action/http_stream.h
@@ -47,12 +47,9 @@ class HttpStreamAction : public HttpHandler {
 private:
     Status _on_header(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
     Status _handle(HttpRequest* req, std::shared_ptr<StreamLoadContext> ctx);
-    Status _data_saved_path(HttpRequest* req, std::string* file_path);
     Status _process_put(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
     void _save_stream_load_record(std::shared_ptr<StreamLoadContext> ctx, const std::string& str);
-    void _parse_format(const std::string& format_str, const std::string& compress_type_str,
-                       TFileFormatType::type* format_type, TFileCompressType::type* compress_type);
-    bool _is_format_support_streaming(TFileFormatType::type format);
+    Status _handle_group_commit(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
 
 private:
     ExecEnv* _exec_env;
diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index eddf31856a03d5..a702be65d05523 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -188,50 +188,8 @@ int StreamLoadAction::on_header(HttpRequest* req) {
     url_decode(req->param(HTTP_DB_KEY), &ctx->db);
     url_decode(req->param(HTTP_TABLE_KEY), &ctx->table);
     ctx->label = req->header(HTTP_LABEL_KEY);
-    Status st = Status::OK();
-    std::string group_commit_mode = req->header(HTTP_GROUP_COMMIT);
-    if (!group_commit_mode.empty() && !iequal(group_commit_mode, "sync_mode") &&
-        !iequal(group_commit_mode, "async_mode") && !iequal(group_commit_mode, "off_mode")) {
-        st = Status::InternalError("group_commit can only be [async_mode, sync_mode, off_mode]");
-    } else if (group_commit_mode.empty() || iequal(group_commit_mode, "off_mode")) {
-        // off_mode and empty
-        group_commit_mode = "off_mode";
-        ctx->group_commit = false;
-    } else {
-        // sync_mode and async_mode
-        ctx->group_commit = true;
-    }
-    auto partial_columns = !req->header(HTTP_PARTIAL_COLUMNS).empty() &&
-                           iequal(req->header(HTTP_PARTIAL_COLUMNS), "true");
     ctx->two_phase_commit = req->header(HTTP_TWO_PHASE_COMMIT) == "true";
-    auto temp_partitions = !req->header(HTTP_TEMP_PARTITIONS).empty();
-    auto partitions = !req->header(HTTP_PARTITIONS).empty();
-    if (!partial_columns && !partitions && !temp_partitions && !ctx->two_phase_commit &&
-        (!group_commit_mode.empty() || config::wait_internal_group_commit_finish)) {
-        if (!config::wait_internal_group_commit_finish && ctx->group_commit &&
-            !ctx->label.empty()) {
-            st = Status::InternalError("label and group_commit can't be set at the same time");
-        }
-        if (iequal(group_commit_mode, "async_mode") || config::wait_internal_group_commit_finish) {
-            ctx->group_commit = true;
-            group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
-            if (iequal(group_commit_mode, "sync_mode")) {
-                size_t max_available_size =
-                        ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
-                LOG(INFO) << "When enable group commit, the data size can't be too large. The data "
-                             "size "
-                             "for this stream load("
-                          << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
-                                      ? 0
-                                      : std::stol(req->header(HttpHeaders::CONTENT_LENGTH)))
-                          << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
-                          << max_available_size
-                          << " Bytes). So we set this load to \"group commit\"=sync_mode "
-                             "automatically.";
-                st = Status::Error<EXCEEDED_LIMIT>("Stream load size too large.");
-            }
-        }
-    }
+    Status st = _handle_group_commit(req, ctx);
     if (!ctx->group_commit && ctx->label.empty()) {
         ctx->label = generate_uuid_string();
     }
@@ -649,7 +607,12 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
         request.__set_memtable_on_sink_node(value);
     }
     if (ctx->group_commit) {
-        request.__set_group_commit_mode(http_req->header(HTTP_GROUP_COMMIT));
+        if (!http_req->header(HTTP_GROUP_COMMIT).empty()) {
+            request.__set_group_commit_mode(http_req->header(HTTP_GROUP_COMMIT));
+        } else {
+            // used for wait_internal_group_commit_finish
+            request.__set_group_commit_mode("sync_mode");
+        }
     }
 
 #ifndef BE_TEST
@@ -672,8 +635,7 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
     }
     if (http_req->header(HTTP_GROUP_COMMIT) == "async_mode") {
         if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
-            size_t content_length = 0;
-            content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
+            size_t content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
             if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
                 ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
                 ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
@@ -730,4 +692,50 @@ void StreamLoadAction::_save_stream_load_record(std::shared_ptr<StreamLoadContex
     }
 }
 
+Status StreamLoadAction::_handle_group_commit(HttpRequest* req,
+                                              std::shared_ptr<StreamLoadContext> ctx) {
+    std::string group_commit_mode = req->header(HTTP_GROUP_COMMIT);
+    if (!group_commit_mode.empty() && !iequal(group_commit_mode, "sync_mode") &&
+        !iequal(group_commit_mode, "async_mode") && !iequal(group_commit_mode, "off_mode")) {
+        return Status::InternalError("group_commit can only be [async_mode, sync_mode, off_mode]");
+    }
+    if (config::wait_internal_group_commit_finish) {
+        group_commit_mode = "sync_mode";
+    }
+    if (group_commit_mode.empty() || iequal(group_commit_mode, "off_mode")) {
+        // off_mode and empty
+        ctx->group_commit = false;
+        return Status::OK();
+    }
+
+    auto partial_columns = !req->header(HTTP_PARTIAL_COLUMNS).empty() &&
+                           iequal(req->header(HTTP_PARTIAL_COLUMNS), "true");
+    auto temp_partitions = !req->header(HTTP_TEMP_PARTITIONS).empty();
+    auto partitions = !req->header(HTTP_PARTITIONS).empty();
+    if (!partial_columns && !partitions && !temp_partitions && !ctx->two_phase_commit) {
+        if (!config::wait_internal_group_commit_finish && !ctx->label.empty()) {
+            return Status::InternalError("label and group_commit can't be set at the same time");
+        }
+        ctx->group_commit = true;
+        if (iequal(group_commit_mode, "async_mode")) {
+            group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
+            if (iequal(group_commit_mode, "sync_mode")) {
+                size_t max_available_size =
+                        ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
+                LOG(INFO) << "When enable group commit, the data size can't be too large or "
+                             "unknown. The data size for this stream load("
+                          << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
+                                      ? 0
+                                      : req->header(HttpHeaders::CONTENT_LENGTH))
+                          << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
+                          << max_available_size
+                          << " Bytes). So we set this load to \"group commit\"=sync_mode\" "
+                             "automatically.";
+                return Status::Error<EXCEEDED_LIMIT>("Stream load size too large.");
+            }
+        }
+    }
+    return Status::OK();
+}
+
 } // namespace doris
diff --git a/be/src/http/action/stream_load.h b/be/src/http/action/stream_load.h
index eac3825f534258..d1de89c9397018 100644
--- a/be/src/http/action/stream_load.h
+++ b/be/src/http/action/stream_load.h
@@ -50,7 +50,7 @@ class StreamLoadAction : public HttpHandler {
     Status _data_saved_path(HttpRequest* req, std::string* file_path);
     Status _process_put(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
     void _save_stream_load_record(std::shared_ptr<StreamLoadContext> ctx, const std::string& str);
-    bool _load_size_smaller_than_wal_limit(HttpRequest* req);
+    Status _handle_group_commit(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
 
 private:
     ExecEnv* _exec_env;
diff --git a/be/src/runtime/group_commit_mgr.h b/be/src/runtime/group_commit_mgr.h
index 35afcc46249b7a..b0553b448769eb 100644
--- a/be/src/runtime/group_commit_mgr.h
+++ b/be/src/runtime/group_commit_mgr.h
@@ -156,7 +156,6 @@ class GroupCommitMgr {
     Status update_load_info(TUniqueId load_id, size_t content_length);
     Status get_load_info(TUniqueId load_id, size_t* content_length);
     Status remove_load_info(TUniqueId load_id);
-    std::condition_variable cv;
 
 private:
     ExecEnv* _exec_env = nullptr;
diff --git a/be/src/vec/sink/group_commit_block_sink.cpp b/be/src/vec/sink/group_commit_block_sink.cpp
index 3e38f9c42fc456..56869665878cc2 100644
--- a/be/src/vec/sink/group_commit_block_sink.cpp
+++ b/be/src/vec/sink/group_commit_block_sink.cpp
@@ -19,8 +19,6 @@
 
 #include <gen_cpp/DataSinks_types.h>
 
-#include <chrono>
-#include <mutex>
 #include <shared_mutex>
 
 #include "common/exception.h"
@@ -29,7 +27,6 @@
 #include "runtime/runtime_state.h"
 #include "util/doris_metrics.h"
 #include "vec/exprs/vexpr.h"
-#include "vec/sink/volap_table_sink.h"
 #include "vec/sink/vtablet_finder.h"
 
 namespace doris {
@@ -74,10 +71,10 @@ Status GroupCommitBlockSink::prepare(RuntimeState* state) {
     _state = state;
 
     // profile must add to state's object pool
-    _profile = state->obj_pool()->add(new RuntimeProfile("OlapTableSink"));
+    _profile = state->obj_pool()->add(new RuntimeProfile("GroupCommitBlockSink"));
     init_sink_common_profile();
-    _mem_tracker =
-            std::make_shared<MemTracker>("OlapTableSink:" + std::to_string(state->load_job_id()));
+    _mem_tracker = std::make_shared<MemTracker>("GroupCommitBlockSink:" +
+                                                std::to_string(state->load_job_id()));
     SCOPED_TIMER(_profile->total_time_counter());
     SCOPED_CONSUME_MEM_TRACKER(_mem_tracker.get());
 
@@ -117,7 +114,7 @@ Status GroupCommitBlockSink::close(RuntimeState* state, Status close_status) {
             (double)state->num_rows_load_filtered() / num_selected_rows > _max_filter_ratio) {
             return Status::DataQualityError("too many filtered rows");
         }
-        RETURN_IF_ERROR(_add_blocks(_group_commit_mode != TGroupCommitMode::SYNC_MODE, true));
+        RETURN_IF_ERROR(_add_blocks(true));
     }
     if (_load_block_queue) {
         _load_block_queue->remove_load_id(_load_id);
@@ -223,15 +220,15 @@ Status GroupCommitBlockSink::_add_block(RuntimeState* state,
         _blocks.emplace_back(output_block);
     } else {
         if (!_is_block_appended) {
-            RETURN_IF_ERROR(_add_blocks(_group_commit_mode != TGroupCommitMode::SYNC_MODE, false));
+            RETURN_IF_ERROR(_add_blocks(false));
         }
         RETURN_IF_ERROR(_load_block_queue->add_block(
-                output_block, _group_commit_mode != TGroupCommitMode::SYNC_MODE));
+                output_block, _group_commit_mode == TGroupCommitMode::ASYNC_MODE));
     }
     return Status::OK();
 }
 
-Status GroupCommitBlockSink::_add_blocks(bool write_wal, bool is_blocks_contain_all_load_data) {
+Status GroupCommitBlockSink::_add_blocks(bool is_blocks_contain_all_load_data) {
     DCHECK(_is_block_appended == false);
     TUniqueId load_id;
     load_id.__set_hi(_load_id.hi);
@@ -241,18 +238,15 @@ Status GroupCommitBlockSink::_add_blocks(bool write_wal, bool is_blocks_contain_
             RETURN_IF_ERROR(_state->exec_env()->group_commit_mgr()->get_first_block_load_queue(
                     _db_id, _table_id, _base_schema_version, load_id, _load_block_queue,
                     _state->be_exec_version()));
-            if (write_wal) {
+            if (_group_commit_mode == TGroupCommitMode::ASYNC_MODE) {
                 _group_commit_mode = _load_block_queue->has_enough_wal_disk_space(
                                              _blocks, load_id, is_blocks_contain_all_load_data)
                                              ? TGroupCommitMode::ASYNC_MODE
                                              : TGroupCommitMode::SYNC_MODE;
                 if (_group_commit_mode == TGroupCommitMode::SYNC_MODE) {
-                    LOG(INFO)
-                            << "Load label " << _load_block_queue->label
-                            << " will not write wal because wal disk space usage reachs max limit.";
-                } else {
-                    LOG(INFO) << "Load label " << _load_block_queue->label << " will write wal to "
-                              << _load_block_queue->wal_base_path << ".";
+                    LOG(INFO) << "Load id=" << print_id(_state->query_id())
+                              << ", use group commit label=" << _load_block_queue->label
+                              << " will not write wal because wal disk space usage reach max limit";
                 }
             }
             _state->set_import_label(_load_block_queue->label);
@@ -263,7 +257,7 @@ Status GroupCommitBlockSink::_add_blocks(bool write_wal, bool is_blocks_contain_
     }
     for (auto it = _blocks.begin(); it != _blocks.end(); ++it) {
         RETURN_IF_ERROR(_load_block_queue->add_block(
-                *it, _group_commit_mode != TGroupCommitMode::SYNC_MODE));
+                *it, _group_commit_mode == TGroupCommitMode::ASYNC_MODE));
     }
     _is_block_appended = true;
     _blocks.clear();
diff --git a/be/src/vec/sink/group_commit_block_sink.h b/be/src/vec/sink/group_commit_block_sink.h
index c0971f4801a5ac..84ffebf8fe1346 100644
--- a/be/src/vec/sink/group_commit_block_sink.h
+++ b/be/src/vec/sink/group_commit_block_sink.h
@@ -47,7 +47,7 @@ class GroupCommitBlockSink : public DataSink {
 
 private:
     Status _add_block(RuntimeState* state, std::shared_ptr<vectorized::Block> block);
-    Status _add_blocks(bool write_wal, bool is_blocks_contain_all_load_data);
+    Status _add_blocks(bool is_blocks_contain_all_load_data);
 
     vectorized::VExprContextSPtrs _output_vexpr_ctxs;
 
@@ -70,6 +70,8 @@ class GroupCommitBlockSink : public DataSink {
     std::vector<std::shared_ptr<vectorized::Block>> _blocks;
     bool _is_block_appended = false;
     double _max_filter_ratio = 0.0;
+
+    // used for find_partition
     VOlapTablePartitionParam* _vpartition = nullptr;
     // reuse for find_tablet.
     std::vector<VOlapTablePartition*> _partitions;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
index 8106aed1ac2401..1aff92b4d21e64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
@@ -154,6 +154,7 @@ public class NativeInsertStmt extends InsertStmt {
     private long tableId = -1;
     public boolean isGroupCommitStreamLoadSql = false;
     private GroupCommitPlanner groupCommitPlanner;
+    private boolean reuseGroupCommitPlan = false;
 
     private boolean isFromDeleteOrUpdateStmt = false;
 
@@ -1162,12 +1163,18 @@ public boolean isGroupCommit() {
         return isGroupCommit;
     }
 
+    public boolean isReuseGroupCommitPlan() {
+        return reuseGroupCommitPlan;
+    }
+
     public GroupCommitPlanner planForGroupCommit(TUniqueId queryId) throws UserException, TException {
         OlapTable olapTable = (OlapTable) getTargetTable();
         if (groupCommitPlanner != null && olapTable.getBaseSchemaVersion() == baseSchemaVersion) {
             LOG.debug("reuse group commit plan, table={}", olapTable);
+            reuseGroupCommitPlan = true;
             return groupCommitPlanner;
         }
+        reuseGroupCommitPlan = false;
         if (!targetColumns.isEmpty()) {
             Analyzer analyzerTmp = analyzer;
             reset();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitBlockSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitBlockSink.java
index 8030dcf4e5cfec..86f588e525f642 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitBlockSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitBlockSink.java
@@ -58,6 +58,8 @@ public static TGroupCommitMode parseGroupCommit(String groupCommit) {
             return TGroupCommitMode.ASYNC_MODE;
         } else if (groupCommit.equalsIgnoreCase("sync_mode")) {
             return TGroupCommitMode.SYNC_MODE;
+        } else if (groupCommit.equalsIgnoreCase("off_mode")) {
+            return TGroupCommitMode.OFF_MODE;
         } else {
             return null;
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 02402e69c70150..d594fd395e772f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -33,6 +33,7 @@
 import org.apache.doris.nereids.rules.RuleType;
 import org.apache.doris.planner.GroupCommitBlockSink;
 import org.apache.doris.qe.VariableMgr.VarAttr;
+import org.apache.doris.thrift.TGroupCommitMode;
 import org.apache.doris.thrift.TQueryOptions;
 import org.apache.doris.thrift.TResourceLimit;
 import org.apache.doris.thrift.TRuntimeFilterType;
@@ -3176,7 +3177,9 @@ public void checkSqlDialect(String sqlDialect) {
     }
 
     public boolean isEnableInsertGroupCommit() {
-        return Config.wait_internal_group_commit_finish || GroupCommitBlockSink.parseGroupCommit(groupCommit) != null;
+        return Config.wait_internal_group_commit_finish
+                || GroupCommitBlockSink.parseGroupCommit(groupCommit) == TGroupCommitMode.ASYNC_MODE
+                || GroupCommitBlockSink.parseGroupCommit(groupCommit) == TGroupCommitMode.SYNC_MODE;
     }
 
     public String getGroupCommit() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index a897c6e8683cac..2851304dc372e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -1919,6 +1919,7 @@ private void handleInsertStmt() throws Exception {
         String errMsg = "";
         TableType tblType = insertStmt.getTargetTable().getType();
         boolean isGroupCommit = false;
+        boolean reuseGroupCommitPlan = false;
         if (context.isTxnModel()) {
             if (insertStmt.getQueryStmt() instanceof SelectStmt) {
                 if (((SelectStmt) insertStmt.getQueryStmt()).getTableRefs().size() > 0) {
@@ -1935,6 +1936,7 @@ private void handleInsertStmt() throws Exception {
             int maxRetry = 3;
             for (int i = 0; i < maxRetry; i++) {
                 GroupCommitPlanner groupCommitPlanner = nativeInsertStmt.planForGroupCommit(context.queryId);
+                reuseGroupCommitPlan = nativeInsertStmt.isReuseGroupCommitPlan();
                 List<InternalService.PDataRow> rows = groupCommitPlanner.getRows(nativeInsertStmt);
                 PGroupCommitInsertResponse response = groupCommitPlanner.executeGroupCommitInsert(context, rows);
                 TStatusCode code = TStatusCode.findByValue(response.getStatus().getStatusCode());
@@ -2106,6 +2108,9 @@ private void handleInsertStmt() throws Exception {
         }
         if (isGroupCommit) {
             sb.append(", 'query_id':'").append(DebugUtil.printId(context.queryId)).append("'");
+            if (reuseGroupCommitPlan) {
+                sb.append(", 'reuse_group_commit_plan':'").append(true).append("'");
+            }
         }
         sb.append("}");
 
diff --git a/regression-test/suites/insert_p0/insert_group_commit_into.groovy b/regression-test/suites/insert_p0/insert_group_commit_into.groovy
index 4b4fa430de632b..391e106aa2f6c8 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_into.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_into.groovy
@@ -215,7 +215,7 @@ suite("insert_group_commit_into") {
         }
 
         // test connect to observer fe
-        /*try {
+        try {
             def fes = sql_return_maparray "show frontends"
             logger.info("frontends: ${fes}")
             if (fes.size() > 1) {
@@ -233,6 +233,7 @@ suite("insert_group_commit_into") {
                         sql """ set group_commit = async_mode; """
                         sql """ set enable_nereids_dml = false; """
                         sql """ set enable_profile= true; """
+                        sql """ set enable_nereids_planner = false; """
 
                         // 1. insert into
                         def server_info = group_commit_insert """ insert into ${table}(name, id) values('c', 3);  """, 1
@@ -261,7 +262,7 @@ suite("insert_group_commit_into") {
                 logger.info("only one fe, skip test connect to observer fe")
             }
         } finally {
-        }*/
+        }
 
         // table with array type
         tableName = "insert_group_commit_into_duplicate_array"
diff --git a/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy b/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
index 2a88adf5bf64bb..0d8d7a8416b00e 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
@@ -18,10 +18,15 @@
 import com.mysql.cj.ServerPreparedQuery
 import com.mysql.cj.jdbc.ConnectionImpl
 import com.mysql.cj.jdbc.JdbcStatement
-import com.mysql.cj.jdbc.ServerPreparedStatement
+import com.mysql.cj.jdbc.ServerPreparedStatement;
 import com.mysql.cj.jdbc.StatementImpl
+import com.mysql.cj.jdbc.result.ResultSetImpl
+import com.mysql.cj.jdbc.result.ResultSetInternalMethods
 
 import java.lang.reflect.Field
+import java.sql.ResultSet
+import java.util.ArrayList
+import java.util.List
 import java.util.concurrent.CopyOnWriteArrayList
 
 suite("insert_group_commit_with_prepare_stmt") {
@@ -72,22 +77,38 @@ suite("insert_group_commit_with_prepare_stmt") {
         stmt.addBatch()
     }
 
-    def group_commit_insert = { stmt, expected_row_count ->
+    def group_commit_insert = { stmt, expected_row_count, reuse_plan = false ->
         def result = stmt.executeBatch()
         logger.info("insert result: " + result)
         def results = ((StatementImpl) stmt).results
-        if (results == null) {
-            logger.warn("result is null")
-            return
-        }
-        def serverInfo = results.getServerInfo()
-        logger.info("result server info: " + serverInfo)
-        if (result != expected_row_count) {
-            logger.warn("insert result: " + result + ", expected_row_count: " + expected_row_count)
+        if (results != null) {
+            def serverInfo = results.getServerInfo()
+            logger.info("result server info: " + serverInfo)
+            if (result != expected_row_count) {
+                logger.warn("insert result: " + result + ", expected_row_count: " + expected_row_count)
+            }
+            assertTrue(serverInfo.contains("'status':'PREPARE'"))
+            assertTrue(serverInfo.contains("'label':'group_commit_"))
+            assertEquals(reuse_plan, serverInfo.contains("reuse_group_commit_plan"))
+        } else {
+            // for batch insert
+            ConnectionImpl connection = (ConnectionImpl) stmt.getConnection()
+            Field field = ConnectionImpl.class.getDeclaredField("openStatements")
+            field.setAccessible(true)
+            CopyOnWriteArrayList<JdbcStatement> openStatements = (CopyOnWriteArrayList<JdbcStatement>) field.get(connection)
+            for (JdbcStatement openStatement : openStatements) {
+                ServerPreparedStatement serverPreparedStatement = (ServerPreparedStatement) openStatement;
+                Field field2 = StatementImpl.class.getDeclaredField("results");
+                field2.setAccessible(true);
+                ResultSet resultSet = (ResultSetInternalMethods) field2.get(serverPreparedStatement);
+                if (resultSet != null) {
+                    ResultSetImpl resultSetImpl = (ResultSetImpl) resultSet;
+                    String serverInfo = resultSetImpl.getServerInfo();
+                    logger.info("serverInfo = " + serverInfo);
+                }
+            }
         }
         // assertEquals(result, expected_row_count)
-        assertTrue(serverInfo.contains("'status':'PREPARE'"))
-        assertTrue(serverInfo.contains("'label':'group_commit_"))
     }
 
     def getStmtId = { stmt ->
@@ -140,12 +161,12 @@ suite("insert_group_commit_with_prepare_stmt") {
             insert_prepared insert_stmt, 2, null, 20
             insert_prepared insert_stmt, 3, "c", null
             insert_prepared insert_stmt, 4, "d", 40
-            group_commit_insert insert_stmt, 3
+            group_commit_insert insert_stmt, 3, true
             assertEquals(stmtId, getStmtId(insert_stmt))
 
             insert_prepared insert_stmt, 5, "e", null
             insert_prepared insert_stmt, 6, "f", 40
-            group_commit_insert insert_stmt, 2
+            group_commit_insert insert_stmt, 2, true
             assertEquals(stmtId, getStmtId(insert_stmt))
 
             getRowCount(6)
@@ -161,7 +182,7 @@ suite("insert_group_commit_with_prepare_stmt") {
 
             insert_prepared_partial insert_stmt, 'e', 7, 0
             insert_prepared_partial insert_stmt, null, 8, 0
-            group_commit_insert insert_stmt, 2
+            group_commit_insert insert_stmt, 2, true
             assertEquals(stmtId2, getStmtId(insert_stmt))
 
             getRowCount(7)

From 99a1e066b5ca3014fdfe46c7e47d59c250447809 Mon Sep 17 00:00:00 2001
From: huanghaibin <284824253@qq.com>
Date: Fri, 29 Dec 2023 00:26:38 +0800
Subject: [PATCH 134/299] [fix](group_commit) group_commit is not support on
 table with property light_schema_change=false (#29244)

---
 be/src/olap/wal_manager.cpp                               | 6 ++----
 .../java/org/apache/doris/analysis/NativeInsertStmt.java  | 8 +++++++-
 .../trees/plans/commands/InsertIntoTableCommand.java      | 2 +-
 .../org/apache/doris/service/FrontendServiceImpl.java     | 8 ++++++++
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/be/src/olap/wal_manager.cpp b/be/src/olap/wal_manager.cpp
index c62996da19b415..0fe59cdf9cac3f 100644
--- a/be/src/olap/wal_manager.cpp
+++ b/be/src/olap/wal_manager.cpp
@@ -377,12 +377,10 @@ Status WalManager::replay() {
             std::lock_guard<std::shared_mutex> wrlock(_lock);
             auto it = _table_map.begin();
             while (it != _table_map.end()) {
-                if (it->second->size() == 0) {
-                    it = _table_map.erase(it);
-                } else {
+                if (it->second->size() > 0) {
                     replay_tables.push_back(it->first);
-                    it++;
                 }
+                it++;
             }
         }
         for (const auto& table_id : replay_tables) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
index 1aff92b4d21e64..0fcd5050fc83cb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
@@ -1109,7 +1109,12 @@ public RedirectStatus getRedirectStatus() {
         }
     }
 
-    public void analyzeGroupCommit(Analyzer analyzer) {
+    public void analyzeGroupCommit(Analyzer analyzer) throws AnalysisException {
+        if (isGroupCommitStreamLoadSql && (targetTable instanceof OlapTable)
+                && !((OlapTable) targetTable).getTableProperty().getUseSchemaLightChange()) {
+            throw new AnalysisException(
+                    "table light_schema_change is false, can't do http_stream with group commit mode");
+        }
         if (isGroupCommit) {
             return;
         }
@@ -1124,6 +1129,7 @@ public void analyzeGroupCommit(Analyzer analyzer) {
         if (!partialUpdate && ConnectContext.get().getSessionVariable().isEnableInsertGroupCommit()
                 && ConnectContext.get().getSessionVariable().getSqlMode() != SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES
                 && targetTable instanceof OlapTable
+                && ((OlapTable) targetTable).getTableProperty().getUseSchemaLightChange()
                 && !ConnectContext.get().isTxnModel()
                 && getQueryStmt() instanceof SelectStmt
                 && ((SelectStmt) getQueryStmt()).getTableRefs().isEmpty() && targetPartitionNames == null
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
index 11a7926e5dd8b8..0427fea005ad53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertIntoTableCommand.java
@@ -249,7 +249,7 @@ private boolean analyzeGroupCommit(ConnectContext ctx, DataSink sink,
         return ConnectContext.get().getSessionVariable().getSqlMode() != SqlModeHelper.MODE_NO_BACKSLASH_ESCAPES
                 && physicalOlapTableSink.getTargetTable() instanceof OlapTable && !ConnectContext.get().isTxnModel()
                 && sink.getFragment().getPlanRoot() instanceof UnionNode && physicalOlapTableSink.getPartitionIds()
-                .isEmpty();
+                .isEmpty() && physicalOlapTableSink.getTargetTable().getTableProperty().getUseSchemaLightChange();
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 75405854cabed2..e5b0f651016ce9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -2001,6 +2001,10 @@ private TExecPlanFragmentParams streamLoadPutImpl(TStreamLoadPutRequest request,
         }
         long timeoutMs = request.isSetThriftRpcTimeoutMs() ? request.getThriftRpcTimeoutMs() : 5000;
         Table table = db.getTableOrMetaException(request.getTbl(), TableType.OLAP);
+        if (!((OlapTable) table).getTableProperty().getUseSchemaLightChange() && (request.getGroupCommitMode() != null
+                && !request.getGroupCommitMode().equals("off_mode"))) {
+            throw new UserException("table light_schema_change is false, can't do stream load with group commit mode");
+        }
         result.setDbId(db.getId());
         result.setTableId(table.getId());
         result.setBaseSchemaVersion(((OlapTable) table).getBaseSchemaVersion());
@@ -2062,6 +2066,10 @@ private TPipelineFragmentParams pipelineStreamLoadPutImpl(TStreamLoadPutRequest
         }
         long timeoutMs = request.isSetThriftRpcTimeoutMs() ? request.getThriftRpcTimeoutMs() : 5000;
         Table table = db.getTableOrMetaException(request.getTbl(), TableType.OLAP);
+        if (!((OlapTable) table).getTableProperty().getUseSchemaLightChange() && (request.getGroupCommitMode() != null
+                && !request.getGroupCommitMode().equals("off_mode"))) {
+            throw new UserException("table light_schema_change is false, can't do stream load with group commit mode");
+        }
         return this.generatePipelineStreamLoadPut(request, db, fullDbName, (OlapTable) table, timeoutMs,
                 1, false);
     }

From efea006f3abc2dc3654b34e6840d236f93ae235e Mon Sep 17 00:00:00 2001
From: zhengyu <freeman.zhang1992@gmail.com>
Date: Fri, 29 Dec 2023 00:33:34 +0800
Subject: [PATCH 135/299] [ut](move-memtable) add CLOSE_LOAD before EOS ut case
 (#29253)

Signed-off-by: freemandealer <freeman.zhang1992@gmail.com>
---
 be/test/runtime/load_stream_test.cpp | 42 +++++++++++++++++++++++++++-
 1 file changed, 41 insertions(+), 1 deletion(-)

diff --git a/be/test/runtime/load_stream_test.cpp b/be/test/runtime/load_stream_test.cpp
index 7cba945dd230c2..6509f5aa35d856 100644
--- a/be/test/runtime/load_stream_test.cpp
+++ b/be/test/runtime/load_stream_test.cpp
@@ -572,7 +572,7 @@ class LoadStreamMgrTest : public testing::Test {
     }
 
     void wait_for_close() {
-        for (int i = 0; i < 1000 && _load_stream_mgr->get_load_stream_num() != 0; i++) {
+        for (int i = 0; i < 3000 && _load_stream_mgr->get_load_stream_num() != 0; i++) {
             bthread_usleep(1000);
         }
     }
@@ -851,6 +851,46 @@ TEST_F(LoadStreamMgrTest, one_client_one_index_one_tablet_single_segment0_zero_b
     EXPECT_EQ(_load_stream_mgr->get_load_stream_num(), 0);
 }
 
+TEST_F(LoadStreamMgrTest, close_load_before_recv_eos) {
+    MockSinkClient client;
+    auto st = client.connect_stream();
+    EXPECT_TRUE(st.ok());
+
+    reset_response_stat();
+
+    // append data
+    butil::IOBuf append_buf;
+    PStreamHeader header;
+    std::string data = "file1 hello world 123 !@#$%^&*()_+";
+    write_one_tablet(client, NORMAL_LOAD_ID, NORMAL_SENDER_ID, NORMAL_INDEX_ID, NORMAL_TABLET_ID, 0,
+                     data.length(), data, false);
+
+    EXPECT_EQ(g_response_stat.num, 0);
+    // CLOSE_LOAD before EOS
+    close_load(client);
+    wait_for_ack(1);
+    EXPECT_EQ(g_response_stat.num, 1);
+    EXPECT_EQ(g_response_stat.success_tablet_ids.size(), 0);
+    EXPECT_EQ(g_response_stat.failed_tablet_ids.size(), 1);
+    // server will close stream on CLOSE_LOAD
+    wait_for_close();
+    EXPECT_EQ(_load_stream_mgr->get_load_stream_num(), 0);
+
+    // then the late EOS, will not be handled
+    write_one_tablet(client, NORMAL_LOAD_ID, NORMAL_SENDER_ID, NORMAL_INDEX_ID, NORMAL_TABLET_ID, 0,
+                     data.length(), data, true);
+
+    // duplicated close, will not be handled
+    close_load(client);
+    wait_for_ack(2);
+    EXPECT_EQ(g_response_stat.num, 1);
+    EXPECT_EQ(g_response_stat.success_tablet_ids.size(), 0);
+    EXPECT_EQ(g_response_stat.failed_tablet_ids.size(), 1);
+
+    auto written_data = read_data(NORMAL_TXN_ID, NORMAL_PARTITION_ID, NORMAL_TABLET_ID, 0);
+    EXPECT_EQ(written_data, "");
+}
+
 TEST_F(LoadStreamMgrTest, one_client_one_index_one_tablet_single_segment0) {
     MockSinkClient client;
     auto st = client.connect_stream();

From fb0ed8c253f5241c23c71fb88f031604b0bf5a1f Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Fri, 29 Dec 2023 00:33:58 +0800
Subject: [PATCH 136/299] [fix](move-memtable) check missing tablets before
 commit (#29223)

---
 be/src/vec/sink/writer/vtablet_writer_v2.cpp | 23 ++++++++++++++++++++
 be/src/vec/sink/writer/vtablet_writer_v2.h   |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index f413c00dc0c7e3..f68d82911ce8a9 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -535,10 +535,32 @@ Status VTabletWriterV2::close(Status exec_status) {
                     TTabletCommitInfo commit_info;
                     commit_info.tabletId = tablet_id;
                     commit_info.backendId = node_id;
+                    _missing_tablets.erase(tablet_id);
                     tablet_commit_infos.emplace_back(std::move(commit_info));
                 }
             }
         }
+        if (!_missing_tablets.empty()) {
+            std::stringstream ss("pre-commit check failed, ");
+            ss << "missing " << _missing_tablets.size() << " tablets:";
+            int print_limit = 3;
+            for (auto tablet_id : _missing_tablets | std::ranges::views::take(print_limit)) {
+                ss << " (tablet_id=" << tablet_id;
+                auto backends = _location->find_tablet(tablet_id)->node_ids;
+                ss << ", backend_id=[";
+                bool first = true;
+                for (auto& backend_id : backends) {
+                    (first ? ss : ss << ',') << backend_id;
+                    first = false;
+                }
+                ss << "])";
+            }
+            if (_missing_tablets.size() > print_limit) {
+                ss << ", ...";
+            }
+            LOG(INFO) << ss.str() << ", load_id=" << print_id(_load_id);
+            return Status::InternalError(ss.str());
+        }
         _state->tablet_commit_infos().insert(_state->tablet_commit_infos().end(),
                                              std::make_move_iterator(tablet_commit_infos.begin()),
                                              std::make_move_iterator(tablet_commit_infos.end()));
@@ -569,6 +591,7 @@ Status VTabletWriterV2::_close_load(const Streams& streams) {
     for (auto [tablet_id, tablet] : _tablets_for_node[node_id]) {
         if (_tablet_finder->partition_ids().contains(tablet.partition_id())) {
             tablets_to_commit.push_back(tablet);
+            _missing_tablets.insert(tablet_id);
         }
     }
     for (const auto& stream : streams) {
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.h b/be/src/vec/sink/writer/vtablet_writer_v2.h
index 916bad430a580d..fa21071da839b1 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.h
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.h
@@ -214,7 +214,7 @@ class VTabletWriterV2 final : public AsyncResultWriter {
     RuntimeState* _state = nullptr;     // not owned, set when open
     RuntimeProfile* _profile = nullptr; // not owned, set when open
 
-    std::unordered_set<int64_t> _opened_partitions;
+    std::unordered_set<int64_t> _missing_tablets;
 
     std::unordered_map<int64_t, std::unordered_map<int64_t, PTabletID>> _tablets_for_node;
     std::unordered_map<int64_t, std::vector<PTabletID>> _indexes_from_node;

From 825732a395f3d1c3e607537b0f05fecf09fcd5df Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Fri, 29 Dec 2023 00:45:34 +0800
Subject: [PATCH 137/299] [fix](regression) fix unstable test case (#29264)

---
 regression-test/suites/datatype_p0/decimalv3/test_load.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy
index 726b1ef048561e..48cfb286f45ae7 100644
--- a/regression-test/suites/datatype_p0/decimalv3/test_load.groovy
+++ b/regression-test/suites/datatype_p0/decimalv3/test_load.groovy
@@ -99,5 +99,5 @@ suite("test_load") {
         """
         exception "error"
     }
-    qt_decimalv3_insert "select * from test_decimalv3_insert order by 1;"
+    qt_decimalv3_insert "select * from test_decimalv3_insert order by 1, 2;"
 }

From 69c90b1640d566ab03a045f89a0e50b8b6f7b00f Mon Sep 17 00:00:00 2001
From: huanghaibin <284824253@qq.com>
Date: Fri, 29 Dec 2023 00:50:22 +0800
Subject: [PATCH 138/299] [fix](group commit)fix group commit regresstion test
 (#29079)

---
 .../apache/doris/httpv2/rest/LoadAction.java  |  5 +-
 ...mit_http_stream_lineitem_schema_change.out |  3 -
 ...t_insert_into_lineitem_multiple_client.out |  2 +-
 ...it_insert_into_lineitem_multiple_table.out | 23 ++---
 ...oup_commit_insert_into_lineitem_normal.out |  2 +-
 ...mit_insert_into_lineitem_scheme_change.out | 10 +--
 ...mit_stream_load_lineitem_schema_change.out |  2 +-
 ..._commit_http_stream_lineitem_normal.groovy |  1 +
 ..._http_stream_lineitem_schema_change.groovy | 59 ++++++++-----
 ...nsert_into_lineitem_multiple_client.groovy | 59 ++++++-------
 ...insert_into_lineitem_multiple_table.groovy | 59 ++++++-------
 ..._commit_insert_into_lineitem_normal.groovy | 55 ++++++------
 ..._insert_into_lineitem_scheme_change.groovy | 87 ++++++++++---------
 ..._commit_stream_load_lineitem_normal.groovy |  1 +
 ..._stream_load_lineitem_schema_change.groovy | 61 ++++++++-----
 15 files changed, 236 insertions(+), 193 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
index 122cbab49eecdc..f4a0953949d19f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/LoadAction.java
@@ -85,11 +85,10 @@ public Object streamLoad(HttpServletRequest request,
                              @PathVariable(value = DB_KEY) String db, @PathVariable(value = TABLE_KEY) String table) {
         boolean groupCommit = false;
         String groupCommitStr = request.getHeader("group_commit");
-        if (groupCommitStr != null && groupCommitStr.equals("true")) {
+        if (groupCommitStr != null && groupCommitStr.equals("async_mode")) {
             groupCommit = true;
             try {
                 String[] pair = new String[] {db, table};
-                LOG.info(pair[0] + ":" + pair[1]);
                 if (isGroupCommitBlock(pair)) {
                     String msg = "insert table " + pair[1] + " is blocked on schema change";
                     return new RestBaseResult(msg);
@@ -128,7 +127,7 @@ public Object streamLoadWithSql(HttpServletRequest request,
         LOG.info("streaming load sql={}", sql);
         boolean groupCommit = false;
         String groupCommitStr = request.getHeader("group_commit");
-        if (groupCommitStr != null && groupCommitStr.equals("true")) {
+        if (groupCommitStr != null && groupCommitStr.equals("async_mode")) {
             groupCommit = true;
             try {
                 String[] pair = parseDbAndTb(sql);
diff --git a/regression-test/data/insert_p2/test_group_commit_http_stream_lineitem_schema_change.out b/regression-test/data/insert_p2/test_group_commit_http_stream_lineitem_schema_change.out
index 1be3e26c88ee35..2abf38b9beffc3 100644
--- a/regression-test/data/insert_p2/test_group_commit_http_stream_lineitem_schema_change.out
+++ b/regression-test/data/insert_p2/test_group_commit_http_stream_lineitem_schema_change.out
@@ -11,6 +11,3 @@
 -- !sql --
 6001215
 
--- !sql --
-6001215
-
diff --git a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.out b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.out
index ef23823e23ec95..da7aa0f1454214 100644
--- a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.out
+++ b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.out
@@ -1,4 +1,4 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-6001215
+600572
 
diff --git a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.out b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.out
index 86534c175022fa..71cdc122f3f131 100644
--- a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.out
+++ b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.out
@@ -1,31 +1,34 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-600572
+60000
 
 -- !sql --
-600659
+60000
 
 -- !sql --
-599397
+60000
 
 -- !sql --
-600124
+60000
 
 -- !sql --
-599647
+60000
 
 -- !sql --
-599931
+60000
 
 -- !sql --
-601365
+60000
 
 -- !sql --
-599301
+60000
 
 -- !sql --
-600504
+60000
 
 -- !sql --
-599715
+60000
+
+-- !sql --
+572
 
diff --git a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_normal.out b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_normal.out
index ef23823e23ec95..da7aa0f1454214 100644
--- a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_normal.out
+++ b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_normal.out
@@ -1,4 +1,4 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-6001215
+600572
 
diff --git a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.out b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.out
index 4f530919417800..f2cd979eb49cf7 100644
--- a/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.out
+++ b/regression-test/data/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.out
@@ -1,16 +1,16 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-3000816
+180000
 
 -- !sql --
-6001215
+300000
 
 -- !sql --
-4673070
+240065
 
 -- !sql --
-6001215
+300000
 
 -- !sql --
-6001215
+300000
 
diff --git a/regression-test/data/insert_p2/test_group_commit_stream_load_lineitem_schema_change.out b/regression-test/data/insert_p2/test_group_commit_stream_load_lineitem_schema_change.out
index 1be3e26c88ee35..50a4e09f4e2318 100644
--- a/regression-test/data/insert_p2/test_group_commit_stream_load_lineitem_schema_change.out
+++ b/regression-test/data/insert_p2/test_group_commit_stream_load_lineitem_schema_change.out
@@ -12,5 +12,5 @@
 6001215
 
 -- !sql --
-6001215
+12002430
 
diff --git a/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_normal.groovy b/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_normal.groovy
index d1d4e476b4a0e0..44d9a0ff94ba67 100644
--- a/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_normal.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_normal.groovy
@@ -85,6 +85,7 @@ PROPERTIES (
     def process = {
         int total = 0;
         for (int k = 0; k < 3; k++) {
+            logger.info("round:" + k)
             for (int i = 1; i <= 10; i++) {
                 streamLoad {
                     set 'version', '1'
diff --git a/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy b/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy
index 8a979bf6f33a4c..dd0d40e645ca50 100644
--- a/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy
@@ -151,22 +151,34 @@ PROPERTIES (
     }
 
     def insert_data = { i, table_name ->
-        streamLoad {
-            set 'version', '1'
-            set 'sql', """
-                    insert into ${db}.${stream_load_table}(l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, 
+        int j = 0;
+        while (true) {
+            if (j >= 18) {
+                throw new Exception("""fail to much time""")
+            }
+            try {
+                streamLoad {
+                    set 'version', '1'
+                    set 'sql', """
+                    insert into ${db}.${table_name}(l_orderkey, l_partkey, l_suppkey, l_linenumber, l_quantity, 
 l_extendedprice, l_discount, l_tax, l_returnflag,l_linestatus, l_shipdate,l_commitdate,l_receiptdate,l_shipinstruct,
 l_shipmode,l_comment) select c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16 from http_stream
                     ("format"="csv", "column_separator"="|")
             """
 
-            set 'group_commit', 'async_mode'
-            file """${getS3Url()}/regression/tpch/sf1/lineitem.tbl.""" + i
-            unset 'label'
+                    set 'group_commit', 'async_mode'
+                    file """${getS3Url()}/regression/tpch/sf1/lineitem.tbl.""" + i
+                    unset 'label'
 
-            check { result, exception, startTime, endTime ->
-                checkStreamLoadResult(exception, result, rowCountArray[i - 1], rowCountArray[i - 1], 0, 0)
+                    check { result, exception, startTime, endTime ->
+                        checkStreamLoadResult(exception, result, rowCountArray[i - 1], rowCountArray[i - 1], 0, 0)
+                    }
+                }
+                break
+            } catch (Exception e) {
+                Thread.sleep(10000)
             }
+            j++;
         }
         total += rowCountArray[i - 1];
     }
@@ -328,22 +340,25 @@ l_comment) select c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c14, c15, c
     def change_order = { table_name ->
         create_stream_load_table(table_name)
         total = 0;
-        for (int i = 1; i <= 10; i++) {
-            logger.info("process file:" + i)
-            if (i == 2) {
-                def retry = 0
-                while (retry < 10) {
-                    try {
-                        sql """ alter table ${table_name} order by (l_orderkey,l_shipdate,l_linenumber, l_partkey,l_suppkey,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment); """
-                        break
-                    } catch (Exception e) {
-                        log.info("exception:", e)
+        for (int k = 0; k < 2; k++) {
+            logger.info("round:" + k)
+            for (int i = 1; i <= 10; i++) {
+                logger.info("process file:" + i)
+                if (k == 0 && i == 2) {
+                    def retry = 0
+                    while (retry < 10) {
+                        try {
+                            sql """ alter table ${table_name} order by (l_orderkey,l_shipdate,l_linenumber, l_partkey,l_suppkey,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment); """
+                            break
+                        } catch (Exception e) {
+                            log.info("exception:", e)
+                        }
+                        Thread.sleep(2000)
+                        retry++
                     }
-                    Thread.sleep(2000)
-                    retry++
                 }
+                insert_data(i, table_name)
             }
-            insert_data(i, table_name)
         }
         logger.info("process change order total:" + total)
         assertTrue(getAlterTableState(table_name), "modify column order should success")
diff --git a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.groovy b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.groovy
index 3a094c17ce93d3..e53299a00195e4 100644
--- a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_client.groovy
@@ -35,22 +35,22 @@ String[] getFiles(String dirName, int num) {
 suite("test_group_commit_insert_into_lineitem_multiple_client") {
     String[] file_array;
     def prepare = {
-        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_multiple_client/"
+        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_multiple_client"
         File dir = new File(dataDir)
         if (!dir.exists()) {
-            new File("${context.config.cacheDataPath}/insert_into_lineitem_multiple_client/").mkdir()
-            for (int i = 1; i <= 10; i++) {
-                logger.info("download lineitem.tbl.${i}")
-                def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.${i}
---output ${context.config.cacheDataPath}/insert_into_lineitem_multiple_client/lineitem.tbl.${i}""".execute().getText()
-            }
+            new File(dataDir).mkdir()
+            logger.info("download lineitem")
+            def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.1
+--output ${dataDir}/lineitem.tbl.1""".execute().getText()
+            def split_file = """split -l 60000 ${dataDir}/lineitem.tbl.1 ${dataDir}/""".execute().getText()
+            def rm_file = """rm ${dataDir}/lineitem.tbl.1""".execute().getText()
         }
-        file_array = getFiles(dataDir, 10)
+        file_array = getFiles(dataDir, 11)
         for (String s : file_array) {
             logger.info(s)
         }
     }
-    def insert_table = "test_insert_into_lineitem_multiple_client_sf1"
+    def insert_table = "test_insert_into_lineitem_multiple_client"
     def batch = 100;
     def total = 0;
     def rwLock = new ReentrantReadWriteLock();
@@ -107,7 +107,24 @@ PROPERTIES (
         sql """ set enable_nereids_dml = false; """
     }
 
-    def do_insert_into = { file_name ->
+    def do_insert_into = { exp_str, num ->
+        def i = 0;
+        while (true) {
+            try {
+                def result = insert_into_sql(exp_str, num);
+                logger.info("result:" + result);
+                break
+            } catch (Exception e) {
+                logger.info("got exception:" + e)
+            }
+            i++;
+            if (i >= 30) {
+                throw new Exception("""fail to much time""")
+            }
+        }
+    }
+
+    def insert_file = { file_name ->
         logger.info("file:" + file_name)
         sql """ set group_commit = async_mode; """
         sql """ set enable_nereids_dml = false; """
@@ -128,15 +145,7 @@ PROPERTIES (
                 if (c == batch) {
                     sb.append(";");
                     String exp = sb.toString();
-                    while (true) {
-                        try {
-                            def result = insert_into_sql(exp, c);
-                            logger.info("result:" + result);
-                            break
-                        } catch (Exception e) {
-                            logger.info("got exception:" + e)
-                        }
-                    }
+                    do_insert_into(exp, c);
                     c = 0;
                 }
                 s = reader.readLine();
@@ -162,15 +171,7 @@ PROPERTIES (
                 } else if (c > 0) {
                     sb.append(";");
                     String exp = sb.toString();
-                    while (true) {
-                        try {
-                            def result = insert_into_sql(exp, c);
-                            logger.info("result:" + result);
-                            break
-                        } catch (Exception e) {
-                            logger.info("got exception:" + e)
-                        }
-                    }
+                    do_insert_into(exp, c);
                     break;
                 } else {
                     break;
@@ -198,7 +199,7 @@ PROPERTIES (
             String file_name = file_array[n]
             logger.info("insert into file:" + file_name)
             threads.add(Thread.startDaemon {
-                do_insert_into(file_name)
+                insert_file(file_name)
             })
         }
         for (Thread th in threads) {
diff --git a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.groovy b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.groovy
index eef2d2bc3ed488..3241b44b191c80 100644
--- a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_multiple_table.groovy
@@ -35,22 +35,22 @@ String[] getFiles(String dirName, int num) {
 suite("test_group_commit_insert_into_lineitem_multiple_table") {
     String[] file_array;
     def prepare = {
-        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_multiple_table/"
+        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_multiple_table"
         File dir = new File(dataDir)
         if (!dir.exists()) {
-            new File("${context.config.cacheDataPath}/insert_into_lineitem_multiple_table/").mkdir()
-            for (int i = 1; i <= 10; i++) {
-                logger.info("download lineitem.tbl.${i}")
-                def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.${i}
---output ${context.config.cacheDataPath}/insert_into_lineitem_multiple_table/lineitem.tbl.${i}""".execute().getText()
-            }
+            new File(dataDir).mkdir()
+            logger.info("download lineitem")
+            def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.1
+--output ${dataDir}/lineitem.tbl.1""".execute().getText()
+            def split_file = """split -l 60000 ${dataDir}/lineitem.tbl.1 ${dataDir}/""".execute().getText()
+            def rm_file = """rm ${dataDir}/lineitem.tbl.1""".execute().getText()
         }
-        file_array = getFiles(dataDir, 10)
+        file_array = getFiles(dataDir, 11)
         for (String s : file_array) {
             logger.info(s)
         }
     }
-    def insert_table_base = "test_insert_into_lineitem_multiple_table_sf1"
+    def insert_table_base = "test_insert_into_lineitem_multiple_table"
     def batch = 100;
     def total = 0;
     def rwLock = new ReentrantReadWriteLock();
@@ -107,7 +107,24 @@ PROPERTIES (
         sql """ set enable_nereids_dml = false; """
     }
 
-    def do_insert_into = { file_name, table_name ->
+    def do_insert_into = { exp_str, num ->
+        def i = 0;
+        while (true) {
+            try {
+                def result = insert_into_sql(exp_str, num);
+                logger.info("result:" + result);
+                break
+            } catch (Exception e) {
+                logger.info("got exception:" + e)
+            }
+            i++;
+            if (i >= 30) {
+                throw new Exception("""fail to much time""")
+            }
+        }
+    }
+
+    def insert_file = { file_name, table_name ->
         sql """ set group_commit = async_mode; """
         sql """ set enable_nereids_dml = false; """
         logger.info("file:" + file_name)
@@ -128,15 +145,7 @@ PROPERTIES (
                 if (c == batch) {
                     sb.append(";");
                     String exp = sb.toString();
-                    while (true) {
-                        try {
-                            def result = insert_into_sql(exp, c);
-                            logger.info("result:" + result);
-                            break
-                        } catch (Exception e) {
-                            logger.info("got exception:" + e)
-                        }
-                    }
+                    do_insert_into(exp, c);
                     c = 0;
                 }
                 s = reader.readLine();
@@ -162,15 +171,7 @@ PROPERTIES (
                 } else if (c > 0) {
                     sb.append(";");
                     String exp = sb.toString();
-                    while (true) {
-                        try {
-                            def result = insert_into_sql(exp, c);
-                            logger.info("result:" + result);
-                            break
-                        } catch (Exception e) {
-                            logger.info("got exception:" + e)
-                        }
-                    }
+                    do_insert_into(exp, c);
                     break;
                 } else {
                     break;
@@ -201,7 +202,7 @@ PROPERTIES (
             create_insert_table(table_name)
             logger.info("insert into file:" + file_name)
             threads.add(Thread.startDaemon {
-                do_insert_into(file_name, table_name)
+                insert_file(file_name, table_name)
             })
         }
         for (Thread th in threads) {
diff --git a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_normal.groovy b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_normal.groovy
index 9fee8508999064..a47784e7d517e9 100644
--- a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_normal.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_normal.groovy
@@ -31,22 +31,22 @@ String[] getFiles(String dirName, int num) {
 suite("test_group_commit_insert_into_lineitem_normal") {
     String[] file_array;
     def prepare = {
-        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_normal/"
+        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_normal"
         File dir = new File(dataDir)
         if (!dir.exists()) {
-            new File("${context.config.cacheDataPath}/insert_into_lineitem_normal/").mkdir()
-            for (int i = 1; i <= 10; i++) {
-                logger.info("download lineitem.tbl.${i}")
-                def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.${i}
---output ${context.config.cacheDataPath}/insert_into_lineitem_normal/lineitem.tbl.${i}""".execute().getText()
-            }
+            new File(dataDir).mkdir()
+            logger.info("download lineitem")
+            def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.1
+--output ${dataDir}/lineitem.tbl.1""".execute().getText()
+            def split_file = """split -l 60000 ${dataDir}/lineitem.tbl.1 ${dataDir}/""".execute().getText()
+            def rm_file = """rm ${dataDir}/lineitem.tbl.1""".execute().getText()
         }
-        file_array = getFiles(dataDir, 10)
+        file_array = getFiles(dataDir, 11)
         for (String s : file_array) {
             logger.info(s)
         }
     }
-    def insert_table = "test_insert_into_lineitem_sf1"
+    def insert_table = "test_insert_into_lineitem_normal"
     def batch = 100;
     def count = 0;
     def total = 0;
@@ -102,6 +102,23 @@ PROPERTIES (
         sql """ set enable_nereids_dml = false; """
     }
 
+    def do_insert_into = { exp_str, num ->
+        def i = 0;
+        while (true) {
+            try {
+                def result = insert_into_sql(exp_str, num);
+                logger.info("result:" + result);
+                break
+            } catch (Exception e) {
+                logger.info("got exception:" + e)
+            }
+            i++;
+            if (i >= 30) {
+                throw new Exception("""fail to much time""")
+            }
+        }
+    }
+
     def process = {
         for (String file : file_array) {
             logger.info("insert into file: " + file)
@@ -120,15 +137,7 @@ PROPERTIES (
                     if (count == batch) {
                         sb.append(";");
                         String exp = sb.toString();
-                        while (true) {
-                            try {
-                                def result = insert_into_sql(exp, count);
-                                logger.info("result:" + result);
-                                break
-                            } catch (Exception e) {
-                                logger.info("got exception:" + e)
-                            }
-                        }
+                        do_insert_into(exp, count);
                         count = 0;
                     }
                     s = reader.readLine();
@@ -154,15 +163,7 @@ PROPERTIES (
                     } else if (count > 0) {
                         sb.append(";");
                         String exp = sb.toString();
-                        while (true) {
-                            try {
-                                def result = insert_into_sql(exp, count);
-                                logger.info("result:" + result);
-                                break
-                            } catch (Exception e) {
-                                logger.info("got exception:" + e)
-                            }
-                        }
+                        do_insert_into(exp, count);
                         break;
                     } else {
                         break;
diff --git a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.groovy b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.groovy
index 917cadf31b45f6..1c77dcbeb5b028 100644
--- a/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_insert_into_lineitem_scheme_change.groovy
@@ -52,33 +52,37 @@ String[] getFiles(String dirName, int num) {
     if (num != datas.length) {
         throw new Exception("num not equals,expect:" + num + " vs real:" + datas.length)
     }
-    String[] array = new String[datas.length];
+    String[] tmp_array = new String[datas.length];
     for (int i = 0; i < datas.length; i++) {
-        array[i] = datas[i].getPath();
+        tmp_array[i] = datas[i].getPath();
+    }
+    Arrays.sort(tmp_array);
+    String[] array = new String[5];
+    for (int i = 0; i < 5; i++) {
+        array[i] = tmp_array[i];
     }
-    Arrays.sort(array);
     return array;
 }
 
 suite("test_group_commit_insert_into_lineitem_scheme_change") {
     String[] file_array;
     def prepare = {
-        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_scheme_change/"
+        def dataDir = "${context.config.cacheDataPath}/insert_into_lineitem_scheme_change"
         File dir = new File(dataDir)
         if (!dir.exists()) {
-            new File("${context.config.cacheDataPath}/insert_into_lineitem_scheme_change/").mkdir()
-            for (int i = 1; i <= 10; i++) {
-                logger.info("download lineitem.tbl.${i}")
-                def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.${i}
---output ${context.config.cacheDataPath}/insert_into_lineitem_scheme_change/lineitem.tbl.${i}""".execute().getText()
-            }
+            new File(dataDir).mkdir()
+            logger.info("download lineitem")
+            def download_file = """/usr/bin/curl ${getS3Url()}/regression/tpch/sf1/lineitem.tbl.1
+--output ${dataDir}/lineitem.tbl.1""".execute().getText()
+            def split_file = """split -l 60000 ${dataDir}/lineitem.tbl.1 ${dataDir}/""".execute().getText()
+            def rm_file = """rm ${dataDir}/lineitem.tbl.1""".execute().getText()
         }
-        file_array = getFiles(dataDir, 10)
+        file_array = getFiles(dataDir, 11)
         for (String s : file_array) {
             logger.info(s)
         }
     }
-    def insert_table = "test_lineitem_scheme_change_sf1"
+    def insert_table = "test_lineitem_scheme_change"
     def batch = 100;
     def count = 0;
     def total = 0;
@@ -168,6 +172,27 @@ PROPERTIES (
 
     }
 
+    def do_insert_into = { exp_str, num ->
+        def i = 0;
+        while (true) {
+            try {
+                def result = insert_into_sql(exp_str, num);
+                logger.info("result:" + result);
+                break
+            } catch (Exception e) {
+                logger.info("msg:" + e.getMessage())
+                logger.info("got exception:" + e)
+                if (e.getMessage().contains("is blocked on schema change")) {
+                    Thread.sleep(10000)
+                }
+            }
+            i++;
+            if (i >= 30) {
+                throw new Exception("""fail to much time""")
+            }
+        }
+    }
+
     def insert_data = { file_name, table_name, index ->
         BufferedReader reader;
         try {
@@ -184,15 +209,7 @@ PROPERTIES (
                 if (count == batch) {
                     sb.append(";");
                     String exp = sb.toString();
-                    while (true) {
-                        try {
-                            def result = insert_into_sql(exp, count);
-                            logger.info("result:" + result);
-                            break
-                        } catch (Exception e) {
-                            logger.info("got exception:" + e)
-                        }
-                    }
+                    do_insert_into(exp, count)
                     count = 0;
                 }
                 s = reader.readLine();
@@ -229,15 +246,7 @@ PROPERTIES (
                 } else if (count > 0) {
                     sb.append(";");
                     String exp = sb.toString();
-                    while (true) {
-                        try {
-                            def result = insert_into_sql(exp, count);
-                            logger.info("result:" + result);
-                            break
-                        } catch (Exception e) {
-                            logger.info("got exception:" + e)
-                        }
-                    }
+                    do_insert_into(exp, count)
                     break;
                 } else {
                     break;
@@ -275,7 +284,7 @@ PROPERTIES (
         for (int i = 0; i < file_array.length; i++) {
             String fileName = file_array[i]
             logger.info("process file:" + fileName)
-            if (i == 5) {
+            if (i == (int) (file_array.length / 2)) {
                 getRowCount(total, table_name)
                 def retry = 0
                 while (retry < 10) {
@@ -303,13 +312,13 @@ PROPERTIES (
         for (int i = 0; i < file_array.length; i++) {
             String fileName = file_array[i]
             logger.info("process file:" + fileName)
-            if (i == 5) {
+            if (i == (int) (file_array.length / 2)) {
                 def retry = 0
                 while (retry < 10) {
                     try {
-                        def rowCount = sql """select count(*) from ${table_name} where l_orderkey >=1000000 and l_orderkey <=5000000;"""
+                        def rowCount = sql """select count(*) from ${table_name} where l_orderkey >=10000;"""
                         log.info("rowCount:" + rowCount[0][0])
-                        sql """ delete from ${table_name} where l_orderkey >=1000000 and l_orderkey <=5000000; """
+                        sql """ delete from ${table_name} where l_orderkey >=10000; """
                         total -= rowCount[0][0]
                         break
                     } catch (Exception e) {
@@ -332,7 +341,7 @@ PROPERTIES (
         for (int i = 0; i < file_array.length; i++) {
             String fileName = file_array[i]
             logger.info("process file:" + fileName)
-            if (i == 5) {
+            if (i == (int) (file_array.length / 2)) {
                 def retry = 0
                 while (retry < 10) {
                     try {
@@ -345,7 +354,7 @@ PROPERTIES (
                     Thread.sleep(2000)
                 }
             }
-            if (i < 5) {
+            if (i < (int) (file_array.length / 2)) {
                 insert_data(fileName, table_name, STATE.NORMAL.value)
             } else {
                 insert_data(fileName, table_name, STATE.DROP_COLUMN.value)
@@ -363,7 +372,7 @@ PROPERTIES (
         for (int i = 0; i < file_array.length; i++) {
             String fileName = file_array[i]
             logger.info("process file:" + fileName)
-            if (i == 5) {
+            if (i == (int) (file_array.length / 2)) {
                 def retry = 0
                 while (retry < 10) {
                     try {
@@ -394,7 +403,7 @@ PROPERTIES (
         for (int i = 0; i < file_array.length; i++) {
             String fileName = file_array[i]
             logger.info("process file:" + fileName)
-            if (i == 5) {
+            if (i == (int) (file_array.length / 2)) {
                 def retry = 0
                 while (retry < 10) {
                     try {
@@ -417,7 +426,7 @@ PROPERTIES (
 
 
     def process = { table_name ->
-        for (int i = 1; i <= 4; i++) {
+        for (int i = 1; i <= 5; i++) {
             switch (i) {
                 case SC.TRUNCATE_TABLE.value:
                     truncate(table_name)
diff --git a/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_normal.groovy b/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_normal.groovy
index 5f3d86116e125c..83784555ca8a9a 100644
--- a/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_normal.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_normal.groovy
@@ -86,6 +86,7 @@ l_tax, l_returnflag,l_linestatus, l_shipdate,l_commitdate,l_receiptdate,l_shipin
     def process = {
         int total = 0;
         for (int k = 0; k < 3; k++) {
+            logger.info("round:" + k)
             for (int i = 1; i <= 10; i++) {
                 streamLoad {
                     table stream_load_table
diff --git a/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_schema_change.groovy b/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_schema_change.groovy
index d7cf67d54218ab..a3e362ea321dcf 100644
--- a/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_schema_change.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_stream_load_lineitem_schema_change.groovy
@@ -152,18 +152,30 @@ PROPERTIES (
     }
 
     def insert_data = { i, table_name ->
-        streamLoad {
-            table table_name
+        int j = 0;
+        while (true) {
+            if (j >= 18) {
+                throw new Exception("""fail to much time""")
+            }
+            try {
+                streamLoad {
+                    table table_name
 
-            set 'column_separator', '|'
-            set 'columns', columns + ",lo_dummy"
-            set 'group_commit', 'async_mode'
-            unset 'label'
-            file """${getS3Url()}/regression/tpch/sf1/lineitem.tbl.""" + i
+                    set 'column_separator', '|'
+                    set 'columns', columns + ",lo_dummy"
+                    set 'group_commit', 'async_mode'
+                    unset 'label'
+                    file """${getS3Url()}/regression/tpch/sf1/lineitem.tbl.""" + i
 
-            check { result, exception, startTime, endTime ->
-                checkStreamLoadResult(exception, result, rowCountArray[i - 1], rowCountArray[i - 1], 0, 0)
+                    check { result, exception, startTime, endTime ->
+                        checkStreamLoadResult(exception, result, rowCountArray[i - 1], rowCountArray[i - 1], 0, 0)
+                    }
+                }
+                break;
+            } catch (Exception e) {
+                Thread.sleep(10000)
             }
+            j++;
         }
         total += rowCountArray[i - 1];
     }
@@ -296,22 +308,25 @@ PROPERTIES (
     def change_order = { table_name ->
         create_stream_load_table(table_name)
         total = 0;
-        for (int i = 1; i <= 10; i++) {
-            logger.info("process file:" + i)
-            if (i == 2) {
-                def retry = 0
-                while (retry < 10) {
-                    try {
-                        sql """ alter table ${table_name} order by (l_orderkey,l_shipdate,l_linenumber, l_partkey,l_suppkey,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment); """
-                        break
-                    } catch (Exception e) {
-                        log.info("exception:", e)
+        for (int k = 0; k < 2; k++) {
+            logger.info("round:" + k)
+            for (int i = 1; i <= 10; i++) {
+                logger.info("process file:" + i)
+                if (k == 0 && i == 2) {
+                    def retry = 0
+                    while (retry < 10) {
+                        try {
+                            sql """ alter table ${table_name} order by (l_orderkey,l_shipdate,l_linenumber, l_partkey,l_suppkey,l_quantity,l_extendedprice,l_discount,l_tax,l_returnflag,l_linestatus,l_commitdate,l_receiptdate,l_shipinstruct,l_shipmode,l_comment); """
+                            break
+                        } catch (Exception e) {
+                            log.info("exception:", e)
+                        }
+                        Thread.sleep(2000)
+                        retry++
                     }
-                    Thread.sleep(2000)
-                    retry++
                 }
+                insert_data(i, table_name)
             }
-            insert_data(i, table_name)
         }
         logger.info("process change order total:" + total)
         assertTrue(getAlterTableState(table_name), "modify column order should success")
@@ -321,7 +336,7 @@ PROPERTIES (
 
 
     def process = { table_name ->
-        for (int i = 1; i <= 4; i++) {
+        for (int i = 1; i <= 5; i++) {
             switch (i) {
                 case SC.TRUNCATE_TABLE.value:
                     truncate(table_name)

From ce13a1d951b46e232086168ff779a0952b571bd2 Mon Sep 17 00:00:00 2001
From: minghong <englefly@gmail.com>
Date: Fri, 29 Dec 2023 01:00:27 +0800
Subject: [PATCH 139/299]  [fix](nereids) make runtime filter order stable
 #29203

---
 .../doris/nereids/processor/post/RuntimeFilterContext.java  | 6 +++---
 .../nereids/processor/post/RuntimeFilterGenerator.java      | 6 ++++--
 .../data/nereids_tpcds_shape_sf1000_p0/shape/query95.out    | 4 ++--
 .../nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out | 4 ++--
 .../nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out | 4 ++--
 .../data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out  | 4 ++--
 .../data/nereids_tpcds_shape_sf100_p0/shape/query95.out     | 4 ++--
 7 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
index 05467588f807ee..4360e5659c5944 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java
@@ -121,17 +121,17 @@ public boolean equals(Object other) {
     // you can see disjoint set data structure to learn the processing detailed.
     private final Map<NamedExpression, Pair<PhysicalRelation, Slot>> aliasTransferMap = Maps.newHashMap();
 
-    private final Map<Slot, ScanNode> scanNodeOfLegacyRuntimeFilterTarget = Maps.newHashMap();
+    private final Map<Slot, ScanNode> scanNodeOfLegacyRuntimeFilterTarget = Maps.newLinkedHashMap();
 
     private final Set<Plan> effectiveSrcNodes = Sets.newHashSet();
 
     // cte to related joins map which can extract common runtime filter to cte inside
-    private final Map<CTEId, Set<PhysicalHashJoin>> cteToJoinsMap = Maps.newHashMap();
+    private final Map<CTEId, Set<PhysicalHashJoin>> cteToJoinsMap = Maps.newLinkedHashMap();
 
     // cte candidates which can be pushed into common runtime filter into from outside
     private final Map<PhysicalCTEProducer, Map<EqualTo, PhysicalHashJoin>> cteRFPushDownMap = Maps.newLinkedHashMap();
 
-    private final Map<CTEId, PhysicalCTEProducer> cteProducerMap = Maps.newHashMap();
+    private final Map<CTEId, PhysicalCTEProducer> cteProducerMap = Maps.newLinkedHashMap();
 
     // cte whose runtime filter has been extracted
     private final Set<CTEId> processedCTE = Sets.newHashSet();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
index bae483c29397f3..45a14d3b633b03 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterGenerator.java
@@ -71,6 +71,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -392,7 +393,7 @@ private void pushDownRuntimeFilterCommon(PhysicalHashJoin<? extends Plan, ? exte
     private void collectPushDownCTEInfos(PhysicalHashJoin<? extends Plan, ? extends Plan> join,
             CascadesContext context) {
         RuntimeFilterContext ctx = context.getRuntimeFilterContext();
-        Set<CTEId> cteIds = new HashSet<>();
+        Set<CTEId> cteIds = new LinkedHashSet<>(); // use LinkedHashSet to make runtime filter order stable
         PhysicalPlan leftChild = (PhysicalPlan) join.left();
         PhysicalPlan rightChild = (PhysicalPlan) join.right();
 
@@ -404,7 +405,8 @@ private void collectPushDownCTEInfos(PhysicalHashJoin<? extends Plan, ? extends
         if ((leftHasCTE && !rightHasCTE) || (!leftHasCTE && rightHasCTE)) {
             for (CTEId id : cteIds) {
                 if (ctx.getCteToJoinsMap().get(id) == null) {
-                    Set<PhysicalHashJoin> newJoin = new HashSet<>();
+                    // use LinkedHashSet to make runtime filter order stable
+                    Set<PhysicalHashJoin> newJoin = new LinkedHashSet<>();
                     newJoin.add(join);
                     ctx.getCteToJoinsMap().put(id, newJoin);
                 } else {
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
index 6720aab10b8d4e..c50e58dd00e7c1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF6 ws_order_number->[ws_order_number,ws_order_number]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
 ----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
index d2ecc4813b5cdc..9fb38391e5e0d6 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF6 ws_order_number->[ws_order_number,ws_order_number]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
 ----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
index d2ecc4813b5cdc..9fb38391e5e0d6 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF6 ws_order_number->[ws_order_number,ws_order_number]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
 ----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
index bde9dbef392330..acee79a67ec2f7 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF6 ws_order_number->[ws_order_number,ws_order_number]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
 ----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
index bde9dbef392330..acee79a67ec2f7 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query95.out
@@ -21,14 +21,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = web_returns.wr_order_number)) otherCondition=()
 ----------------------PhysicalProject
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF6 wr_order_number->[ws_order_number,ws_order_number]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF5 wr_order_number->[ws_order_number];RF7 wr_order_number->[ws_order_number,ws_order_number]
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 --------------------------PhysicalDistribute
 ----------------------------PhysicalProject
 ------------------------------PhysicalOlapScan[web_returns]
-----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF7 ws_order_number->[ws_order_number,ws_order_number]
+----------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((ws1.ws_order_number = ws_wh.ws_order_number)) otherCondition=() build RFs:RF4 ws_order_number->[ws_order_number];RF6 ws_order_number->[ws_order_number,ws_order_number]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
 ----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )

From 7d44c5a1f1983e3c413683aded2f9dfac44e21f7 Mon Sep 17 00:00:00 2001
From: amory <wangqiannan@selectdb.com>
Date: Fri, 29 Dec 2023 01:00:47 +0800
Subject: [PATCH 140/299] [FIX](map)fix element_at in old planner make fe
 exception and regress cases from ck #29241

---
 .../doris/catalog/FunctionTypeDeducers.java   |   3 +-
 .../map_functions/test_basic_map_function.out | 101 +++++++++++++++++
 .../test_basic_map_function.groovy            | 104 ++++++++++++++++++
 3 files changed, 207 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/data/datatype_p0/nested_types/query/map_functions/test_basic_map_function.out
 create mode 100644 regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java
index 34b1d1f768e5ea..5e64d6378a86ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionTypeDeducers.java
@@ -52,7 +52,8 @@ public Type deduce(int argIdx, Type[] args) {
                 } else if (args[0].isNull()) {
                     // first arg is null, later element is not contains
                     return args[argIdx];
-                } else if (Type.isImplicitlyCastable(args[argIdx], ((ArrayType) args[0]).getItemType(), false, true)) {
+                } else if (args[0] instanceof ArrayType
+                        && Type.isImplicitlyCastable(args[argIdx], ((ArrayType) args[0]).getItemType(), false, true)) {
                     return args[argIdx];
                 } else {
                     return null;
diff --git a/regression-test/data/datatype_p0/nested_types/query/map_functions/test_basic_map_function.out b/regression-test/data/datatype_p0/nested_types/query/map_functions/test_basic_map_function.out
new file mode 100644
index 00000000000000..0bfc002238da3e
--- /dev/null
+++ b/regression-test/data/datatype_p0/nested_types/query/map_functions/test_basic_map_function.out
@@ -0,0 +1,101 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+sum-map-value
+
+-- !sql --
+1	1	4	6	0
+2	1	2	3	0
+
+-- !sql --
+literal-array-map
+
+-- !sql --
+2	[{"l":0.000000000000, "h":10000.000000000000, "t":0.100000000000}, {"l":10001.000000000000, "h":100000000000000.000000000000, "t":0.200000000000}]
+
+-- !sql --
+10000.000000000000	0.100000000000	10001.000000000000	0.200000000000
+
+-- !sql --
+insert-into-select-literal
+
+-- !sql --
+2020-01-01T00:00	{1234567898765432123456789:"c", -1234567898765432123456789:"d", -1:"a", 0:"b"}
+
+-- !sql --
+a	b	c	d
+
+-- !sql --
+b	a	b
+
+-- !sql --
+cast-literal
+
+-- !sql --
+{"amory":26, "amory_up":34}
+
+-- !sql --
+{"amory":26, "amory_up":34}
+
+-- !sql --
+{"amory":[26, 34], "commiter":[2023, 2024]}
+
+-- !sql --
+{"amory":[26, 34], "commiter":[2023, 2024]}
+
+-- !sql --
+{"amory":{"in":2023}, "amory_up":{"commiter":2024}}
+
+-- !sql --
+{"amory":{"in":2023}, "amory_up":{"commiter":2024}}
+
+-- !sql --
+sum-map-value
+
+-- !nereid_sql --
+1	1	4	6	0
+2	1	2	3	0
+
+-- !sql --
+literal-array-map
+
+-- !nereid_sql --
+1	[{"a":1.100000000000, "b":2.200000000000}, {"c":3.300000000000, "d":4.400000000000}]
+2	[{"l":0.000000000000, "h":10000.000000000000, "t":0.100000000000}, {"l":10001.000000000000, "h":100000000000000.000000000000, "t":0.200000000000}]
+
+-- !nereid_sql --
+\N	\N	10001.000000000000	0.200000000000
+1.100000000000	2.200000000000	\N	\N
+
+-- !sql --
+insert-into-select-literal
+
+-- !nereid_sql --
+2020-01-01T00:00	{-1:"a", 0:"b", 1234567898765432123456789:"c", -1234567898765432123456789:"d"}
+
+-- !nereid_sql --
+a	b	c	d
+
+-- !nereid_sql --
+b	a	b
+
+-- !sql --
+cast-literal
+
+-- !nereid_sql --
+{"amory":26, "amory_up":34}
+
+-- !nereid_sql --
+{"amory":26, "amory_up":34}
+
+-- !nereid_sql --
+{"amory":[26, 34], "commiter":[2023, 2024]}
+
+-- !nereid_sql --
+{"amory":[26, 34], "commiter":[2023, 2024]}
+
+-- !nereid_sql --
+{"amory":{"in":2023}, "amory_up":{"commiter":2024}}
+
+-- !nereid_sql --
+{"amory":{"in":2023}, "amory_up":{"commiter":2024}}
+
diff --git a/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy b/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
new file mode 100644
index 00000000000000..88993c5aeb0f39
--- /dev/null
+++ b/regression-test/suites/datatype_p0/nested_types/query/map_functions/test_basic_map_function.groovy
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_basic_map_function", "p0") {
+    sql """ ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false'); """
+    sql """set enable_nereids_planner=false"""
+    // ============ sum(map-value) ============
+    qt_sql """ SELECT "sum-map-value" """
+    sql """ DROP TABLE IF EXISTS t_map_amory;"""
+    sql """ CREATE TABLE IF NOT EXISTS t_map_amory(id int(11), m Map<String, largeint>) ENGINE=OLAP DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num' = '1');"""
+    sql """ INSERT INTO t_map_amory VALUES(1, map("a", 1, "b", 2, "c", 3));"""
+    sql """ INSERT INTO t_map_amory VALUES(1, map(concat("key",'a') , 1, "b", 2, "c", 3));"""
+    sql """ INSERT INTO t_map_amory VALUES(2, map("a", 1, "b", 2, "c", 3));"""
+    order_qt_sql """ SELECT id, sum(m['a']), sum(m['b']), sum(m['c']), sum(m['c'] = 0) FROM t_map_amory group by id order by id; """
+
+    // ============ literal array-map ============
+    qt_sql """ SELECT "literal-array-map" """
+    sql """ DROP TABLE IF EXISTS arr_nested_map_test_table;"""
+    sql """ CREATE TABLE IF NOT EXISTS arr_nested_map_test_table (id int, arr array<map<string, decimal(28,12)>>) ENGINE=OLAP DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num' = '1');"""
+    // old planner can't support literal array-map "No matching function with signature"
+    // sql """ INSERT INTO arr_nested_map_test_table VALUES(1, array(map("a", 1.1, "b", 2.2), map("c", 3.3, "d", 4.4)));"""
+    sql """ INSERT INTO arr_nested_map_test_table VALUES(2, [{'l': 0.0, 'h': 10000.0, 't': 0.1}, {'l': 10001.0, 'h': 100000000000000.0, 't': 0.2}]);"""
+    order_qt_sql """ SELECT * FROM arr_nested_map_test_table order by id ; """
+    order_qt_sql """ SELECT arr[1]['h'], arr[1]['t'], arr[2]['l'], arr[2]['t'] FROM arr_nested_map_test_table order by id ; """
+
+    // ============ insert into select literal ============
+    qt_sql """ SELECT "insert-into-select-literal" """
+    sql """ DROP TABLE IF EXISTS t_map_amory_1;"""
+    sql """ CREATE TABLE IF NOT EXISTS t_map_amory_1(id datetime, m Map<largeint, String>) ENGINE=OLAP DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num' = '1');"""
+    // old planner will make type deduction error
+    // sql """ INSERT INTO t_map_amory_1 SELECT '2020-01-01', map(-1, 'a', 0, 'b', cast('1234567898765432123456789' as largeint), 'c', cast('-1234567898765432123456789' as largeint), 'd');"""
+    sql """ INSERT INTO t_map_amory_1 SELECT '2020-01-01', map(cast('1234567898765432123456789' as largeint), 'c', cast('-1234567898765432123456789' as largeint), 'd', -1, 'a', 0, 'b'); """
+    qt_sql """ SELECT * FROM t_map_amory_1; """
+    qt_sql """ SELECT m[-1], m[0], m[cast('1234567898765432123456789' as largeint)], m[cast('-1234567898765432123456789' as largeint)] FROM t_map_amory_1;"""
+    qt_sql """ SELECT m[cast(0 as BIGINT)],  m[cast(-1 as Largeint)], m[cast(0 as largeint)] FROM t_map_amory_1;"""
+
+    // =================== cast literal ===================
+    qt_sql """ SELECT "cast-literal" """
+    qt_sql """ SELECT CAST({'amory':26, 'amory_up':34} as Map<String, int>);"""
+    qt_sql """ SELECT CAST('{\\'amory\\':26, \\'amory_up\\':34}' as Map<String, int>)"""
+    qt_sql """ SELECT CAST({'amory':[26, 34], 'commiter':[2023, 2024]} as Map<String, Array<int>>)"""
+    qt_sql """ SELECT CAST('{\\'amory\\':[26, 34], \\'commiter\\':[2023, 2024]}' as Map<String, Array<int>>)"""
+    qt_sql """ SELECT CAST({'amory':{'in': 2023}, 'amory_up':{'commiter': 2024}} as Map<String, Map<String, int>>)"""
+    qt_sql """ SELECT CAST('{\\'amory\\':{\\'in\\': 2023}, \\'amory_up\\':{\\'commiter\\': 2024}}' as Map<String, Map<String, int>>)"""
+
+
+    // test in nereids planner
+    sql """set enable_nereids_planner=true"""
+    sql """ set enable_fallback_to_original_planner=false"""
+
+    // ============ sum(map-value) ============
+    qt_sql """ SELECT "sum-map-value" """
+    sql """ DROP TABLE IF EXISTS t_map_amory;"""
+    sql """ CREATE TABLE IF NOT EXISTS t_map_amory(id int(11), m Map<String, largeint>) ENGINE=OLAP DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num' = '1');"""
+    sql """ INSERT INTO t_map_amory VALUES(1, map("a", 1, "b", 2, "c", 3));"""
+    sql """ INSERT INTO t_map_amory VALUES(1, map(concat("key",'a'), 1, "b", 2, "c", 3));"""
+    sql """ INSERT INTO t_map_amory VALUES(2, map("a", 1, "b", 2, "c", 3));"""
+    order_qt_nereid_sql """ SELECT id, sum(m['a']), sum(m['b']), sum(m['c']), sum(m['c'] = 0) FROM t_map_amory group by id order by id; """
+
+    // ============ literal array-map ============
+    qt_sql """ SELECT "literal-array-map" """
+    sql """ DROP TABLE IF EXISTS arr_nested_map_test_table;"""
+    sql """ CREATE TABLE IF NOT EXISTS arr_nested_map_test_table (id int, arr array<map<string, decimal(28,12)>>) ENGINE=OLAP DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num' = '1');"""
+    sql """ INSERT INTO arr_nested_map_test_table VALUES(1, array(map("a", 1.1, "b", 2.2), map("c", 3.3, "d", 4.4)));"""
+    sql """ INSERT INTO arr_nested_map_test_table VALUES(2, [{'l': 0.0, 'h': 10000.0, 't': 0.1}, {'l': 10001.0, 'h': 100000000000000.0, 't': 0.2}]);"""
+    order_qt_nereid_sql """ SELECT * FROM arr_nested_map_test_table order by id ; """
+    order_qt_nereid_sql """ SELECT arr[1]['a'], arr[1]['b'], arr[2]['l'], arr[2]['t'] FROM arr_nested_map_test_table order by id ; """
+
+    // ============ insert into select literal ============
+    qt_sql """ SELECT "insert-into-select-literal" """
+    sql """ DROP TABLE IF EXISTS t_map_amory_1;"""
+    sql """ CREATE TABLE IF NOT EXISTS t_map_amory_1(id datetime, m Map<largeint, String>) ENGINE=OLAP DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num' = '1');"""
+    sql """ INSERT INTO t_map_amory_1 SELECT '2020-01-01', map(-1, 'a', 0, 'b', cast('1234567898765432123456789' as largeint), 'c', cast('-1234567898765432123456789' as largeint), 'd');"""
+    qt_nereid_sql """ SELECT * FROM t_map_amory_1; """
+    qt_nereid_sql """ SELECT m[-1], m[0], m[cast('1234567898765432123456789' as largeint)], m[cast('-1234567898765432123456789' as largeint)] FROM t_map_amory_1;"""
+    qt_nereid_sql """ SELECT m[cast(0 as BIGINT)],  m[cast(-1 as Largeint)], m[cast(0 as largeint)] FROM t_map_amory_1;"""
+
+    // =================== cast literal ===================
+    qt_sql """ SELECT "cast-literal" """
+    qt_nereid_sql """ SELECT CAST({'amory':26, 'amory_up':34} as Map<String, int>);"""
+    qt_nereid_sql """ SELECT CAST('{\\'amory\\':26, \\'amory_up\\':34}' as Map<String, int>)"""
+    qt_nereid_sql """ SELECT CAST({'amory':[26, 34], 'commiter':[2023, 2024]} as Map<String, Array<int>>)"""
+    qt_nereid_sql """ SELECT CAST('{\\'amory\\':[26, 34], \\'commiter\\':[2023, 2024]}' as Map<String, Array<int>>)"""
+    qt_nereid_sql """ SELECT CAST({'amory':{'in': 2023}, 'amory_up':{'commiter': 2024}} as Map<String, Map<String, int>>)"""
+    qt_nereid_sql """ SELECT CAST('{\\'amory\\':{\\'in\\': 2023}, \\'amory_up\\':{\\'commiter\\': 2024}}' as Map<String, Map<String, int>>)"""
+
+
+
+
+}

From c3679a2750ad84ba36df09f29a2dc8146f5c0649 Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Fri, 29 Dec 2023 07:08:12 +0800
Subject: [PATCH 141/299] [opt](Nereids) derive physical properties of Project
 and Filter (#29171)

---
 .../properties/RequestPropertyDeriver.java    |  50 ++++++
 .../shape/query17.out                         |   4 +-
 .../shape/query24.out                         |  12 +-
 .../shape/query25.out                         |   4 +-
 .../shape/query33.out                         | 156 +++++++++---------
 .../shape/query47.out                         |   4 +-
 .../shape/query57.out                         |   4 +-
 .../shape/query89.out                         |   4 +-
 .../noStatsRfPrune/query1.out                 |  22 +--
 .../noStatsRfPrune/query37.out                |  14 +-
 .../noStatsRfPrune/query39.out                |  33 ++--
 .../noStatsRfPrune/query46.out                |  47 +++---
 .../noStatsRfPrune/query65.out                |  40 ++---
 .../noStatsRfPrune/query68.out                |  47 +++---
 .../noStatsRfPrune/query77.out                |  94 ++++++-----
 .../noStatsRfPrune/query82.out                |  14 +-
 .../no_stats_shape/query1.out                 |  22 +--
 .../no_stats_shape/query37.out                |  14 +-
 .../no_stats_shape/query39.out                |  33 ++--
 .../no_stats_shape/query46.out                |  47 +++---
 .../no_stats_shape/query65.out                |  38 ++---
 .../no_stats_shape/query68.out                |  47 +++---
 .../no_stats_shape/query77.out                |  94 ++++++-----
 .../no_stats_shape/query82.out                |  14 +-
 .../rf_prune/query17.out                      |   4 +-
 .../rf_prune/query24.out                      |  12 +-
 .../rf_prune/query25.out                      |   4 +-
 .../rf_prune/query33.out                      | 122 +++++++-------
 .../rf_prune/query39.out                      |  33 ++--
 .../rf_prune/query57.out                      |   4 +-
 .../rf_prune/query72.out                      |  59 ++++---
 .../rf_prune/query77.out                      |  29 ++--
 .../shape/query17.out                         |   4 +-
 .../shape/query24.out                         |  12 +-
 .../shape/query25.out                         |   4 +-
 .../shape/query33.out                         | 122 +++++++-------
 .../shape/query39.out                         |  33 ++--
 .../shape/query57.out                         |   4 +-
 .../shape/query72.out                         |  59 ++++---
 .../shape/query77.out                         |  29 ++--
 40 files changed, 713 insertions(+), 679 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
index ef7d72b095ed5c..52b69399af2fa2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java
@@ -22,6 +22,7 @@
 import org.apache.doris.nereids.jobs.JobContext;
 import org.apache.doris.nereids.memo.GroupExpression;
 import org.apache.doris.nereids.properties.DistributionSpecHash.ShuffleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
 import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -33,11 +34,13 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEAnchor;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalFileSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalSetOperation;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
@@ -47,9 +50,12 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
 /**
@@ -275,6 +281,50 @@ public Void visitPhysicalPartitionTopN(PhysicalPartitionTopN<? extends Plan> par
         return null;
     }
 
+    @Override
+    public Void visitPhysicalProject(PhysicalProject<? extends Plan> project, PlanContext context) {
+        DistributionSpec parentDist = requestPropertyFromParent.getDistributionSpec();
+        if (!(parentDist instanceof DistributionSpecHash)) {
+            return super.visitPhysicalProject(project, context);
+        }
+        DistributionSpecHash hashDist = (DistributionSpecHash) parentDist;
+        Map<ExprId, NamedExpression> exprIdToProjection = project.getProjects().stream()
+                .collect(Collectors.toMap(NamedExpression::getExprId, n -> n, (n1, n2) -> n1));
+        Map<ExprId, ExprId> exprIdMap = Maps.newHashMap();
+        for (ExprId exprId : hashDist.getExprIdToEquivalenceSet().keySet()) {
+            if (!exprIdToProjection.containsKey(exprId)) {
+                return super.visitPhysicalProject(project, context);
+            }
+            NamedExpression projection = exprIdToProjection.get(exprId);
+            if (projection instanceof Alias) {
+                if (((Alias) projection).child() instanceof SlotReference) {
+                    exprIdMap.put(exprId, ((SlotReference) ((Alias) projection).child()).getExprId());
+                } else {
+                    return super.visitPhysicalProject(project, context);
+                }
+            } else if (projection instanceof SlotReference) {
+                exprIdMap.put(exprId, exprId);
+            } else {
+                return super.visitPhysicalProject(project, context);
+            }
+        }
+        addRequestPropertyToChildren(PhysicalProperties.ANY);
+        addRequestPropertyToChildren(new PhysicalProperties(
+                hashDist.project(exprIdMap, ImmutableSet.of(), DistributionSpecAny.INSTANCE)));
+        return null;
+    }
+
+    @Override
+    public Void visitPhysicalFilter(PhysicalFilter<? extends Plan> filter, PlanContext context) {
+        DistributionSpec parentDist = requestPropertyFromParent.getDistributionSpec();
+        if (!(parentDist instanceof DistributionSpecHash)) {
+            return super.visitPhysicalFilter(filter, context);
+        }
+        addRequestPropertyToChildren(PhysicalProperties.ANY);
+        addRequestPropertyToChildren(new PhysicalProperties(parentDist));
+        return null;
+    }
+
     @Override
     public Void visitPhysicalFileSink(PhysicalFileSink<? extends Plan> fileSink, PlanContext context) {
         addRequestPropertyToChildren(PhysicalProperties.GATHER);
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query17.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query17.out
index a9ca31b6e2ea08..6fe2631fc288ad 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query17.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query17.out
@@ -19,8 +19,8 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3'))
 --------------------------------PhysicalOlapScan[date_dim]
---------------------PhysicalDistribute
-----------------------PhysicalProject
+--------------------PhysicalProject
+----------------------PhysicalDistribute
 ------------------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk]
 --------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[ss_item_sk]
 ----------------------------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out
index 467ba7d635c681..f07e48b8f0475e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query24.out
@@ -8,9 +8,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------hashAgg[LOCAL]
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF5 sr_item_sk->[ss_item_sk];RF6 sr_ticket_number->[ss_ticket_number]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[ss_item_sk]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[ss_item_sk]
+--------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 ca_zip->[s_zip];RF3 c_customer_sk->[ss_customer_sk]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
@@ -29,9 +29,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[customer_address]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[item]
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[item]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[store_returns]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query25.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query25.out
index fca0fef64ccea2..7422ef1f1b1d1b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query25.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query25.out
@@ -18,8 +18,8 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------filter((d3.d_moy <= 10) and (d3.d_moy >= 4) and (d3.d_year = 1999))
 ------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute
---------------------PhysicalProject
+------------------PhysicalProject
+--------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk]
 ------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[ss_item_sk]
 --------------------------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query33.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query33.out
index 4b3ccd3fcf4f8f..5989257c155f64 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query33.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query33.out
@@ -9,93 +9,93 @@ PhysicalResultSink
 ------------hashAgg[LOCAL]
 --------------PhysicalUnion
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF3 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ss_addr_sk]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3
-----------------------------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Books'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF3
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2001))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
-----------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF0 i_manufact_id->[i_manufact_id]
+--------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2001))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[item] apply RFs: RF0
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------filter((item.i_category = 'Books'))
---------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalOlapScan[item]
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF7 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[cs_item_sk]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF6 ca_address_sk->[cs_bill_addr_sk]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6 RF7
-----------------------------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Books'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF7
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[cs_bill_addr_sk]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[cs_sold_date_sk]
 ------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2001))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
-----------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF4 i_manufact_id->[i_manufact_id]
+--------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2001))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[item] apply RFs: RF4
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------filter((item.i_category = 'Books'))
---------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalOlapScan[item]
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF11 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF11 i_item_sk->[ws_item_sk]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF10 ca_address_sk->[ws_bill_addr_sk]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF9 d_date_sk->[ws_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF9 RF10 RF11
-----------------------------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Books'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF11
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 i_item_sk->[ws_item_sk]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF9 ca_address_sk->[ws_bill_addr_sk]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 ------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2001))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
-----------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF8 i_manufact_id->[i_manufact_id]
+--------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9 RF10
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy = 3) and (date_dim.d_year = 2001))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[item] apply RFs: RF8
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------filter((item.i_category = 'Books'))
---------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
index b853af385b6e8c..13becca105c878 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query47.out
@@ -7,8 +7,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------PhysicalQuickSort[LOCAL_SORT]
 ----------PhysicalWindow
 ------------PhysicalQuickSort[LOCAL_SORT]
---------------PhysicalDistribute
-----------------PhysicalProject
+--------------PhysicalProject
+----------------PhysicalDistribute
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute
 ----------------------hashAgg[LOCAL]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out
index df6ac1abdd4ba5..23f840ef0c46ca 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query57.out
@@ -7,8 +7,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------PhysicalQuickSort[LOCAL_SORT]
 ----------PhysicalWindow
 ------------PhysicalQuickSort[LOCAL_SORT]
---------------PhysicalDistribute
-----------------PhysicalProject
+--------------PhysicalProject
+----------------PhysicalDistribute
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute
 ----------------------hashAgg[LOCAL]
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out
index 07b4889ada0ef4..f1b4e84536cfe3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query89.out
@@ -9,8 +9,8 @@ PhysicalResultSink
 ------------filter((if(( not (avg_monthly_sales = 0.0000)), (cast(abs((sum_sales - cast(avg_monthly_sales as DECIMALV3(38, 2)))) as DECIMALV3(38, 10)) / avg_monthly_sales), NULL) > 0.100000))
 --------------PhysicalWindow
 ----------------PhysicalQuickSort[LOCAL_SORT]
-------------------PhysicalDistribute
---------------------PhysicalProject
+------------------PhysicalProject
+--------------------PhysicalDistribute
 ----------------------hashAgg[GLOBAL]
 ------------------------PhysicalDistribute
 --------------------------hashAgg[LOCAL]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out
index 8b0038cf33f904..400e67a5dadaad 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query1.out
@@ -19,9 +19,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------PhysicalDistribute
 --------PhysicalTopN[LOCAL_SORT]
 ----------PhysicalProject
-------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ctr_store_sk]
+------------hashJoin[INNER_JOIN] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DOUBLE) > cast((avg(cast(ctr_total_return as DECIMALV3(38, 4))) * 1.2) as DOUBLE)))
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DOUBLE) > cast((avg(cast(ctr_total_return as DECIMALV3(38, 4))) * 1.2) as DOUBLE)))
+----------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ctr_store_sk]
 ------------------PhysicalDistribute
 --------------------hashJoin[INNER_JOIN] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ctr_customer_sk]
 ----------------------PhysicalDistribute
@@ -29,14 +29,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer]
-------------------hashAgg[GLOBAL]
+------------------PhysicalDistribute
+--------------------PhysicalProject
+----------------------filter((store.s_state = 'SD'))
+------------------------PhysicalOlapScan[store]
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute
+------------------hashAgg[LOCAL]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
---------------PhysicalDistribute
-----------------PhysicalProject
-------------------filter((store.s_state = 'SD'))
---------------------PhysicalOlapScan[store]
+----------------------PhysicalProject
+------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query37.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query37.out
index c385358a418bdc..0c15104a502c5b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query37.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query37.out
@@ -12,9 +12,9 @@ PhysicalResultSink
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+----------------------PhysicalDistribute
 ------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
 --------------------------PhysicalProject
 ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100))
@@ -23,8 +23,8 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21'))
 --------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856))
-------------------------------PhysicalOlapScan[item]
+----------------------PhysicalDistribute
+------------------------PhysicalProject
+--------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856))
+----------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out
index c96febe88c9ba5..ab47ceb3357b29 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query39.out
@@ -4,25 +4,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalCteProducer ( cteId=CTEId#0 )
 ----PhysicalProject
 ------filter((if((mean = 0), 0, (stdev / mean)) > 1))
---------hashAgg[GLOBAL]
-----------PhysicalDistribute
-------------hashAgg[LOCAL]
---------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=()
-------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=()
-------------------------PhysicalOlapScan[inventory] apply RFs: RF1
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[item]
---------------------PhysicalDistribute
-----------------------PhysicalProject
-------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
---------------------------PhysicalOlapScan[date_dim]
+--------hashAgg[LOCAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=()
+--------------PhysicalDistribute
+----------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=()
+----------------------PhysicalOlapScan[inventory] apply RFs: RF1
+----------------------PhysicalDistribute
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[item]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------PhysicalOlapScan[warehouse]
+----------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
+------------------------PhysicalOlapScan[date_dim]
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------PhysicalOlapScan[warehouse]
 --PhysicalResultSink
 ----PhysicalQuickSort[MERGE_SORT]
 ------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out
index 04b461fb20ef89..d80df89216d62e 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query46.out
@@ -11,33 +11,32 @@ PhysicalResultSink
 ----------------hashJoin[INNER_JOIN] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=()
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
-------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
-------------------------------------------PhysicalProject
---------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
-------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------PhysicalDistribute
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=()
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+------------------------------------PhysicalProject
+--------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
---------------------------------------------PhysicalOlapScan[household_demographics]
+------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
+----------------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------------PhysicalDistribute
 --------------------------------------PhysicalProject
-----------------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
-------------------------------------------PhysicalOlapScan[store]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer_address]
+----------------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
+----------------------------------------PhysicalOlapScan[store]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[customer_address]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query65.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query65.out
index 1c31961c47d327..2ccd3a7b6403aa 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query65.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query65.out
@@ -5,9 +5,9 @@ PhysicalResultSink
 ----PhysicalDistribute
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=()
+----------hashJoin[INNER_JOIN] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))) build RFs:RF4 ss_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))) build RFs:RF3 ss_store_sk->[ss_store_sk]
+--------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=()
 ----------------PhysicalDistribute
 ------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = sc.ss_item_sk)) otherCondition=()
 --------------------hashAgg[GLOBAL]
@@ -16,7 +16,7 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF3
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF4
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
 ----------------------------------filter((date_dim.d_month_seq <= 1232) and (date_dim.d_month_seq >= 1221))
@@ -24,22 +24,22 @@ PhysicalResultSink
 --------------------PhysicalDistribute
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[item]
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashAgg[GLOBAL]
---------------------------PhysicalDistribute
-----------------------------hashAgg[LOCAL]
+----------------PhysicalDistribute
+------------------PhysicalProject
+--------------------PhysicalOlapScan[store]
+------------hashAgg[GLOBAL]
+--------------PhysicalDistribute
+----------------hashAgg[LOCAL]
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((date_dim.d_month_seq <= 1232) and (date_dim.d_month_seq >= 1221))
-----------------------------------------PhysicalOlapScan[date_dim]
-------------PhysicalDistribute
---------------PhysicalProject
-----------------PhysicalOlapScan[store]
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------filter((date_dim.d_month_seq <= 1232) and (date_dim.d_month_seq >= 1221))
+------------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out
index b9d8e6d3a32ad0..becdba20be1f70 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query68.out
@@ -11,33 +11,32 @@ PhysicalResultSink
 ----------------hashJoin[INNER_JOIN] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=()
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
-------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
-------------------------------------------PhysicalProject
---------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000))
-------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------PhysicalDistribute
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=()
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+------------------------------------PhysicalProject
+--------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
---------------------------------------------PhysicalOlapScan[household_demographics]
+------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000))
+----------------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------------PhysicalDistribute
 --------------------------------------PhysicalProject
-----------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
-------------------------------------------PhysicalOlapScan[store]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer_address]
+----------------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
+----------------------------------------PhysicalOlapScan[store]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[customer_address]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[customer]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query77.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query77.out
index a25d7bcded0ba0..13fd3a801e3977 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query77.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query77.out
@@ -13,28 +13,27 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((ss.s_store_sk = sr.s_store_sk)) otherCondition=()
 ------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=()
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=()
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[store]
-------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_store_sk = store.s_store_sk)) otherCondition=()
+----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[store]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------hashAgg[LOCAL]
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_store_sk = store.s_store_sk)) otherCondition=()
+----------------------------------PhysicalDistribute
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[sr_returned_date_sk]
 ----------------------------------------PhysicalProject
@@ -43,9 +42,9 @@ PhysicalResultSink
 ------------------------------------------PhysicalProject
 --------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
 ----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[store]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[store]
 --------------------PhysicalProject
 ----------------------NestedLoopJoin[CROSS_JOIN]
 ------------------------PhysicalProject
@@ -76,28 +75,27 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((ws.wp_web_page_sk = wr.wp_web_page_sk)) otherCondition=()
 ------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF9 wp_web_page_sk->[ws_web_page_sk]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF9 wp_web_page_sk->[ws_web_page_sk]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 --------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[web_page]
-------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
+----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[web_page]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------hashAgg[LOCAL]
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
+----------------------------------PhysicalDistribute
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[wr_returned_date_sk]
 ----------------------------------------PhysicalProject
@@ -106,7 +104,7 @@ PhysicalResultSink
 ------------------------------------------PhysicalProject
 --------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
 ----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[web_page]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[web_page]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query82.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query82.out
index a9891820a37c81..e5b1ee4038b413 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query82.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query82.out
@@ -12,9 +12,9 @@ PhysicalResultSink
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[store_sales] apply RFs: RF2
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+----------------------PhysicalDistribute
 ------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
 --------------------------PhysicalProject
 ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100))
@@ -23,8 +23,8 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09'))
 --------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639))
-------------------------------PhysicalOlapScan[item]
+----------------------PhysicalDistribute
+------------------------PhysicalProject
+--------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639))
+----------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out
index 8b0038cf33f904..400e67a5dadaad 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query1.out
@@ -19,9 +19,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------PhysicalDistribute
 --------PhysicalTopN[LOCAL_SORT]
 ----------PhysicalProject
-------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ctr_store_sk]
+------------hashJoin[INNER_JOIN] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DOUBLE) > cast((avg(cast(ctr_total_return as DECIMALV3(38, 4))) * 1.2) as DOUBLE)))
 --------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((ctr1.ctr_store_sk = ctr2.ctr_store_sk)) otherCondition=((cast(ctr_total_return as DOUBLE) > cast((avg(cast(ctr_total_return as DECIMALV3(38, 4))) * 1.2) as DOUBLE)))
+----------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = ctr1.ctr_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ctr_store_sk]
 ------------------PhysicalDistribute
 --------------------hashJoin[INNER_JOIN] hashCondition=((ctr1.ctr_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF1 c_customer_sk->[ctr_customer_sk]
 ----------------------PhysicalDistribute
@@ -29,14 +29,14 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
 --------------------------PhysicalOlapScan[customer]
-------------------hashAgg[GLOBAL]
+------------------PhysicalDistribute
+--------------------PhysicalProject
+----------------------filter((store.s_state = 'SD'))
+------------------------PhysicalOlapScan[store]
+--------------hashAgg[GLOBAL]
+----------------PhysicalDistribute
+------------------hashAgg[LOCAL]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
---------------PhysicalDistribute
-----------------PhysicalProject
-------------------filter((store.s_state = 'SD'))
---------------------PhysicalOlapScan[store]
+----------------------PhysicalProject
+------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query37.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query37.out
index c385358a418bdc..0c15104a502c5b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query37.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query37.out
@@ -12,9 +12,9 @@ PhysicalResultSink
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[catalog_sales] apply RFs: RF2
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+----------------------PhysicalDistribute
 ------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
 --------------------------PhysicalProject
 ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100))
@@ -23,8 +23,8 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------filter((date_dim.d_date <= '1999-04-22') and (date_dim.d_date >= '1999-02-21'))
 --------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856))
-------------------------------PhysicalOlapScan[item]
+----------------------PhysicalDistribute
+------------------------PhysicalProject
+--------------------------filter((item.i_current_price <= 75.00) and (item.i_current_price >= 45.00) and i_manufact_id IN (1000, 707, 747, 856))
+----------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out
index cdd758313a3bd5..5822f2b3c2759a 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query39.out
@@ -4,25 +4,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalCteProducer ( cteId=CTEId#0 )
 ----PhysicalProject
 ------filter((if((mean = 0), 0, (stdev / mean)) > 1))
---------hashAgg[GLOBAL]
-----------PhysicalDistribute
-------------hashAgg[LOCAL]
---------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=() build RFs:RF2 w_warehouse_sk->[inv_warehouse_sk]
-------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk]
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[inv_item_sk]
-------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------PhysicalOlapScan[item]
---------------------PhysicalDistribute
-----------------------PhysicalProject
-------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
---------------------------PhysicalOlapScan[date_dim]
+--------hashAgg[LOCAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=() build RFs:RF2 w_warehouse_sk->[inv_warehouse_sk]
+--------------PhysicalDistribute
+----------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[inv_date_sk]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF0 i_item_sk->[inv_item_sk]
+----------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2
+----------------------PhysicalDistribute
+------------------------PhysicalProject
+--------------------------PhysicalOlapScan[item]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------PhysicalOlapScan[warehouse]
+----------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
+------------------------PhysicalOlapScan[date_dim]
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------PhysicalOlapScan[warehouse]
 --PhysicalResultSink
 ----PhysicalQuickSort[MERGE_SORT]
 ------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out
index dd973723ac0b37..478db9b52a4e44 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query46.out
@@ -11,33 +11,32 @@ PhysicalResultSink
 ----------------hashJoin[INNER_JOIN] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
-------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
-------------------------------------------PhysicalProject
---------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
-------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------PhysicalDistribute
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+------------------------------------PhysicalProject
+--------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
---------------------------------------------PhysicalOlapScan[household_demographics]
+------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------filter(d_dow IN (0, 6) and d_year IN (1999, 2000, 2001))
+----------------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------------PhysicalDistribute
 --------------------------------------PhysicalProject
-----------------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
-------------------------------------------PhysicalOlapScan[store]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer_address]
+----------------------------------------filter(((household_demographics.hd_dep_count = 6) OR (household_demographics.hd_vehicle_count = 0)))
+------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------filter(s_city IN ('Centerville', 'Fairview', 'Five Points', 'Liberty', 'Oak Grove'))
+----------------------------------------PhysicalOlapScan[store]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[customer_address]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[customer] apply RFs: RF5
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query65.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query65.out
index 8ee05d0a26aeae..ab0f422a10fb43 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query65.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query65.out
@@ -5,9 +5,9 @@ PhysicalResultSink
 ----PhysicalDistribute
 ------PhysicalTopN[LOCAL_SORT]
 --------PhysicalProject
-----------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF4 s_store_sk->[ss_store_sk]
+----------hashJoin[INNER_JOIN] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))) build RFs:RF4 ss_store_sk->[ss_store_sk]
 ------------PhysicalProject
---------------hashJoin[INNER_JOIN] hashCondition=((sb.ss_store_sk = sc.ss_store_sk)) otherCondition=((cast(revenue as DOUBLE) <= cast((0.1 * ave) as DOUBLE))) build RFs:RF3 ss_store_sk->[ss_store_sk]
+--------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = sc.ss_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk]
 ----------------PhysicalDistribute
 ------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = sc.ss_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk]
 --------------------hashAgg[GLOBAL]
@@ -24,22 +24,22 @@ PhysicalResultSink
 --------------------PhysicalDistribute
 ----------------------PhysicalProject
 ------------------------PhysicalOlapScan[item]
-----------------hashAgg[GLOBAL]
-------------------PhysicalDistribute
---------------------hashAgg[LOCAL]
-----------------------PhysicalProject
-------------------------hashAgg[GLOBAL]
---------------------------PhysicalDistribute
-----------------------------hashAgg[LOCAL]
+----------------PhysicalDistribute
+------------------PhysicalProject
+--------------------PhysicalOlapScan[store]
+------------hashAgg[GLOBAL]
+--------------PhysicalDistribute
+----------------hashAgg[LOCAL]
+------------------PhysicalProject
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((date_dim.d_month_seq <= 1232) and (date_dim.d_month_seq >= 1221))
-----------------------------------------PhysicalOlapScan[date_dim]
-------------PhysicalDistribute
---------------PhysicalProject
-----------------PhysicalOlapScan[store]
+--------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------filter((date_dim.d_month_seq <= 1232) and (date_dim.d_month_seq >= 1221))
+------------------------------------PhysicalOlapScan[date_dim]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out
index a9067c6eb54ee9..6e468ca21d9279 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query68.out
@@ -11,33 +11,32 @@ PhysicalResultSink
 ----------------hashJoin[INNER_JOIN] hashCondition=((dn.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
-----------------------hashAgg[GLOBAL]
-------------------------PhysicalDistribute
---------------------------hashAgg[LOCAL]
-----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
-------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
-------------------------------------------PhysicalProject
---------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000))
-------------------------------------------------PhysicalOlapScan[date_dim]
---------------------------------------PhysicalDistribute
+----------------------hashAgg[LOCAL]
+------------------------PhysicalProject
+--------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF3 ca_address_sk->[ss_addr_sk]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF2 s_store_sk->[ss_store_sk]
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF1 hd_demo_sk->[ss_hdemo_sk]
+------------------------------------PhysicalProject
+--------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ----------------------------------------PhysicalProject
-------------------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
---------------------------------------------PhysicalOlapScan[household_demographics]
+------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2 RF3 RF4
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------filter((date_dim.d_dom <= 2) and (date_dim.d_dom >= 1) and d_year IN (1998, 1999, 2000))
+----------------------------------------------PhysicalOlapScan[date_dim]
 ------------------------------------PhysicalDistribute
 --------------------------------------PhysicalProject
-----------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
-------------------------------------------PhysicalOlapScan[store]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer_address]
+----------------------------------------filter(((household_demographics.hd_dep_count = 8) OR (household_demographics.hd_vehicle_count = -1)))
+------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------filter(s_city IN ('Five Points', 'Pleasant Hill'))
+----------------------------------------PhysicalOlapScan[store]
+----------------------------PhysicalDistribute
+------------------------------PhysicalProject
+--------------------------------PhysicalOlapScan[customer_address]
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[customer] apply RFs: RF5
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query77.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query77.out
index 49dbb5aeabccac..b3c830559371bf 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query77.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query77.out
@@ -13,28 +13,27 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((ss.s_store_sk = sr.s_store_sk)) otherCondition=()
 ------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF3 s_store_sk->[ss_store_sk]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[store]
-------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[sr_store_sk]
+----------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[store]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------hashAgg[LOCAL]
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_store_sk = store.s_store_sk)) otherCondition=() build RFs:RF1 s_store_sk->[sr_store_sk]
+----------------------------------PhysicalDistribute
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_returns.sr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[sr_returned_date_sk]
 ----------------------------------------PhysicalProject
@@ -43,9 +42,9 @@ PhysicalResultSink
 ------------------------------------------PhysicalProject
 --------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
 ----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[store]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[store]
 --------------------PhysicalProject
 ----------------------NestedLoopJoin[CROSS_JOIN]
 ------------------------PhysicalProject
@@ -76,28 +75,27 @@ PhysicalResultSink
 --------------------PhysicalProject
 ----------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((ws.wp_web_page_sk = wr.wp_web_page_sk)) otherCondition=()
 ------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF9 wp_web_page_sk->[ws_web_page_sk]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF9 wp_web_page_sk->[ws_web_page_sk]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
 --------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[web_page]
-------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
+----------------------------------------PhysicalOlapScan[web_sales] apply RFs: RF8 RF9
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[web_page]
+------------------------PhysicalDistribute
+--------------------------PhysicalProject
+----------------------------hashAgg[LOCAL]
+------------------------------PhysicalProject
+--------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
+----------------------------------PhysicalDistribute
 ------------------------------------PhysicalProject
 --------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[wr_returned_date_sk]
 ----------------------------------------PhysicalProject
@@ -106,7 +104,7 @@ PhysicalResultSink
 ------------------------------------------PhysicalProject
 --------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
 ----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
---------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[web_page]
+----------------------------------PhysicalDistribute
+------------------------------------PhysicalProject
+--------------------------------------PhysicalOlapScan[web_page]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query82.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query82.out
index a9891820a37c81..e5b1ee4038b413 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query82.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query82.out
@@ -12,9 +12,9 @@ PhysicalResultSink
 ------------------PhysicalDistribute
 --------------------PhysicalProject
 ----------------------PhysicalOlapScan[store_sales] apply RFs: RF2
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+------------------PhysicalProject
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+----------------------PhysicalDistribute
 ------------------------hashJoin[INNER_JOIN] hashCondition=((date_dim.d_date_sk = inventory.inv_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
 --------------------------PhysicalProject
 ----------------------------filter((inventory.inv_quantity_on_hand <= 500) and (inventory.inv_quantity_on_hand >= 100))
@@ -23,8 +23,8 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------filter((date_dim.d_date <= '1999-09-07') and (date_dim.d_date >= '1999-07-09'))
 --------------------------------PhysicalOlapScan[date_dim]
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639))
-------------------------------PhysicalOlapScan[item]
+----------------------PhysicalDistribute
+------------------------PhysicalProject
+--------------------------filter((item.i_current_price <= 47.00) and (item.i_current_price >= 17.00) and i_manufact_id IN (138, 169, 339, 639))
+----------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query17.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query17.out
index 050c26304b09f9..8f0b805f6eaee5 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query17.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query17.out
@@ -19,8 +19,8 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3'))
 --------------------------------PhysicalOlapScan[date_dim]
---------------------PhysicalDistribute
-----------------------PhysicalProject
+--------------------PhysicalProject
+----------------------PhysicalDistribute
 ------------------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 --------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=()
 ----------------------------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out
index c15f6c92a3ecf1..e8ad4ab9231542 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query24.out
@@ -8,9 +8,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------hashAgg[LOCAL]
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=()
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=()
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=()
+--------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=()
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
@@ -29,9 +29,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[customer_address]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[item]
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[item]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[store_returns]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query25.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query25.out
index 7e87daacbc9bd4..610a113c744057 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query25.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query25.out
@@ -18,8 +18,8 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------filter((d3.d_moy <= 10) and (d3.d_moy >= 4) and (d3.d_year = 2000))
 ------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute
---------------------PhysicalProject
+------------------PhysicalProject
+--------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=()
 ------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=()
 --------------------------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query33.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query33.out
index 1b3dccd334b980..7ed55c113ad5ea 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query33.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query33.out
@@ -9,74 +9,78 @@ PhysicalResultSink
 ------------hashAgg[LOCAL]
 --------------PhysicalUnion
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF3 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ss_addr_sk]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3
-----------------------------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Home'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF3
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=()
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
-----------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF0 i_manufact_id->[i_manufact_id]
+--------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[item] apply RFs: RF0
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------filter((item.i_category = 'Home'))
---------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalOlapScan[item]
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF7 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[cs_item_sk]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF6 ca_address_sk->[cs_bill_addr_sk]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6 RF7
-----------------------------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Home'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF7
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[cs_bill_addr_sk]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[cs_sold_date_sk]
 ------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
-----------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF4 i_manufact_id->[i_manufact_id]
+--------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[item] apply RFs: RF4
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------filter((item.i_category = 'Home'))
---------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalOlapScan[item]
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF11 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF11 i_manufact_id->[i_manufact_id]
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Home'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF11
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
 ----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 ws_item_sk->[i_item_sk]
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[item] apply RFs: RF10 RF11
+--------------------------------PhysicalOlapScan[item] apply RFs: RF10
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
 ----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF9 ca_address_sk->[ws_bill_addr_sk]
@@ -93,8 +97,4 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
 ------------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------filter((item.i_category = 'Home'))
-----------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out
index 414bf43582ffe1..a105c7bec931e1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query39.out
@@ -4,25 +4,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalCteProducer ( cteId=CTEId#0 )
 ----PhysicalProject
 ------filter((if((mean = 0), 0, (stdev / mean)) > 1))
---------hashAgg[GLOBAL]
-----------PhysicalDistribute
-------------hashAgg[LOCAL]
---------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=()
-------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=()
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
-------------------------PhysicalOlapScan[inventory] apply RFs: RF0
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
-------------------------------PhysicalOlapScan[date_dim]
+--------hashAgg[LOCAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=()
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=()
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
+----------------------PhysicalOlapScan[inventory] apply RFs: RF0
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------PhysicalOlapScan[item]
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------PhysicalOlapScan[warehouse]
+--------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
+----------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[item]
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------PhysicalOlapScan[warehouse]
 --PhysicalResultSink
 ----PhysicalQuickSort[MERGE_SORT]
 ------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
index 229cf1936a2086..c53df8f4e699aa 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query57.out
@@ -7,8 +7,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------PhysicalQuickSort[LOCAL_SORT]
 ----------PhysicalWindow
 ------------PhysicalQuickSort[LOCAL_SORT]
---------------PhysicalDistribute
-----------------PhysicalProject
+--------------PhysicalProject
+----------------PhysicalDistribute
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute
 ----------------------hashAgg[LOCAL]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query72.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query72.out
index cb73cf02330945..9f007e62aa1571 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query72.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query72.out
@@ -19,40 +19,39 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_week_seq = d2.d_week_seq)) otherCondition=()
-----------------------------------PhysicalProject
-------------------------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=()
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=()
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, INTERVAL 5 DAY)))
-------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)) otherCondition=() build RFs:RF2 cd_demo_sk->[cs_bill_cdemo_sk]
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[cs_bill_hdemo_sk]
---------------------------------------------------------PhysicalProject
-----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
---------------------------------------------------------PhysicalDistribute
-----------------------------------------------------------PhysicalProject
-------------------------------------------------------------filter((household_demographics.hd_buy_potential = '501-1000'))
---------------------------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=()
+------------------------------------PhysicalProject
+--------------------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=()
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, INTERVAL 5 DAY)))
+----------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)) otherCondition=() build RFs:RF2 cd_demo_sk->[cs_bill_cdemo_sk]
+------------------------------------------------PhysicalDistribute
+--------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+----------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[cs_bill_hdemo_sk]
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2
 ------------------------------------------------------PhysicalDistribute
 --------------------------------------------------------PhysicalProject
-----------------------------------------------------------filter((d1.d_year = 2002))
-------------------------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF5
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------PhysicalProject
-------------------------------------------------------filter((customer_demographics.cd_marital_status = 'W'))
---------------------------------------------------------PhysicalOlapScan[customer_demographics]
+----------------------------------------------------------filter((household_demographics.hd_buy_potential = '501-1000'))
+------------------------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------------------------PhysicalDistribute
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------filter((d1.d_year = 2002))
+----------------------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF5
 ------------------------------------------------PhysicalDistribute
 --------------------------------------------------PhysicalProject
-----------------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------PhysicalOlapScan[item]
---------------------------------------PhysicalDistribute
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[promotion]
+----------------------------------------------------filter((customer_demographics.cd_marital_status = 'W'))
+------------------------------------------------------PhysicalOlapScan[customer_demographics]
+----------------------------------------------PhysicalDistribute
+------------------------------------------------PhysicalProject
+--------------------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------PhysicalOlapScan[item]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[promotion]
 ----------------------------------PhysicalDistribute
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[date_dim]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query77.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query77.out
index a25d7bcded0ba0..abe938dcd83ef9 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query77.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query77.out
@@ -93,20 +93,19 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[web_page]
 ------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[wr_returned_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF6 RF7
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[wr_returned_date_sk]
 --------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[web_page]
+----------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF6 RF7
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[web_page]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query17.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query17.out
index a9ca31b6e2ea08..6fe2631fc288ad 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query17.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query17.out
@@ -19,8 +19,8 @@ PhysicalResultSink
 ----------------------------PhysicalProject
 ------------------------------filter(d_quarter_name IN ('2001Q1', '2001Q2', '2001Q3'))
 --------------------------------PhysicalOlapScan[date_dim]
---------------------PhysicalDistribute
-----------------------PhysicalProject
+--------------------PhysicalProject
+----------------------PhysicalDistribute
 ------------------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk]
 --------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[ss_item_sk]
 ----------------------------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out
index 893c2c0b7eaee9..485880ab7e0a77 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query24.out
@@ -8,9 +8,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------hashAgg[LOCAL]
 ------------PhysicalProject
 --------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = store_returns.sr_item_sk) and (store_sales.ss_ticket_number = store_returns.sr_ticket_number)) otherCondition=() build RFs:RF5 sr_item_sk->[ss_item_sk];RF6 sr_ticket_number->[ss_ticket_number]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[ss_item_sk]
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[ss_item_sk]
+--------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((store.s_zip = customer_address.ca_zip) and (store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 ca_zip->[s_zip];RF3 c_customer_sk->[ss_customer_sk]
 ------------------------PhysicalDistribute
 --------------------------PhysicalProject
@@ -29,9 +29,9 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalOlapScan[customer_address]
-----------------------PhysicalDistribute
-------------------------PhysicalProject
---------------------------PhysicalOlapScan[item]
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[item]
 ----------------PhysicalDistribute
 ------------------PhysicalProject
 --------------------PhysicalOlapScan[store_returns]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query25.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query25.out
index d1188acc23b4af..0f3f59ab4eca9f 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query25.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query25.out
@@ -18,8 +18,8 @@ PhysicalResultSink
 --------------------------PhysicalProject
 ----------------------------filter((d3.d_moy <= 10) and (d3.d_moy >= 4) and (d3.d_year = 2000))
 ------------------------------PhysicalOlapScan[date_dim]
-------------------PhysicalDistribute
---------------------PhysicalProject
+------------------PhysicalProject
+--------------------PhysicalDistribute
 ----------------------hashJoin[INNER_JOIN] hashCondition=((store.s_store_sk = store_sales.ss_store_sk)) otherCondition=() build RFs:RF6 s_store_sk->[ss_store_sk]
 ------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = store_sales.ss_item_sk)) otherCondition=() build RFs:RF5 i_item_sk->[ss_item_sk]
 --------------------------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out
index 1b3dccd334b980..a8cc78c96dbace 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query33.out
@@ -9,74 +9,78 @@ PhysicalResultSink
 ------------hashAgg[LOCAL]
 --------------PhysicalUnion
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF3 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF3 i_item_sk->[ss_item_sk]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF2 ca_address_sk->[ss_addr_sk]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[ss_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF1 RF2 RF3
-----------------------------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Home'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF3
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF2 i_item_sk->[ss_item_sk]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF1 ca_address_sk->[ss_addr_sk]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[ss_sold_date_sk]
 ------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
-----------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF0 i_manufact_id->[i_manufact_id]
+--------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF0 RF1 RF2
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[item] apply RFs: RF0
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------filter((item.i_category = 'Home'))
---------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalOlapScan[item]
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF7 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF7 i_item_sk->[cs_item_sk]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF6 ca_address_sk->[cs_bill_addr_sk]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6 RF7
-----------------------------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Home'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF7
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
+----------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF6 i_item_sk->[cs_item_sk]
+------------------------------PhysicalDistribute
+--------------------------------PhysicalProject
+----------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF5 ca_address_sk->[cs_bill_addr_sk]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[cs_sold_date_sk]
 ------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
-----------------------------------------------PhysicalOlapScan[date_dim]
-----------------------------------PhysicalDistribute
-------------------------------------PhysicalProject
---------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
-----------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF4 i_manufact_id->[i_manufact_id]
+--------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF4 RF5 RF6
+------------------------------------------PhysicalDistribute
+--------------------------------------------PhysicalProject
+----------------------------------------------filter((date_dim.d_moy = 1) and (date_dim.d_year = 2002))
+------------------------------------------------PhysicalOlapScan[date_dim]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
+------------------------------------------PhysicalOlapScan[customer_address]
+------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[item] apply RFs: RF4
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------filter((item.i_category = 'Home'))
---------------------------------------PhysicalOlapScan[item]
+----------------------------------PhysicalOlapScan[item]
 ----------------PhysicalProject
-------------------hashAgg[GLOBAL]
+------------------hashJoin[RIGHT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF11 i_manufact_id->[i_manufact_id]
 --------------------PhysicalDistribute
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((item.i_manufact_id = item.i_manufact_id)) otherCondition=() build RFs:RF11 i_manufact_id->[i_manufact_id]
+----------------------PhysicalProject
+------------------------filter((item.i_category = 'Home'))
+--------------------------PhysicalOlapScan[item] apply RFs: RF11
+--------------------hashAgg[GLOBAL]
+----------------------PhysicalDistribute
+------------------------hashAgg[LOCAL]
+--------------------------PhysicalProject
 ----------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF10 ws_item_sk->[i_item_sk]
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[item] apply RFs: RF10 RF11
+--------------------------------PhysicalOlapScan[item] apply RFs: RF10
 ------------------------------PhysicalDistribute
 --------------------------------PhysicalProject
 ----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_bill_addr_sk = customer_address.ca_address_sk)) otherCondition=() build RFs:RF9 ca_address_sk->[ws_bill_addr_sk]
@@ -93,8 +97,4 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------filter((customer_address.ca_gmt_offset = -5.00))
 ------------------------------------------PhysicalOlapScan[customer_address]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------filter((item.i_category = 'Home'))
-----------------------------------PhysicalOlapScan[item]
 
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out
index 7e25a27c4defb1..3743c5863bc705 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query39.out
@@ -4,25 +4,24 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --PhysicalCteProducer ( cteId=CTEId#0 )
 ----PhysicalProject
 ------filter((if((mean = 0), 0, (stdev / mean)) > 1))
---------hashAgg[GLOBAL]
-----------PhysicalDistribute
-------------hashAgg[LOCAL]
---------------PhysicalProject
-----------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=() build RFs:RF2 w_warehouse_sk->[inv_warehouse_sk]
-------------------PhysicalProject
---------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
-----------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
-------------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2
-------------------------PhysicalDistribute
---------------------------PhysicalProject
-----------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
-------------------------------PhysicalOlapScan[date_dim]
+--------hashAgg[LOCAL]
+----------PhysicalProject
+------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_warehouse_sk = warehouse.w_warehouse_sk)) otherCondition=() build RFs:RF2 w_warehouse_sk->[inv_warehouse_sk]
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_item_sk = item.i_item_sk)) otherCondition=() build RFs:RF1 i_item_sk->[inv_item_sk]
+--------------------hashJoin[INNER_JOIN] hashCondition=((inventory.inv_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF0 d_date_sk->[inv_date_sk]
+----------------------PhysicalOlapScan[inventory] apply RFs: RF0 RF1 RF2
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------PhysicalOlapScan[item]
-------------------PhysicalDistribute
---------------------PhysicalProject
-----------------------PhysicalOlapScan[warehouse]
+--------------------------filter((date_dim.d_year = 1998) and d_moy IN (1, 2))
+----------------------------PhysicalOlapScan[date_dim]
+--------------------PhysicalDistribute
+----------------------PhysicalProject
+------------------------PhysicalOlapScan[item]
+--------------PhysicalDistribute
+----------------PhysicalProject
+------------------PhysicalOlapScan[warehouse]
 --PhysicalResultSink
 ----PhysicalQuickSort[MERGE_SORT]
 ------PhysicalDistribute
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
index da1132baa93a09..fc6a26940772ba 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query57.out
@@ -7,8 +7,8 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------PhysicalQuickSort[LOCAL_SORT]
 ----------PhysicalWindow
 ------------PhysicalQuickSort[LOCAL_SORT]
---------------PhysicalDistribute
-----------------PhysicalProject
+--------------PhysicalProject
+----------------PhysicalDistribute
 ------------------hashAgg[GLOBAL]
 --------------------PhysicalDistribute
 ----------------------hashAgg[LOCAL]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
index e9370fe109e460..a034f3f4fb787d 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query72.out
@@ -19,40 +19,39 @@ PhysicalResultSink
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------hashJoin[INNER_JOIN] hashCondition=((d1.d_week_seq = d2.d_week_seq)) otherCondition=() build RFs:RF5 d_week_seq->[d_week_seq]
-----------------------------------PhysicalProject
-------------------------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=()
---------------------------------------PhysicalProject
-----------------------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk]
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, INTERVAL 5 DAY))) build RFs:RF3 d_date_sk->[cs_ship_date_sk]
-------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)) otherCondition=() build RFs:RF2 cd_demo_sk->[cs_bill_cdemo_sk]
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
-------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[cs_bill_hdemo_sk]
---------------------------------------------------------PhysicalProject
-----------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 RF3 RF4
---------------------------------------------------------PhysicalDistribute
-----------------------------------------------------------PhysicalProject
-------------------------------------------------------------filter((household_demographics.hd_buy_potential = '501-1000'))
---------------------------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------hashJoin[LEFT_OUTER_JOIN] hashCondition=((catalog_sales.cs_promo_sk = promotion.p_promo_sk)) otherCondition=()
+------------------------------------PhysicalProject
+--------------------------------------hashJoin[INNER_JOIN] hashCondition=((item.i_item_sk = catalog_sales.cs_item_sk)) otherCondition=() build RFs:RF4 i_item_sk->[cs_item_sk]
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_ship_date_sk = d3.d_date_sk)) otherCondition=((d3.d_date > days_add(d_date, INTERVAL 5 DAY))) build RFs:RF3 d_date_sk->[cs_ship_date_sk]
+----------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_cdemo_sk = customer_demographics.cd_demo_sk)) otherCondition=() build RFs:RF2 cd_demo_sk->[cs_bill_cdemo_sk]
+------------------------------------------------PhysicalDistribute
+--------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = d1.d_date_sk)) otherCondition=() build RFs:RF1 d_date_sk->[cs_sold_date_sk]
+----------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_bill_hdemo_sk = household_demographics.hd_demo_sk)) otherCondition=() build RFs:RF0 hd_demo_sk->[cs_bill_hdemo_sk]
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF0 RF1 RF2 RF3 RF4
 ------------------------------------------------------PhysicalDistribute
 --------------------------------------------------------PhysicalProject
-----------------------------------------------------------filter((d1.d_year = 2002))
-------------------------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF5
---------------------------------------------------PhysicalDistribute
-----------------------------------------------------PhysicalProject
-------------------------------------------------------filter((customer_demographics.cd_marital_status = 'W'))
---------------------------------------------------------PhysicalOlapScan[customer_demographics]
+----------------------------------------------------------filter((household_demographics.hd_buy_potential = '501-1000'))
+------------------------------------------------------------PhysicalOlapScan[household_demographics]
+----------------------------------------------------PhysicalDistribute
+------------------------------------------------------PhysicalProject
+--------------------------------------------------------filter((d1.d_year = 2002))
+----------------------------------------------------------PhysicalOlapScan[date_dim] apply RFs: RF5
 ------------------------------------------------PhysicalDistribute
 --------------------------------------------------PhysicalProject
-----------------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------------PhysicalDistribute
---------------------------------------------PhysicalProject
-----------------------------------------------PhysicalOlapScan[item]
---------------------------------------PhysicalDistribute
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[promotion]
+----------------------------------------------------filter((customer_demographics.cd_marital_status = 'W'))
+------------------------------------------------------PhysicalOlapScan[customer_demographics]
+----------------------------------------------PhysicalDistribute
+------------------------------------------------PhysicalProject
+--------------------------------------------------PhysicalOlapScan[date_dim]
+----------------------------------------PhysicalDistribute
+------------------------------------------PhysicalProject
+--------------------------------------------PhysicalOlapScan[item]
+------------------------------------PhysicalDistribute
+--------------------------------------PhysicalProject
+----------------------------------------PhysicalOlapScan[promotion]
 ----------------------------------PhysicalDistribute
 ------------------------------------PhysicalProject
 --------------------------------------PhysicalOlapScan[date_dim]
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out
index 49dbb5aeabccac..5a5a9c657af8d3 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query77.out
@@ -93,20 +93,19 @@ PhysicalResultSink
 --------------------------------------PhysicalProject
 ----------------------------------------PhysicalOlapScan[web_page]
 ------------------------PhysicalProject
---------------------------hashAgg[GLOBAL]
-----------------------------PhysicalDistribute
-------------------------------hashAgg[LOCAL]
---------------------------------PhysicalProject
-----------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
-------------------------------------PhysicalProject
---------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[wr_returned_date_sk]
-----------------------------------------PhysicalProject
-------------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF6 RF7
-----------------------------------------PhysicalDistribute
-------------------------------------------PhysicalProject
---------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
-----------------------------------------------PhysicalOlapScan[date_dim]
-------------------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
+------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_web_page_sk = web_page.wp_web_page_sk)) otherCondition=() build RFs:RF7 wp_web_page_sk->[wr_web_page_sk]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------hashJoin[INNER_JOIN] hashCondition=((web_returns.wr_returned_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[wr_returned_date_sk]
 --------------------------------------PhysicalProject
-----------------------------------------PhysicalOlapScan[web_page]
+----------------------------------------PhysicalOlapScan[web_returns] apply RFs: RF6 RF7
+--------------------------------------PhysicalDistribute
+----------------------------------------PhysicalProject
+------------------------------------------filter((date_dim.d_date <= '1998-09-04') and (date_dim.d_date >= '1998-08-05'))
+--------------------------------------------PhysicalOlapScan[date_dim]
+--------------------------------PhysicalDistribute
+----------------------------------PhysicalProject
+------------------------------------PhysicalOlapScan[web_page]
 

From 2f29dda5aa5ef236910541b351fda5d35e2463c7 Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Fri, 29 Dec 2023 09:07:59 +0800
Subject: [PATCH 142/299] [Fix](core) Fix file system scan deleted file
 (#29266)

---
 be/src/io/fs/local_file_system.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp
index d7e1372ffe9672..78ba222d1ffd8c 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -32,6 +32,7 @@
 #include <system_error>
 #include <utility>
 
+#include "common/exception.h"
 #include "gutil/macros.h"
 #include "io/fs/err_utils.h"
 #include "io/fs/file_system.h"
@@ -182,7 +183,11 @@ Status LocalFileSystem::directory_size(const Path& dir_path, size_t* dir_size) {
     if (std::filesystem::exists(dir_path) && std::filesystem::is_directory(dir_path)) {
         for (const auto& entry : std::filesystem::recursive_directory_iterator(dir_path)) {
             if (std::filesystem::is_regular_file(entry)) {
-                *dir_size += std::filesystem::file_size(entry);
+                try {
+                    *dir_size += std::filesystem::file_size(entry);
+                } catch (const std::exception& e) {
+                    LOG(INFO) << "{}", e.what();
+                }
             }
         }
         return Status::OK();

From 269c1b189d55d0c342180052932649fb7b22413a Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Fri, 29 Dec 2023 09:11:24 +0800
Subject: [PATCH 143/299] [improve](vtablet_writer)  check runtime state is
 cancel when back pressure (#29260)

---
 be/src/vec/sink/writer/vtablet_writer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index 3093dbc5f6b632..40b1c544e237cd 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -478,7 +478,7 @@ Status VNodeChannel::add_block(vectorized::Block* block, const Payload* payload,
     // But there is still some unfinished things, we do mem limit here temporarily.
     // _cancelled may be set by rpc callback, and it's possible that _cancelled might be set in any of the steps below.
     // It's fine to do a fake add_block() and return OK, because we will check _cancelled in next add_block() or mark_close().
-    while (!_cancelled && _pending_batches_num > 0 &&
+    while (!_cancelled && !_state->is_cancelled() && _pending_batches_num > 0 &&
            _pending_batches_bytes > _max_pending_batches_bytes) {
         SCOPED_RAW_TIMER(&_stat.mem_exceeded_block_ns);
         std::this_thread::sleep_for(std::chrono::milliseconds(10));

From 61677d1d4b11a4e4826d0b04fcc18efbdfec01d3 Mon Sep 17 00:00:00 2001
From: Dongyang Li <hello_stephen@qq.com>
Date: Fri, 29 Dec 2023 09:26:15 +0800
Subject: [PATCH 144/299] [ci](perf) 1. add perf check of tpcds, 2. adjust
 clickbench and tpch check (#28431)

---
 .licenserc.yaml                               |   1 +
 .../pipeline/common/doris-utils.sh            | 154 +++-
 .../pipeline/common/get-or-set-tmp-env.sh     |  38 +
 .../pipeline/common/github-utils.sh           |  55 +-
 regression-test/pipeline/performance/clean.sh |  47 ++
 .../pipeline/performance/compile.sh           | 143 ++++
 .../pipeline/performance/conf/README.md       |   5 +
 .../pipeline/performance/conf/be_custom.conf  |  21 +
 .../pipeline/performance/conf/custom_env.sh   |  20 +
 .../pipeline/performance/conf/fe_custom.conf  |  28 +
 .../performance/conf/session_variables        |   0
 .../pipeline/performance/deploy.sh            | 115 +++
 .../pipeline/performance/prepare.sh           |  96 +++
 .../pipeline/performance/run-clickbench.sh    | 323 ++++++++
 .../pipeline/performance/run-load.sh          | 688 ++++++++++++++++++
 .../pipeline/performance/run-tpcds.sh         | 156 ++++
 .../pipeline/performance/run-tpch.sh          | 159 ++++
 .../run-clickbench-queries.sh                 |  36 +-
 tools/tpcds-tools/bin/run-tpcds-queries.sh    |  20 +
 tools/tpch-tools/bin/run-tpch-queries.sh      |  20 +
 20 files changed, 2103 insertions(+), 22 deletions(-)
 create mode 100644 regression-test/pipeline/common/get-or-set-tmp-env.sh
 create mode 100644 regression-test/pipeline/performance/clean.sh
 create mode 100644 regression-test/pipeline/performance/compile.sh
 create mode 100644 regression-test/pipeline/performance/conf/README.md
 create mode 100644 regression-test/pipeline/performance/conf/be_custom.conf
 create mode 100644 regression-test/pipeline/performance/conf/custom_env.sh
 create mode 100644 regression-test/pipeline/performance/conf/fe_custom.conf
 create mode 100644 regression-test/pipeline/performance/conf/session_variables
 create mode 100644 regression-test/pipeline/performance/deploy.sh
 create mode 100644 regression-test/pipeline/performance/prepare.sh
 create mode 100644 regression-test/pipeline/performance/run-clickbench.sh
 create mode 100644 regression-test/pipeline/performance/run-load.sh
 create mode 100644 regression-test/pipeline/performance/run-tpcds.sh
 create mode 100644 regression-test/pipeline/performance/run-tpch.sh

diff --git a/.licenserc.yaml b/.licenserc.yaml
index 7e3207b299973b..ea761f044f46ef 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -88,6 +88,7 @@ header:
     - "conf/mysql_ssl_default_certificate/client_certificate/client-cert.pem"
     - "conf/mysql_ssl_default_certificate/client_certificate/client-key.pem"
     - "regression-test/ssl_default_certificate/*"
+    - "regression-test/pipeline/performance/conf/session_variables"
     - "extension/beats/go.mod"
     - "extension/beats/go.sum"
     - "pytest/hdfs"
diff --git a/regression-test/pipeline/common/doris-utils.sh b/regression-test/pipeline/common/doris-utils.sh
index dddc1fc5f5d22c..f90b9a077ae478 100644
--- a/regression-test/pipeline/common/doris-utils.sh
+++ b/regression-test/pipeline/common/doris-utils.sh
@@ -53,12 +53,15 @@ function set_doris_conf_value() {
 
 function start_doris_fe() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
-    if ! java -version >/dev/null; then sudo apt install openjdk-8-jdk -y >/dev/null; fi
+    if ! java -version >/dev/null ||
+        [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]]; then
+        sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
+    fi
     JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
     export JAVA_HOME
     "${DORIS_HOME}"/fe/bin/start_fe.sh --daemon
 
-    if ! mysql --version >/dev/null; then sudo apt install -y mysql-client; fi
+    if ! mysql --version >/dev/null; then sudo apt update && sudo apt install -y mysql-client; fi
     query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
     cl="mysql -h127.0.0.1 -P${query_port} -uroot "
     local i=1
@@ -71,12 +74,14 @@ function start_doris_fe() {
         fi
     done
     if [[ ${i} -ge 60 ]]; then echo "ERROR: Start Doris Frontend Failed after 2 mins wait..." && return 1; fi
-
 }
 
 function start_doris_be() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
-    if ! java -version >/dev/null; then sudo apt install openjdk-8-jdk -y >/dev/null; fi
+    if ! java -version >/dev/null ||
+        [[ -z "$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*')" ]]; then
+        sudo apt update && sudo apt install openjdk-8-jdk -y >/dev/null
+    fi
     JAVA_HOME="$(find /usr/lib/jvm -maxdepth 1 -type d -name 'java-8-*' | sed -n '1p')"
     export JAVA_HOME
     sysctl -w vm.max_map_count=2000000 &&
@@ -101,7 +106,7 @@ function start_doris_be() {
 
 function add_doris_be_to_fe() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
-    if ! mysql --version >/dev/null; then sudo apt install -y mysql-client; fi
+    if ! mysql --version >/dev/null; then sudo sudo apt update && apt install -y mysql-client; fi
     query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
     heartbeat_service_port=$(get_doris_conf_value "${DORIS_HOME}"/be/conf/be.conf heartbeat_service_port)
     cl="mysql -h127.0.0.1 -P${query_port} -uroot "
@@ -116,7 +121,7 @@ function add_doris_be_to_fe() {
             echo 'Wait for Backends ready, sleep 2 seconds ...' && sleep 2
         fi
     done
-    if [[ ${i} -eq 60 ]]; then echo "ERROR: Add Doris Backend Failed after 2 mins wait..." && return 1; fi
+    if [[ ${i} -ge 60 ]]; then echo "ERROR: Add Doris Backend Failed after 2 mins wait..." && return 1; fi
 }
 
 function stop_doris() {
@@ -129,6 +134,15 @@ function stop_doris() {
     fi
 }
 
+function restart_doris() {
+    if stop_doris; then echo; fi
+    if ! start_doris_fe; then return 1; fi
+    if ! start_doris_be; then return 1; fi
+    # wait 10s for doris totally started, otherwize may encounter the error below,
+    # ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium.
+    sleep 10s
+}
+
 function check_tpch_table_rows() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
     db_name="$1"
@@ -138,20 +152,133 @@ function check_tpch_table_rows() {
     query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
     cl="mysql -h127.0.0.1 -P${query_port} -uroot "
     declare -A table_rows
-    if [[ "${scale_factor}" == "100" ]]; then
+    if [[ "${scale_factor}" == "1" ]]; then
+        table_rows=(['region']=5 ['nation']=25 ['supplier']=10000 ['customer']=150000 ['part']=200000 ['partsupp']=800000 ['orders']=1500000 ['lineitem']=6001215)
+    elif [[ "${scale_factor}" == "100" ]]; then
         table_rows=(['region']=5 ['nation']=25 ['supplier']=1000000 ['customer']=15000000 ['part']=20000000 ['partsupp']=80000000 ['orders']=150000000 ['lineitem']=600037902)
     else
-        table_rows=(['region']=5 ['nation']=25 ['supplier']=10000 ['customer']=150000 ['part']=200000 ['partsupp']=800000 ['orders']=1500000 ['lineitem']=6001215)
+        echo "ERROR: unsupported scale_factor ${scale_factor} for tpch" && return 1
+    fi
+    for table in ${!table_rows[*]}; do
+        rows_actual=$(${cl} -D"${db_name}" -e"SELECT count(*) FROM ${table}" | sed -n '2p')
+        rows_expect=${table_rows[${table}]}
+        if [[ ${rows_actual} -ne ${rows_expect} ]]; then
+            echo "ERROR: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
+        fi
+    done
+}
+
+function check_tpcds_table_rows() {
+    if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
+    db_name="$1"
+    scale_factor="$2"
+    if [[ -z "${scale_factor}" ]]; then return 1; fi
+
+    query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
+    cl="mysql -h127.0.0.1 -P${query_port} -uroot "
+    declare -A table_rows
+    if [[ "${scale_factor}" == "1" ]]; then
+        table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=5 ['reason']=35 ['web_site']=30 ['call_center']=6 ['store']=12 ['promotion']=300 ['household_demographics']=7200 ['web_page']=60 ['catalog_page']=11718 ['time_dim']=86400 ['date_dim']=73049 ['item']=18000 ['customer_demographics']=1920800 ['customer_address']=50000 ['customer']=100000 ['web_returns']=71763 ['catalog_returns']=144067 ['store_returns']=287514 ['inventory']=11745000 ['web_sales']=719384 ['catalog_sales']=1441548 ['store_sales']=2880404)
+    elif [[ "${scale_factor}" == "100" ]]; then
+        table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=15 ['reason']=55 ['web_site']=24 ['call_center']=30 ['store']=402 ['promotion']=1000 ['household_demographics']=7200 ['web_page']=2040 ['catalog_page']=20400 ['time_dim']=86400 ['date_dim']=73049 ['item']=204000 ['customer_demographics']=1920800 ['customer_address']=1000000 ['customer']=2000000 ['web_returns']=7197670 ['catalog_returns']=14404374 ['store_returns']=28795080 ['inventory']=399330000 ['web_sales']=72001237 ['catalog_sales']=143997065 ['store_sales']=287997024)
+    elif [[ "${scale_factor}" == "1000" ]]; then
+        table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=20 ['reason']=65 ['web_site']=54 ['call_center']=42 ['store']=1002 ['promotion']=1500 ['household_demographics']=7200 ['web_page']=3000 ['catalog_page']=30000 ['time_dim']=86400 ['date_dim']=73049 ['item']=300000 ['customer_demographics']=1920800 ['customer_address']=6000000 ['customer']=12000000 ['web_returns']=71997522 ['catalog_returns']=143996756 ['store_returns']=287999764 ['inventory']=783000000 ['web_sales']=720000376 ['catalog_sales']=1439980416 ['store_sales']=2879987999)
+    elif [[ "${scale_factor}" == "3000" ]]; then
+        table_rows=(['income_band']=20 ['ship_mode']=20 ['warehouse']=22 ['reason']=67 ['web_site']=66 ['call_center']=48 ['store']=1350 ['promotion']=1800 ['household_demographics']=7200 ['web_page']=3600 ['catalog_page']=36000 ['time_dim']=86400 ['date_dim']=73049 ['item']=360000 ['customer_demographics']=1920800 ['customer_address']=15000000 ['customer']=30000000 ['web_returns']=216003761 ['catalog_returns']=432018033 ['store_returns']=863989652 ['inventory']=1033560000 ['web_sales']=2159968881 ['catalog_sales']=4320078880 ['store_sales']=8639936081)
+    else
+        echo "ERROR: unsupported scale_factor ${scale_factor} for tpcds" && return 1
     fi
     for table in ${!table_rows[*]}; do
         rows_actual=$(${cl} -D"${db_name}" -e"SELECT count(*) FROM ${table}" | sed -n '2p')
         rows_expect=${table_rows[${table}]}
         if [[ ${rows_actual} -ne ${rows_expect} ]]; then
-            echo "WARNING: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
+            echo "ERROR: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
+        fi
+    done
+}
+
+function check_clickbench_table_rows() {
+    if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
+    db_name="$1"
+    if [[ -z "${db_name}" ]]; then return 1; fi
+    query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
+    cl="mysql -h127.0.0.1 -P${query_port} -uroot "
+    declare -A table_rows
+    table_rows=(['hits']=99997497)
+    if ${DEBUG:-false}; then table_rows=(['hits']=10000); fi
+    for table in ${!table_rows[*]}; do
+        rows_actual=$(${cl} -D"${db_name}" -e"SELECT count(*) FROM ${table}" | sed -n '2p')
+        rows_expect=${table_rows[${table}]}
+        if [[ ${rows_actual} -ne ${rows_expect} ]]; then
+            echo "ERROR: ${table} actual rows: ${rows_actual}, expect rows: ${rows_expect}" && return 1
         fi
     done
 }
 
+function check_tpch_result() {
+    log_file="$1"
+    if [[ -z "${log_file}" ]]; then return 1; fi
+    if ! grep '^Total cold run time' "${log_file}" || ! grep '^Total hot run time' "${log_file}"; then
+        echo "ERROR: can not find 'Total hot run time' in '${log_file}'"
+        return 1
+    else
+        cold_run_time=$(grep '^Total cold run time' "${log_file}" | awk '{print $5}')
+        hot_run_time=$(grep '^Total hot run time' "${log_file}" | awk '{print $5}')
+    fi
+    # 单位是毫秒
+    cold_run_time_threshold=${cold_run_time_threshold:-50000}
+    hot_run_time_threshold=${hot_run_time_threshold:-42000}
+    if [[ ${cold_run_time} -gt ${cold_run_time_threshold} || ${hot_run_time} -gt ${hot_run_time_threshold} ]]; then
+        echo "ERROR:
+    cold_run_time ${cold_run_time} is great than the threshold ${cold_run_time_threshold},
+    or, hot_run_time ${hot_run_time} is great than the threshold ${hot_run_time_threshold}"
+        return 1
+    else
+        echo "INFO:
+    cold_run_time ${cold_run_time} is less than the threshold ${cold_run_time_threshold},
+    hot_run_time ${hot_run_time} is less than the threshold ${hot_run_time_threshold}"
+    fi
+}
+
+function check_tpcds_result() {
+    check_tpch_result "$1"
+}
+
+function check_clickbench_query_result() {
+    echo "TODO"
+}
+
+function check_clickbench_performance_result() {
+    result_file="$1"
+    if [[ -z "${result_file}" ]]; then return 1; fi
+
+    empty_query_time="$(awk -F ',' '{if( ($2=="") || ($3=="") || ($4=="") ){print $1}}' "${result_file}")"
+    if [[ -n ${empty_query_time} ]]; then
+        echo -e "ERROR: find empty query time of:\n${empty_query_time}" && return 1
+    fi
+
+    # 单位是秒
+    cold_run_time_threshold=${cold_run_time_threshold:-200}
+    hot_run_time_threshold=${hot_run_time_threshold:-55}
+    cold_run_sum=$(awk -F ',' '{sum+=$2} END {print sum}' result.csv)
+    hot_run_time=$(awk -F ',' '{if($3<$4){sum+=$3}else{sum+=$4}} END {print sum}' "${result_file}")
+    if [[ $(echo "${hot_run_time} > ${hot_run_time_threshold}" | bc) -eq 1 ]] ||
+        [[ $(echo "${cold_run_sum} > ${cold_run_time_threshold}" | bc) -eq 1 ]]; then
+        echo "ERROR:
+    cold_run_time ${cold_run_time} is great than the threshold ${cold_run_time_threshold},
+    or, hot_run_time ${hot_run_time} is great than the threshold ${hot_run_time_threshold}"
+        return 1
+    else
+        echo "INFO:
+    cold_run_time ${cold_run_time} is less than the threshold ${cold_run_time_threshold},
+    hot_run_time ${hot_run_time} is less than the threshold ${hot_run_time_threshold}"
+    fi
+}
+
+function check_load_performance() {
+    echo "TODO"
+}
+
 get_session_variable() {
     if [[ ! -d "${DORIS_HOME:-}" ]]; then return 1; fi
     usage="
@@ -228,12 +355,11 @@ archive_doris_logs() {
     if [[ -z ${archive_name} ]]; then echo "ERROR: archive file name required" && return 1; fi
     if tar -I pigz \
         --directory "${DORIS_HOME}" \
-        --absolute-names \
         -cf "${DORIS_HOME}/${archive_name}" \
-        "${DORIS_HOME}"/fe/conf \
-        "${DORIS_HOME}"/fe/log \
-        "${DORIS_HOME}"/be/conf \
-        "${DORIS_HOME}"/be/log; then
+        fe/conf \
+        fe/log \
+        be/conf \
+        be/log; then
         echo "${DORIS_HOME}/${archive_name}"
     else
         return 1
diff --git a/regression-test/pipeline/common/get-or-set-tmp-env.sh b/regression-test/pipeline/common/get-or-set-tmp-env.sh
new file mode 100644
index 00000000000000..4e2c11547bb8b7
--- /dev/null
+++ b/regression-test/pipeline/common/get-or-set-tmp-env.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+tmp_env_file_path="${PWD}/.my_tmp_env"
+
+usage() {
+    echo -e "
+Usage:
+    $0 'get'
+    $0 'set' \"export skip_pipeline='true'\"
+    note: 'get' will return env file path; 'set' will add your new item into env file"
+    exit 1
+}
+
+if [[ $1 == 'get' ]]; then
+    if [[ ! -f "${tmp_env_file_path}" ]]; then touch "${tmp_env_file_path}"; fi
+    echo "${tmp_env_file_path}"
+elif [[ $1 == 'set' ]]; then
+    if [[ -z $2 ]]; then usage; fi
+    echo "$2" >>"${tmp_env_file_path}"
+else
+    usage
+fi
diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh
index 3c226955b1552c..ecf7a055df0d25 100644
--- a/regression-test/pipeline/common/github-utils.sh
+++ b/regression-test/pipeline/common/github-utils.sh
@@ -53,7 +53,33 @@ function create_an_issue_comment_tpch() {
     local COMMENT_BODY="$2"
     local machine='aliyun_ecs.c7a.8xlarge_32C64G'
     COMMENT_BODY="
-TPC-H test result on machine: '${machine}'
+TPC-H test result on machine: '${machine}', run with scripts in https://github.com/apache/doris/tree/master/tools/tpch-tools
+\`\`\`
+${COMMENT_BODY}
+\`\`\`
+"
+    create_an_issue_comment "${ISSUE_NUMBER}" "${COMMENT_BODY}"
+}
+
+function create_an_issue_comment_tpcds() {
+    local ISSUE_NUMBER="$1"
+    local COMMENT_BODY="$2"
+    local machine='aliyun_ecs.c7a.8xlarge_32C64G'
+    COMMENT_BODY="
+TPC-DS test result on machine: '${machine}', run with scripts in https://github.com/apache/doris/tree/master/tools/tpcds-tools
+\`\`\`
+${COMMENT_BODY}
+\`\`\`
+"
+    create_an_issue_comment "${ISSUE_NUMBER}" "${COMMENT_BODY}"
+}
+
+function create_an_issue_comment_clickbench() {
+    local ISSUE_NUMBER="$1"
+    local COMMENT_BODY="$2"
+    local machine='aliyun_ecs.c7a.8xlarge_32C64G'
+    COMMENT_BODY="
+ClickBench test result on machine: '${machine}', run with scripts in https://github.com/apache/doris/tree/master/tools/clickbench-tools
 \`\`\`
 ${COMMENT_BODY}
 \`\`\`
@@ -271,3 +297,30 @@ file_changed_ckb() {
     done
     echo "return no need" && return 1
 }
+
+file_changed_perf() {
+    local all_files
+    all_files=$(cat all_files)
+    if _only_modified_regression_conf; then echo "return no need" && return 1; fi
+    if [[ -z ${all_files} ]]; then echo "return need" && return 0; fi
+    for af in ${all_files}; do
+        if [[ "${af}" == 'be'* ]] ||
+            [[ "${af}" == 'bin'* ]] ||
+            [[ "${af}" == 'conf'* ]] ||
+            [[ "${af}" == 'fe'* ]] ||
+            [[ "${af}" == 'gensrc'* ]] ||
+            [[ "${af}" == 'thirdparty'* ]] ||
+            [[ "${af}" == 'build.sh' ]] ||
+            [[ "${af}" == 'env.sh' ]] ||
+            [[ "${af}" == 'regression-test/pipeline/common/github-utils.sh' ]] ||
+            [[ "${af}" == 'regression-test/pipeline/common/doris-utils.sh' ]] ||
+            [[ "${af}" == 'regression-test/pipeline/common/oss-utils.sh' ]] ||
+            [[ "${af}" == 'regression-test/pipeline/performance/'* ]] ||
+            [[ "${af}" == 'tools/tpch-tools/bin/run-tpch-queries.sh' ]] ||
+            [[ "${af}" == 'tools/tpcds-tools/bin/run-tpcds-queries.sh' ]] ||
+            [[ "${af}" == 'regression-test/pipeline/tpch/tpch-sf100/'* ]]; then
+            echo "performance related file changed, return need" && return 0
+        fi
+    done
+    echo "return no need" && return 1
+}
diff --git a/regression-test/pipeline/performance/clean.sh b/regression-test/pipeline/performance/clean.sh
new file mode 100644
index 00000000000000..4cd831d3468140
--- /dev/null
+++ b/regression-test/pipeline/performance/clean.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+
+# Execute step even if some of the previous steps failed
+
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/clean.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance
+    bash -x clean.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/clean.sh" && exit 1
+fi
+EOF
+
+#####################################################################################
+## clean.sh content ##
+
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+
+# shellcheck source=/dev/null
+# stop_doris
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+
+DORIS_HOME="${teamcity_build_checkoutDir}/output"
+export DORIS_HOME
+stop_doris
diff --git a/regression-test/pipeline/performance/compile.sh b/regression-test/pipeline/performance/compile.sh
new file mode 100644
index 00000000000000..9fdb995487f189
--- /dev/null
+++ b/regression-test/pipeline/performance/compile.sh
@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+export DEBUG=true
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/compile.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
+    bash -x compile.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/compile.sh" && exit 1
+fi
+EOF
+
+#####################################################################################
+## compile.sh content ##
+
+if ${DEBUG:-false}; then
+    pull_request_num="28431"
+    commit_id="b052225cd0a180b4576319b5bd6331218dd0d3fe"
+    target_branch="master"
+fi
+if [[ -z "${teamcity_build_checkoutDir}" ]]; then echo "ERROR: env teamcity_build_checkoutDir not set" && exit 2; fi
+if [[ -z "${pull_request_num}" ]]; then echo "ERROR: env pull_request_num not set" && exit 2; fi
+if [[ -z "${commit_id}" ]]; then echo "ERROR: env commit_id not set" && exit 2; fi
+if [[ -z "${target_branch}" ]]; then echo "ERROR: env target_branch not set" && exit 2; fi
+
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+
+merge_pr_to_target_branch_latest() {
+    local pull_request_num="$1"
+    local target_branch="$2"
+    echo "INFO: merge pull request into ${target_branch}"
+    if [[ -z "${teamcity_build_checkoutDir}" ]]; then
+        echo "ERROR: env teamcity_build_checkoutDir not set" && return 1
+    fi
+    cd "${teamcity_build_checkoutDir}" || return 1
+    git reset --hard
+    git fetch origin "${target_branch}"
+    git checkout "${target_branch}"
+    git reset --hard origin/"${target_branch}"
+    git pull origin "${target_branch}"
+    git submodule update --init be/src/clucene
+    git submodule update --init be/src/apache-orc
+    local target_branch_commit_id
+    target_branch_commit_id=$(git rev-parse HEAD)
+    git config user.email "ci@selectdb.com"
+    git config user.name "ci"
+    echo "git fetch origin refs/pull/${pull_request_num}/head"
+    git fetch origin "refs/pull/${pull_request_num}/head"
+    git merge --no-edit --allow-unrelated-histories FETCH_HEAD
+    echo "INFO: merge refs/pull/${pull_request_num}/head into master: ${target_branch_commit_id}"
+    CONFLICTS=$(git ls-files -u | wc -l)
+    if [[ "${CONFLICTS}" -gt 0 ]]; then
+        echo "ERROR: merge refs/pull/${pull_request_num}/head into master failed. Aborting"
+        git merge --abort
+        return 1
+    fi
+}
+
+if [[ "${target_branch}" == "master" ]]; then
+    REMOTE_CCACHE='/mnt/remote_ccache_master'
+    docker_image="apache/doris:build-env-ldb-toolchain-0.19-latest"
+elif [[ "${target_branch}" == "branch-2.0" ]]; then
+    docker_image="apache/doris:build-env-for-2.0"
+    REMOTE_CCACHE='/mnt/remote_ccache_branch_2'
+elif [[ "${target_branch}" == "branch-1.2-lts" ]]; then
+    REMOTE_CCACHE='/mnt/remote_ccache_master'
+    docker_image="apache/doris:build-env-for-1.2"
+else
+    REMOTE_CCACHE='/mnt/remote_ccache_master'
+    docker_image="apache/doris:build-env-ldb-toolchain-latest"
+fi
+if ${merge_target_branch_latest:-true}; then
+    if ! merge_pr_to_target_branch_latest "${pull_request_num}" "${target_branch}"; then
+        exit 1
+    fi
+else
+    echo "INFO: skip merge_pr_to_target_branch_latest"
+fi
+mount_swapfile=""
+if [[ -f /root/swapfile ]]; then mount_swapfile="-v /root/swapfile:/swapfile --memory-swap -1"; fi
+git_storage_path=$(grep storage "${teamcity_build_checkoutDir}"/.git/config | rev | cut -d ' ' -f 1 | rev | awk -F '/lfs' '{print $1}')
+sudo docker container prune -f
+sudo docker image prune -f
+sudo docker pull "${docker_image}"
+docker_name=doris-compile-"${commit_id}"
+if sudo docker ps -a --no-trunc | grep "${docker_name}"; then
+    sudo docker stop "${docker_name}"
+    sudo docker rm "${docker_name}"
+fi
+rm -f custom_env.sh
+cp "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/custom_env.sh .
+rm -rf "${teamcity_build_checkoutDir}"/output
+set -x
+# shellcheck disable=SC2086
+sudo docker run -i --rm \
+    --name "${docker_name}" \
+    -e TZ=Asia/Shanghai \
+    ${mount_swapfile} \
+    -v /etc/localtime:/etc/localtime:ro \
+    -v "${HOME}"/.m2:/root/.m2 \
+    -v "${HOME}"/.npm:/root/.npm \
+    -v /mnt/ccache/.ccache:/root/.ccache \
+    -v "${REMOTE_CCACHE}":/root/ccache \
+    -v "${git_storage_path}":/root/git \
+    -v "${teamcity_build_checkoutDir}":/root/doris \
+    "${docker_image}" \
+    /bin/bash -c "mkdir -p ${git_storage_path} \
+    				&& cp -r /root/git/* ${git_storage_path}/ \
+                    && cd /root/doris \
+                    && export CCACHE_LOGFILE=/tmp/cache.debug \
+                    && export CCACHE_REMOTE_STORAGE=file:///root/ccache \
+                    && export EXTRA_CXX_FLAGS=-O3 \
+                    && export USE_JEMALLOC='ON' \
+					&& export ENABLE_PCH=OFF \
+                    && export CUSTOM_NPM_REGISTRY=https://registry.npmjs.org \
+                    && bash build.sh --fe --be --clean 2>&1 | tee build.log"
+set +x
+succ_symble="BUILD SUCCESS"
+if [[ -d outout ]] && grep "${succ_symble}" "${teamcity_build_checkoutDir}"/build.log; then
+    echo -e "ERROR: BUILD FAILED"
+    exit 1
+fi
diff --git a/regression-test/pipeline/performance/conf/README.md b/regression-test/pipeline/performance/conf/README.md
new file mode 100644
index 00000000000000..d6b8073fc3c3db
--- /dev/null
+++ b/regression-test/pipeline/performance/conf/README.md
@@ -0,0 +1,5 @@
+## 简介
+这里的 conf 用于社区 tpcds 流水线
+基于 master 分支的默认 conf 文件，  
+参考：https://github.com/apache/doris/tree/master/conf  
+添加 fe_custom.conf，be_custom.conf，session_variables 来设置跑流水线的自定义设置。
\ No newline at end of file
diff --git a/regression-test/pipeline/performance/conf/be_custom.conf b/regression-test/pipeline/performance/conf/be_custom.conf
new file mode 100644
index 00000000000000..f6ac4564857b2a
--- /dev/null
+++ b/regression-test/pipeline/performance/conf/be_custom.conf
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+priority_networks=127.0.0.1/24
+storage_root_path=/data/doris-storage
+
+streaming_load_max_mb=102400
diff --git a/regression-test/pipeline/performance/conf/custom_env.sh b/regression-test/pipeline/performance/conf/custom_env.sh
new file mode 100644
index 00000000000000..27a4544655eff3
--- /dev/null
+++ b/regression-test/pipeline/performance/conf/custom_env.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+export DORIS_TOOLCHAIN=gcc
+export BUILD_TYPE=release
diff --git a/regression-test/pipeline/performance/conf/fe_custom.conf b/regression-test/pipeline/performance/conf/fe_custom.conf
new file mode 100644
index 00000000000000..bd6798505321ee
--- /dev/null
+++ b/regression-test/pipeline/performance/conf/fe_custom.conf
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+#####################################################################
+## The uppercase properties are read and exported by bin/start_fe.sh.
+## To see all Frontend configurations,
+## see fe/src/org/apache/doris/common/Config.java
+#####################################################################
+
+priority_networks=127.0.0.1/24
+meta_dir=/data/doris-meta
+
+stream_load_default_timeout_second=3600
+ignore_unknown_metadata_module=true
diff --git a/regression-test/pipeline/performance/conf/session_variables b/regression-test/pipeline/performance/conf/session_variables
new file mode 100644
index 00000000000000..e69de29bb2d1d6
diff --git a/regression-test/pipeline/performance/deploy.sh b/regression-test/pipeline/performance/deploy.sh
new file mode 100644
index 00000000000000..5f2e673f0404a4
--- /dev/null
+++ b/regression-test/pipeline/performance/deploy.sh
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+export DEBUG=true
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/deploy.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance
+    bash -x deploy.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/deploy.sh" && exit 1
+fi
+EOF
+
+## deploy.sh content ##
+
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+
+# shellcheck source=/dev/null
+# upload_doris_log_to_oss
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh
+# shellcheck source=/dev/null
+# start_doris_fe, get_doris_conf_value, start_doris_be, stop_doris,
+# print_doris_fe_log, print_doris_be_log, archive_doris_logs
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+
+if ${DEBUG:-false}; then
+    pull_request_num="28431"
+    commit_id="b052225cd0a180b4576319b5bd6331218dd0d3fe"
+fi
+echo "#### Check env"
+if [[ -z "${teamcity_build_checkoutDir}" ||
+    -z "${pull_request_num}" ||
+    -z "${commit_id}" ]]; then
+    echo "ERROR: env teamcity_build_checkoutDir or pull_request_num or commit_id not set"
+    exit 1
+fi
+
+echo "#### Deploy Doris ####"
+DORIS_HOME="${teamcity_build_checkoutDir}/output"
+export DORIS_HOME
+exit_flag=0
+need_backup_doris_logs=false
+
+echo "#### 1. try to kill old doris process"
+stop_doris
+
+echo "#### 2. copy conf from regression-test/pipeline/performance/conf/"
+rm -f "${DORIS_HOME}"/fe/conf/fe_custom.conf "${DORIS_HOME}"/be/conf/be_custom.conf
+if [[ -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/fe_custom.conf &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/be_custom.conf ]]; then
+    cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/fe_custom.conf "${DORIS_HOME}"/fe/conf/
+    cp -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/be_custom.conf "${DORIS_HOME}"/be/conf/
+else
+    echo "ERROR: doris conf file missing in ${teamcity_build_checkoutDir}/regression-test/pipeline/performance/conf/"
+    exit 1
+fi
+
+echo "#### 3. start Doris"
+meta_dir=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe_custom.conf meta_dir)
+storage_root_path=$(get_doris_conf_value "${DORIS_HOME}"/be/conf/be_custom.conf storage_root_path)
+mkdir -p "${meta_dir}"
+mkdir -p "${storage_root_path}"
+if ! start_doris_fe; then
+    echo "ERROR: Start doris fe failed."
+    print_doris_fe_log
+    need_backup_doris_logs=true
+    exit_flag=1
+fi
+if ! start_doris_be; then
+    echo "ERROR: Start doris be failed."
+    print_doris_be_log
+    need_backup_doris_logs=true
+    exit_flag=1
+fi
+if ! add_doris_be_to_fe; then
+    need_backup_doris_logs=true
+    exit_flag=1
+fi
+# wait 10s for doris totally started, otherwize may encounter the error below,
+# ERROR 1105 (HY000) at line 102: errCode = 2, detailMessage = Failed to find enough backend, please check the replication num,replication tag and storage medium.
+sleep 10s
+
+echo "#### 4. set session variables"
+echo "TODO"
+
+echo "#### 5. check if need backup doris logs"
+if ${need_backup_doris_logs}; then
+    print_doris_fe_log
+    print_doris_be_log
+    if file_name=$(archive_doris_logs "${pull_request_num}_${commit_id}_doris_logs.tar.gz"); then
+        upload_doris_log_to_oss "${file_name}"
+    fi
+fi
+
+exit "${exit_flag}"
diff --git a/regression-test/pipeline/performance/prepare.sh b/regression-test/pipeline/performance/prepare.sh
new file mode 100644
index 00000000000000..c96a0c8629474e
--- /dev/null
+++ b/regression-test/pipeline/performance/prepare.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+
+set -x
+pwd
+rm -rf ../.old/*
+set +x
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/prepare.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
+    bash prepare.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/prepare.sh" && exit 1
+fi
+EOF
+
+#####################################################################################
+## run.sh content ##
+
+if ${DEBUG:-false}; then
+    pull_request_num="28431"
+    commit_id_from_trigger="5f5c4c80564c76ff4267fc4ce6a5408498ed1ab5"
+    commit_id="5f5c4c80564c76ff4267fc4ce6a5408498ed1ab5"
+fi
+echo "#### Check env"
+if [[ -z "${teamcity_build_checkoutDir}" ||
+    -z "${commit_id_from_trigger}" ||
+    -z ${commit_id:-} ||
+    -z ${pull_request_num:-} ]]; then
+    echo "ERROR: env teamcity_build_checkoutDir or commit_id_from_trigger
+    or commit_id or pull_request_num not set" && exit 1
+fi
+commit_id_from_checkout=${commit_id}
+
+echo "#### 1. check if need run"
+if [[ "${commit_id_from_trigger}" != "${commit_id_from_checkout}" ]]; then
+    echo -e "从触发流水线 -> 流水线开始跑，这个时间段中如果有新commit，
+这时候流水线 checkout 出来的 commit 就不是触发时的传过来的 commit了，
+这种情况不需要跑，预期pr owner会重新触发。"
+    echo -e "ERROR: PR(${pull_request_num}),
+    the lastest commit id
+    ${commit_id_from_checkout}
+    not equail to the commit_id_from_trigger
+    ${commit_id_from_trigger}
+    commit_id_from_trigger is outdate"
+    exit 1
+fi
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+# shellcheck source=/dev/null
+# _get_pr_changed_files file_changed_perf
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh
+if _get_pr_changed_files "${pull_request_num}"; then
+    if ! file_changed_perf; then
+        bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'set' "export skip_pipeline=true"
+        exit 0
+    fi
+fi
+
+echo "#### 2. check if tpch depending files exist"
+if ! [[ -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/be_custom.conf &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/custom_env.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/conf/fe_custom.conf &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/prepare.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/compile.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/deploy.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/run-tpch.sh &&
+    -f "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/run-tpcds.sh &&
+    -f "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh &&
+    -f "${teamcity_build_checkoutDir}"/tools/tpcds-tools/bin/run-tpcds-queries.sh ]]; then
+    echo "ERROR: depending files missing" && exit 1
+fi
diff --git a/regression-test/pipeline/performance/run-clickbench.sh b/regression-test/pipeline/performance/run-clickbench.sh
new file mode 100644
index 00000000000000..225fa802ec6908
--- /dev/null
+++ b/regression-test/pipeline/performance/run-clickbench.sh
@@ -0,0 +1,323 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+export DEBUG=true
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/run-clickbench.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
+    bash -x run-clickbench.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/run-clickbench.sh" && exit 1
+fi
+EOF
+
+#####################################################################################
+## run-clickbench.sh content ##
+
+# shellcheck source=/dev/null
+# check_clickbench_table_rows, stop_doris, set_session_variable, check_clickbench_result
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+# shellcheck source=/dev/null
+# create_an_issue_comment
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh
+# shellcheck source=/dev/null
+# upload_doris_log_to_oss
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh
+
+if ${DEBUG:-false}; then
+    pull_request_num="28431"
+    commit_id="5f5c4c80564c76ff4267fc4ce6a5408498ed1ab5"
+fi
+echo "#### Check env"
+if [[ -z "${teamcity_build_checkoutDir}" ||
+    -z "${pull_request_num}" ||
+    -z "${commit_id}" ]]; then
+    echo "ERROR: env teamcity_build_checkoutDir or pull_request_num or commit_id not set"
+    exit 1
+fi
+
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+
+echo "#### Run clickbench test on Doris ####"
+DORIS_HOME="${teamcity_build_checkoutDir}/output"
+export DORIS_HOME
+cold_run_time_threshold=${cold_run_time_threshold:-666} # 单位 秒
+hot_run_time_threshold=${hot_run_time_threshold:-555}   # 单位 秒
+exit_flag=0
+
+(
+    set -e
+    shopt -s inherit_errexit
+
+    host="127.0.0.1"
+    query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
+    backup_session_variables_file="${teamcity_build_checkoutDir}/regression-test/pipeline/performance/backup_session_variables_file.sql"
+    opt_session_variables_file="${teamcity_build_checkoutDir}/regression-test/pipeline/performance/opt_session_variables_file.sql"
+
+    echo "####optimize doris config"
+    echo "
+priority_networks=127.0.0.1/24
+meta_dir=/data/doris-meta
+stream_load_default_timeout_second=3600
+ignore_unknown_metadata_module=true
+enable_full_auto_analyze=false
+" | tee "${DORIS_HOME}"/fe/conf/fe_custom.conf
+
+    echo "
+priority_networks=127.0.0.1/24
+storage_root_path=/data/doris-storage
+load_channel_memory_refresh_sleep_time_ms=1000
+soft_mem_limit_frac=1
+track_new_delete=false
+streaming_load_max_mb=102400
+doris_scanner_thread_pool_thread_num=8
+tc_enable_aggressive_memory_decommit=false
+enable_new_scan_node=false
+#mem_limit=100%
+mem_limit=90%
+#write_buffer_size=1609715200
+write_buffer_size=1209715200
+load_process_max_memory_limit_percent=100
+#load_process_soft_mem_limit_percent=80
+disable_auto_compaction=true
+disable_storage_page_cache=false
+disable_chunk_allocator=false
+enable_simdjson_reader = true
+" | tee "${DORIS_HOME}"/be/conf/be_custom.conf
+
+    opt_session_variables="
+set global exec_mem_limit=34359738368;
+set global parallel_fragment_exec_instance_num=16;
+set global parallel_pipeline_task_num=16;
+set global enable_single_distinct_column_opt=true;
+set global enable_function_pushdown=true;
+set global forbid_unknown_col_stats=false;
+set global runtime_filter_mode=global;
+"
+    echo -e "${opt_session_variables}" | tee "${opt_session_variables_file}"
+
+    backup_session_variables() {
+        _IFS="${IFS}"
+        IFS=$'\n'
+        for line in ${opt_session_variables}; do
+            k="${line/set global /}"
+            k="${k%=*}"
+            v=$(mysql -h"${host}" -P"${query_port}" -uroot -e"show variables like '${k}'\G" | grep " Value: ")
+            v="${v/*Value: /}"
+            echo "set global ${k}=${v};" >>"${backup_session_variables_file}"
+        done
+        IFS="${_IFS}"
+    }
+    backup_session_variables
+    mysql -h"${host}" -P"${query_port}" -uroot -e"source ${opt_session_variables_file};"
+
+    echo "#### 1. Restart doris"
+    if ! restart_doris; then echo "ERROR: Restart doris failed" && exit 1; fi
+
+    echo "#### 2. check if need to load data"
+    data_home="/data/clickbench" # no / at the end
+    db_name="clickbench"
+    if ! check_clickbench_table_rows "${db_name}"; then
+        echo "INFO: need to load clickbench data"
+        if ${force_load_data:-false}; then echo "INFO: force_load_data is true"; else echo "ERROR: force_load_data is false" && exit 1; fi
+        # prepare data
+        mkdir -p "${data_home}"
+
+        # create table and load data
+        mysql -h"${host}" -P"${query_port}" -uroot -e "DROP DATABASE IF EXISTS ${db_name}"
+        mysql -h"${host}" -P"${query_port}" -uroot -e "CREATE DATABASE IF NOT EXISTS ${db_name}" && sleep 10
+        mysql -h"${host}" -P"${query_port}" -uroot "${db_name}" -e"
+            CREATE TABLE IF NOT EXISTS  hits (
+                CounterID INT NOT NULL, 
+                EventDate DateV2 NOT NULL, 
+                UserID BIGINT NOT NULL, 
+                EventTime DateTimeV2 NOT NULL, 
+                WatchID BIGINT NOT NULL, 
+                JavaEnable SMALLINT NOT NULL,
+                Title STRING NOT NULL,
+                GoodEvent SMALLINT NOT NULL,
+                ClientIP INT NOT NULL,
+                RegionID INT NOT NULL,
+                CounterClass SMALLINT NOT NULL,
+                OS SMALLINT NOT NULL,
+                UserAgent SMALLINT NOT NULL,
+                URL STRING NOT NULL,
+                Referer STRING NOT NULL,
+                IsRefresh SMALLINT NOT NULL,
+                RefererCategoryID SMALLINT NOT NULL,
+                RefererRegionID INT NOT NULL,
+                URLCategoryID SMALLINT NOT NULL,
+                URLRegionID INT NOT NULL,
+                ResolutionWidth SMALLINT NOT NULL,
+                ResolutionHeight SMALLINT NOT NULL,
+                ResolutionDepth SMALLINT NOT NULL,
+                FlashMajor SMALLINT NOT NULL,
+                FlashMinor SMALLINT NOT NULL,
+                FlashMinor2 STRING NOT NULL,
+                NetMajor SMALLINT NOT NULL,
+                NetMinor SMALLINT NOT NULL,
+                UserAgentMajor SMALLINT NOT NULL,
+                UserAgentMinor VARCHAR(255) NOT NULL,
+                CookieEnable SMALLINT NOT NULL,
+                JavascriptEnable SMALLINT NOT NULL,
+                IsMobile SMALLINT NOT NULL,
+                MobilePhone SMALLINT NOT NULL,
+                MobilePhoneModel STRING NOT NULL,
+                Params STRING NOT NULL,
+                IPNetworkID INT NOT NULL,
+                TraficSourceID SMALLINT NOT NULL,
+                SearchEngineID SMALLINT NOT NULL,
+                SearchPhrase STRING NOT NULL,
+                AdvEngineID SMALLINT NOT NULL,
+                IsArtifical SMALLINT NOT NULL,
+                WindowClientWidth SMALLINT NOT NULL,
+                WindowClientHeight SMALLINT NOT NULL,
+                ClientTimeZone SMALLINT NOT NULL,
+                ClientEventTime DateTimeV2 NOT NULL,
+                SilverlightVersion1 SMALLINT NOT NULL,
+                SilverlightVersion2 SMALLINT NOT NULL,
+                SilverlightVersion3 INT NOT NULL,
+                SilverlightVersion4 SMALLINT NOT NULL,
+                PageCharset STRING NOT NULL,
+                CodeVersion INT NOT NULL,
+                IsLink SMALLINT NOT NULL,
+                IsDownload SMALLINT NOT NULL,
+                IsNotBounce SMALLINT NOT NULL,
+                FUniqID BIGINT NOT NULL,
+                OriginalURL STRING NOT NULL,
+                HID INT NOT NULL,
+                IsOldCounter SMALLINT NOT NULL,
+                IsEvent SMALLINT NOT NULL,
+                IsParameter SMALLINT NOT NULL,
+                DontCountHits SMALLINT NOT NULL,
+                WithHash SMALLINT NOT NULL,
+                HitColor CHAR NOT NULL,
+                LocalEventTime DateTimeV2 NOT NULL,
+                Age SMALLINT NOT NULL,
+                Sex SMALLINT NOT NULL,
+                Income SMALLINT NOT NULL,
+                Interests SMALLINT NOT NULL,
+                Robotness SMALLINT NOT NULL,
+                RemoteIP INT NOT NULL,
+                WindowName INT NOT NULL,
+                OpenerName INT NOT NULL,
+                HistoryLength SMALLINT NOT NULL,
+                BrowserLanguage STRING NOT NULL,
+                BrowserCountry STRING NOT NULL,
+                SocialNetwork STRING NOT NULL,
+                SocialAction STRING NOT NULL,
+                HTTPError SMALLINT NOT NULL,
+                SendTiming INT NOT NULL,
+                DNSTiming INT NOT NULL,
+                ConnectTiming INT NOT NULL,
+                ResponseStartTiming INT NOT NULL,
+                ResponseEndTiming INT NOT NULL,
+                FetchTiming INT NOT NULL,
+                SocialSourceNetworkID SMALLINT NOT NULL,
+                SocialSourcePage STRING NOT NULL,
+                ParamPrice BIGINT NOT NULL,
+                ParamOrderID STRING NOT NULL,
+                ParamCurrency STRING NOT NULL,
+                ParamCurrencyID SMALLINT NOT NULL,
+                OpenstatServiceName STRING NOT NULL,
+                OpenstatCampaignID STRING NOT NULL,
+                OpenstatAdID STRING NOT NULL,
+                OpenstatSourceID STRING NOT NULL,
+                UTMSource STRING NOT NULL,
+                UTMMedium STRING NOT NULL,
+                UTMCampaign STRING NOT NULL,
+                UTMContent STRING NOT NULL,
+                UTMTerm STRING NOT NULL,
+                FromTag STRING NOT NULL,
+                HasGCLID SMALLINT NOT NULL,
+                RefererHash BIGINT NOT NULL,
+                URLHash BIGINT NOT NULL,
+                CLID INT NOT NULL
+            )  
+            DUPLICATE KEY (CounterID, EventDate, UserID, EventTime, WatchID) 
+            DISTRIBUTED BY HASH(UserID) BUCKETS 16
+            PROPERTIES ( \"replication_num\"=\"1\");
+        "
+        echo "####load data"
+        if [[ ! -f "${data_home}"/hits.tsv ]] || [[ $(wc -c "${data_home}"/hits.tsv | awk '{print $1}') != '74807831229' ]]; then
+            cd "${data_home}"
+            wget --continue 'https://datasets.clickhouse.com/hits_compatible/hits.tsv.gz'
+            gzip -d hits.tsv.gz
+            if ${DEBUG:-false}; then head -n 10000 hits.tsv >hits.tsv.10000; fi
+            cd -
+        fi
+        data_file_name="${data_home}/hits.tsv"
+        if ${DEBUG:-false}; then data_file_name="${data_home}/hits.tsv.10000"; fi
+        echo "start loading ..."
+        START=$(date +%s)
+        curl --location-trusted \
+            -u root: \
+            -T "${data_file_name}" \
+            -H "label:hits_${START}" \
+            -H "columns: WatchID,JavaEnable,Title,GoodEvent,EventTime,EventDate,CounterID,ClientIP,RegionID,UserID,CounterClass,OS,UserAgent,URL,Referer,IsRefresh,RefererCategoryID,RefererRegionID,URLCategoryID,URLRegionID,ResolutionWidth,ResolutionHeight,ResolutionDepth,FlashMajor,FlashMinor,FlashMinor2,NetMajor,NetMinor,UserAgentMajor,UserAgentMinor,CookieEnable,JavascriptEnable,IsMobile,MobilePhone,MobilePhoneModel,Params,IPNetworkID,TraficSourceID,SearchEngineID,SearchPhrase,AdvEngineID,IsArtifical,WindowClientWidth,WindowClientHeight,ClientTimeZone,ClientEventTime,SilverlightVersion1,SilverlightVersion2,SilverlightVersion3,SilverlightVersion4,PageCharset,CodeVersion,IsLink,IsDownload,IsNotBounce,FUniqID,OriginalURL,HID,IsOldCounter,IsEvent,IsParameter,DontCountHits,WithHash,HitColor,LocalEventTime,Age,Sex,Income,Interests,Robotness,RemoteIP,WindowName,OpenerName,HistoryLength,BrowserLanguage,BrowserCountry,SocialNetwork,SocialAction,HTTPError,SendTiming,DNSTiming,ConnectTiming,ResponseStartTiming,ResponseEndTiming,FetchTiming,SocialSourceNetworkID,SocialSourcePage,ParamPrice,ParamOrderID,ParamCurrency,ParamCurrencyID,OpenstatServiceName,OpenstatCampaignID,OpenstatAdID,OpenstatSourceID,UTMSource,UTMMedium,UTMCampaign,UTMContent,UTMTerm,FromTag,HasGCLID,RefererHash,URLHash,CLID" \
+            "http://localhost:8030/api/${db_name}/hits/_stream_load"
+        END=$(date +%s)
+        LOADTIME=$(echo "${END} - ${START}" | bc)
+        echo "INFO: ClickBench Load data costs ${LOADTIME} seconds"
+        echo "${LOADTIME}" >clickbench_loadtime
+
+        if ! check_clickbench_table_rows "${db_name}"; then
+            exit 1
+        fi
+        data_reload="true"
+    fi
+
+    echo "#### 3. run clickbench query"
+    bash "${teamcity_build_checkoutDir}"/tools/clickbench-tools/run-clickbench-queries.sh
+    # result.csv 来自 run-clickbench-queries.sh 的产出
+    if ! check_clickbench_performance_result result.csv; then exit 1; fi
+    if ! check_clickbench_query_result; then exit 1; fi
+    cold_run_sum=$(awk -F ',' '{sum+=$2} END {print sum}' result.csv)
+    best_hot_run_sum=$(awk -F ',' '{if($3<$4){sum+=$3}else{sum+=$4}} END {print sum}' result.csv)
+    comment_body="ClickBench test result on commit ${commit_id:-}, data reload: ${data_reload:-"false"}
+
+$(sed 's|,|\t|g' result.csv)
+Total cold run time: ${cold_run_sum} s
+Total hot run time: ${best_hot_run_sum} s"
+
+    echo "#### 4. comment result on clickbench"
+    comment_body=$(echo "${comment_body}" | sed -e ':a;N;$!ba;s/\t/\\t/g;s/\n/\\n/g') # 将所有的 Tab字符替换为\t 换行符替换为\n
+    create_an_issue_comment_clickbench "${pull_request_num:-}" "${comment_body}"
+    rm -f result.csv
+    echo "INFO: Restore session variables"
+    mysql -h"${host}" -P"${query_port}" -uroot -e "source ${backup_session_variables_file};"
+    rm -f "${backup_session_variables_file}"
+)
+exit_flag="$?"
+
+echo "#### 5. check if need backup doris logs"
+if [[ ${exit_flag} != "0" ]]; then
+    stop_doris
+    print_doris_fe_log
+    print_doris_be_log
+    if file_name=$(archive_doris_logs "${pull_request_num}_${commit_id}_doris_logs.tar.gz"); then
+        upload_doris_log_to_oss "${file_name}"
+    fi
+fi
+
+exit "${exit_flag}"
diff --git a/regression-test/pipeline/performance/run-load.sh b/regression-test/pipeline/performance/run-load.sh
new file mode 100644
index 00000000000000..784ac38aafa21a
--- /dev/null
+++ b/regression-test/pipeline/performance/run-load.sh
@@ -0,0 +1,688 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+export DEBUG=true
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/run-load.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
+    bash -x run-load.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/run-load.sh" && exit 1
+fi
+EOF
+
+#####################################################################################
+## run-load.sh content ##
+
+# shellcheck source=/dev/null
+# restart_doris, set_session_variable
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+# shellcheck source=/dev/null
+# create_an_issue_comment
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh
+# shellcheck source=/dev/null
+# upload_doris_log_to_oss
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh
+
+if ${DEBUG:-false}; then
+    pull_request_num="28431"
+    commit_id="5f5c4c80564c76ff4267fc4ce6a5408498ed1ab5"
+fi
+echo "#### Check env"
+if [[ -z "${teamcity_build_checkoutDir}" ||
+    -z "${pull_request_num}" ||
+    -z "${commit_id}" ]]; then
+    echo "ERROR: env teamcity_build_checkoutDir or pull_request_num or commit_id not set"
+    exit 1
+fi
+
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+
+echo "#### Run tpch test on Doris ####"
+DORIS_HOME="${teamcity_build_checkoutDir}/output"
+export DORIS_HOME
+data_home="/data/clickbench/"
+query_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf query_port)
+http_port=$(get_doris_conf_value "${DORIS_HOME}"/fe/conf/fe.conf http_port)
+clt="mysql -h127.0.0.1 -P${query_port} -uroot "
+DB="load_test_db"
+stream_load_json_speed_threshold=${stream_load_json_speed_threshold:-100}      # 单位 MB/s
+stream_load_orc_speed_threshold=${stream_load_orc_speed_threshold:-10}         # 单位 MB/s
+stream_load_parquet_speed_threshold=${stream_load_parquet_speed_threshold:-10} # 单位 MB/s
+insert_into_select_speed_threshold=${insert_into_select_speed_threshold:-310}  # 单位 Krows/s
+exit_flag=0
+
+(
+    set -e
+    shopt -s inherit_errexit
+
+    stream_load_json() {
+        echo "#### create table"
+        ddl="
+            CREATE TABLE IF NOT EXISTS  hits_json (
+                CounterID INT NOT NULL, 
+                EventDate DateV2 NOT NULL, 
+                UserID BIGINT NOT NULL, 
+                EventTime DateTimeV2 NOT NULL, 
+                WatchID BIGINT NOT NULL, 
+                JavaEnable SMALLINT NOT NULL,
+                Title STRING NOT NULL,
+                GoodEvent SMALLINT NOT NULL,
+                ClientIP INT NOT NULL,
+                RegionID INT NOT NULL,
+                CounterClass SMALLINT NOT NULL,
+                OS SMALLINT NOT NULL,
+                UserAgent SMALLINT NOT NULL,
+                URL STRING NOT NULL,
+                Referer STRING NOT NULL,
+                IsRefresh SMALLINT NOT NULL,
+                RefererCategoryID SMALLINT NOT NULL,
+                RefererRegionID INT NOT NULL,
+                URLCategoryID SMALLINT NOT NULL,
+                URLRegionID INT NOT NULL,
+                ResolutionWidth SMALLINT NOT NULL,
+                ResolutionHeight SMALLINT NOT NULL,
+                ResolutionDepth SMALLINT NOT NULL,
+                FlashMajor SMALLINT NOT NULL,
+                FlashMinor SMALLINT NOT NULL,
+                FlashMinor2 STRING NOT NULL,
+                NetMajor SMALLINT NOT NULL,
+                NetMinor SMALLINT NOT NULL,
+                UserAgentMajor SMALLINT NOT NULL,
+                UserAgentMinor VARCHAR(255) NOT NULL,
+                CookieEnable SMALLINT NOT NULL,
+                JavascriptEnable SMALLINT NOT NULL,
+                IsMobile SMALLINT NOT NULL,
+                MobilePhone SMALLINT NOT NULL,
+                MobilePhoneModel STRING NOT NULL,
+                Params STRING NOT NULL,
+                IPNetworkID INT NOT NULL,
+                TraficSourceID SMALLINT NOT NULL,
+                SearchEngineID SMALLINT NOT NULL,
+                SearchPhrase STRING NOT NULL,
+                AdvEngineID SMALLINT NOT NULL,
+                IsArtifical SMALLINT NOT NULL,
+                WindowClientWidth SMALLINT NOT NULL,
+                WindowClientHeight SMALLINT NOT NULL,
+                ClientTimeZone SMALLINT NOT NULL,
+                ClientEventTime DateTimeV2 NOT NULL,
+                SilverlightVersion1 SMALLINT NOT NULL,
+                SilverlightVersion2 SMALLINT NOT NULL,
+                SilverlightVersion3 INT NOT NULL,
+                SilverlightVersion4 SMALLINT NOT NULL,
+                PageCharset STRING NOT NULL,
+                CodeVersion INT NOT NULL,
+                IsLink SMALLINT NOT NULL,
+                IsDownload SMALLINT NOT NULL,
+                IsNotBounce SMALLINT NOT NULL,
+                FUniqID BIGINT NOT NULL,
+                OriginalURL STRING NOT NULL,
+                HID INT NOT NULL,
+                IsOldCounter SMALLINT NOT NULL,
+                IsEvent SMALLINT NOT NULL,
+                IsParameter SMALLINT NOT NULL,
+                DontCountHits SMALLINT NOT NULL,
+                WithHash SMALLINT NOT NULL,
+                HitColor CHAR NOT NULL,
+                LocalEventTime DateTimeV2 NOT NULL,
+                Age SMALLINT NOT NULL,
+                Sex SMALLINT NOT NULL,
+                Income SMALLINT NOT NULL,
+                Interests SMALLINT NOT NULL,
+                Robotness SMALLINT NOT NULL,
+                RemoteIP INT NOT NULL,
+                WindowName INT NOT NULL,
+                OpenerName INT NOT NULL,
+                HistoryLength SMALLINT NOT NULL,
+                BrowserLanguage STRING NOT NULL,
+                BrowserCountry STRING NOT NULL,
+                SocialNetwork STRING NOT NULL,
+                SocialAction STRING NOT NULL,
+                HTTPError SMALLINT NOT NULL,
+                SendTiming INT NOT NULL,
+                DNSTiming INT NOT NULL,
+                ConnectTiming INT NOT NULL,
+                ResponseStartTiming INT NOT NULL,
+                ResponseEndTiming INT NOT NULL,
+                FetchTiming INT NOT NULL,
+                SocialSourceNetworkID SMALLINT NOT NULL,
+                SocialSourcePage STRING NOT NULL,
+                ParamPrice BIGINT NOT NULL,
+                ParamOrderID STRING NOT NULL,
+                ParamCurrency STRING NOT NULL,
+                ParamCurrencyID SMALLINT NOT NULL,
+                OpenstatServiceName STRING NOT NULL,
+                OpenstatCampaignID STRING NOT NULL,
+                OpenstatAdID STRING NOT NULL,
+                OpenstatSourceID STRING NOT NULL,
+                UTMSource STRING NOT NULL,
+                UTMMedium STRING NOT NULL,
+                UTMCampaign STRING NOT NULL,
+                UTMContent STRING NOT NULL,
+                UTMTerm STRING NOT NULL,
+                FromTag STRING NOT NULL,
+                HasGCLID SMALLINT NOT NULL,
+                RefererHash BIGINT NOT NULL,
+                URLHash BIGINT NOT NULL,
+                CLID INT NOT NULL
+            )  
+            DUPLICATE KEY (CounterID, EventDate, UserID, EventTime, WatchID) 
+            DISTRIBUTED BY HASH(UserID) BUCKETS 16
+            PROPERTIES (\"replication_num\"=\"1\");
+        "
+        ${clt} -D"${DB}" -e"${ddl}"
+        echo "#### load data"
+        if [[ ! -d "${data_home}" ]]; then mkdir -p "${data_home}"; fi
+        if [[ ! -f "${data_home}"/hits.json.1000000 ]] || [[ $(wc -c "${data_home}"/hits.json.1000000 | awk '{print $1}') != '2358488459' ]]; then
+            cd "${data_home}"
+            wget --continue 'https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/ClickBench/hits.json.1000000'
+            cd -
+        fi
+        ret=$(
+            curl --location-trusted \
+                -u root: \
+                -T "${data_home}/hits.json.1000000" \
+                -H "format:json" \
+                -H "label:hits_json" \
+                -H "read_json_by_line:true" \
+                -H 'jsonpaths:["$.WatchID","$.JavaEnable","$.Title","$.GoodEvent","$.EventTime","$.EventDate","$.CounterID","$.ClientIP","$.RegionID","$.UserID","$.CounterClass","$.OS","$.UserAgent","$.URL","$.Referer","$.IsRefresh","$.RefererCategoryID","$.RefererRegionID","$.URLCategoryID","$.URLRegionID","$.ResolutionWidth","$.ResolutionHeight","$.ResolutionDepth","$.FlashMajor","$.FlashMinor","$.FlashMinor2","$.NetMajor","$.NetMinor","$.UserAgentMajor","$.UserAgentMinor","$.CookieEnable","$.JavascriptEnable","$.IsMobile","$.MobilePhone","$.MobilePhoneModel","$.Params","$.IPNetworkID","$.TraficSourceID","$.SearchEngineID","$.SearchPhrase","$.AdvEngineID","$.IsArtifical","$.WindowClientWidth","$.WindowClientHeight","$.ClientTimeZone","$.ClientEventTime","$.SilverlightVersion1","$.SilverlightVersion2","$.SilverlightVersion3","$.SilverlightVersion4","$.PageCharset","$.CodeVersion","$.IsLink","$.IsDownload","$.IsNotBounce","$.FUniqID","$.OriginalURL","$.HID","$.IsOldCounter","$.IsEvent","$.IsParameter","$.DontCountHits","$.WithHash","$.HitColor","$.LocalEventTime","$.Age","$.Sex","$.Income","$.Interests","$.Robotness","$.RemoteIP","$.WindowName","$.OpenerName","$.HistoryLength","$.BrowserLanguage","$.BrowserCountry","$.SocialNetwork","$.SocialAction","$.HTTPError","$.SendTiming","$.DNSTiming","$.ConnectTiming","$.ResponseStartTiming","$.ResponseEndTiming","$.FetchTiming","$.SocialSourceNetworkID","$.SocialSourcePage","$.ParamPrice","$.ParamOrderID","$.ParamCurrency","$.ParamCurrencyID","$.OpenstatServiceName","$.OpenstatCampaignID","$.OpenstatAdID","$.OpenstatSourceID","$.UTMSource","$.UTMMedium","$.UTMCampaign","$.UTMContent","$.UTMTerm","$.FromTag","$.HasGCLID","$.RefererHash","$.URLHash","$.CLID"]' \
+                -H "columns: WatchID,JavaEnable,Title,GoodEvent,EventTime,EventDate,CounterID,ClientIP,RegionID,UserID,CounterClass,OS,UserAgent,URL,Referer,IsRefresh,RefererCategoryID,RefererRegionID,URLCategoryID,URLRegionID,ResolutionWidth,ResolutionHeight,ResolutionDepth,FlashMajor,FlashMinor,FlashMinor2,NetMajor,NetMinor,UserAgentMajor,UserAgentMinor,CookieEnable,JavascriptEnable,IsMobile,MobilePhone,MobilePhoneModel,Params,IPNetworkID,TraficSourceID,SearchEngineID,SearchPhrase,AdvEngineID,IsArtifical,WindowClientWidth,WindowClientHeight,ClientTimeZone,ClientEventTime,SilverlightVersion1,SilverlightVersion2,SilverlightVersion3,SilverlightVersion4,PageCharset,CodeVersion,IsLink,IsDownload,IsNotBounce,FUniqID,OriginalURL,HID,IsOldCounter,IsEvent,IsParameter,DontCountHits,WithHash,HitColor,LocalEventTime,Age,Sex,Income,Interests,Robotness,RemoteIP,WindowName,OpenerName,HistoryLength,BrowserLanguage,BrowserCountry,SocialNetwork,SocialAction,HTTPError,SendTiming,DNSTiming,ConnectTiming,ResponseStartTiming,ResponseEndTiming,FetchTiming,SocialSourceNetworkID,SocialSourcePage,ParamPrice,ParamOrderID,ParamCurrency,ParamCurrencyID,OpenstatServiceName,OpenstatCampaignID,OpenstatAdID,OpenstatSourceID,UTMSource,UTMMedium,UTMCampaign,UTMContent,UTMTerm,FromTag,HasGCLID,RefererHash,URLHash,CLID" \
+                "http://${FE_HOST:-127.0.0.1}:${http_port}/api/${DB}/hits_json/_stream_load"
+        )
+        sleep 5
+        if [[ $(${clt} -D"${DB}" -e"select count(*) from hits_json" | sed -n '2p') != 1000000 ]]; then echo "check load fail..." && return 1; fi
+
+        echo "#### record load test result"
+        stream_load_json_size=$(echo "${ret}" | jq '.LoadBytes')
+        stream_load_json_time=$(printf "%.0f" "$(echo "scale=1;$(echo "${ret}" | jq '.LoadTimeMs')/1000" | bc)")
+        stream_load_json_speed=$(echo "${stream_load_json_size} / 1024 / 1024/ ${stream_load_json_time}" | bc)
+        export stream_load_json_size
+        export stream_load_json_time
+        export stream_load_json_speed
+    }
+
+    stream_load_orc() {
+        echo "#### create table"
+        ddl="
+        CREATE TABLE IF NOT EXISTS  hits_orc (
+            CounterID INT NOT NULL, 
+            EventDate INT NOT NULL, 
+            UserID BIGINT NOT NULL, 
+            EventTime INT NOT NULL, 
+            WatchID BIGINT NOT NULL, 
+            JavaEnable SMALLINT NOT NULL,
+            Title STRING NOT NULL,
+            GoodEvent SMALLINT NOT NULL,
+            ClientIP INT NOT NULL,
+            RegionID INT NOT NULL,
+            CounterClass SMALLINT NOT NULL,
+            OS SMALLINT NOT NULL,
+            UserAgent SMALLINT NOT NULL,
+            URL STRING NOT NULL,
+            Referer STRING NOT NULL,
+            IsRefresh SMALLINT NOT NULL,
+            RefererCategoryID SMALLINT NOT NULL,
+            RefererRegionID INT NOT NULL,
+            URLCategoryID SMALLINT NOT NULL,
+            URLRegionID INT NOT NULL,
+            ResolutionWidth SMALLINT NOT NULL,
+            ResolutionHeight SMALLINT NOT NULL,
+            ResolutionDepth SMALLINT NOT NULL,
+            FlashMajor SMALLINT NOT NULL,
+            FlashMinor SMALLINT NOT NULL,
+            FlashMinor2 STRING NOT NULL,
+            NetMajor SMALLINT NOT NULL,
+            NetMinor SMALLINT NOT NULL,
+            UserAgentMajor SMALLINT NOT NULL,
+            UserAgentMinor VARCHAR(255) NOT NULL,
+            CookieEnable SMALLINT NOT NULL,
+            JavascriptEnable SMALLINT NOT NULL,
+            IsMobile SMALLINT NOT NULL,
+            MobilePhone SMALLINT NOT NULL,
+            MobilePhoneModel STRING NOT NULL,
+            Params STRING NOT NULL,
+            IPNetworkID INT NOT NULL,
+            TraficSourceID SMALLINT NOT NULL,
+            SearchEngineID SMALLINT NOT NULL,
+            SearchPhrase STRING NOT NULL,
+            AdvEngineID SMALLINT NOT NULL,
+            IsArtifical SMALLINT NOT NULL,
+            WindowClientWidth SMALLINT NOT NULL,
+            WindowClientHeight SMALLINT NOT NULL,
+            ClientTimeZone SMALLINT NOT NULL,
+            ClientEventTime INT NOT NULL,
+            SilverlightVersion1 SMALLINT NOT NULL,
+            SilverlightVersion2 SMALLINT NOT NULL,
+            SilverlightVersion3 INT NOT NULL,
+            SilverlightVersion4 SMALLINT NOT NULL,
+            PageCharset STRING NOT NULL,
+            CodeVersion INT NOT NULL,
+            IsLink SMALLINT NOT NULL,
+            IsDownload SMALLINT NOT NULL,
+            IsNotBounce SMALLINT NOT NULL,
+            FUniqID BIGINT NOT NULL,
+            OriginalURL STRING NOT NULL,
+            HID INT NOT NULL,
+            IsOldCounter SMALLINT NOT NULL,
+            IsEvent SMALLINT NOT NULL,
+            IsParameter SMALLINT NOT NULL,
+            DontCountHits SMALLINT NOT NULL,
+            WithHash SMALLINT NOT NULL,
+            HitColor CHAR NOT NULL,
+            LocalEventTime INT NOT NULL,
+            Age SMALLINT NOT NULL,
+            Sex SMALLINT NOT NULL,
+            Income SMALLINT NOT NULL,
+            Interests SMALLINT NOT NULL,
+            Robotness SMALLINT NOT NULL,
+            RemoteIP INT NOT NULL,
+            WindowName INT NOT NULL,
+            OpenerName INT NOT NULL,
+            HistoryLength SMALLINT NOT NULL,
+            BrowserLanguage STRING NOT NULL,
+            BrowserCountry STRING NOT NULL,
+            SocialNetwork STRING NOT NULL,
+            SocialAction STRING NOT NULL,
+            HTTPError SMALLINT NOT NULL,
+            SendTiming INT NOT NULL,
+            DNSTiming INT NOT NULL,
+            ConnectTiming INT NOT NULL,
+            ResponseStartTiming INT NOT NULL,
+            ResponseEndTiming INT NOT NULL,
+            FetchTiming INT NOT NULL,
+            SocialSourceNetworkID SMALLINT NOT NULL,
+            SocialSourcePage STRING NOT NULL,
+            ParamPrice BIGINT NOT NULL,
+            ParamOrderID STRING NOT NULL,
+            ParamCurrency STRING NOT NULL,
+            ParamCurrencyID SMALLINT NOT NULL,
+            OpenstatServiceName STRING NOT NULL,
+            OpenstatCampaignID STRING NOT NULL,
+            OpenstatAdID STRING NOT NULL,
+            OpenstatSourceID STRING NOT NULL,
+            UTMSource STRING NOT NULL,
+            UTMMedium STRING NOT NULL,
+            UTMCampaign STRING NOT NULL,
+            UTMContent STRING NOT NULL,
+            UTMTerm STRING NOT NULL,
+            FromTag STRING NOT NULL,
+            HasGCLID SMALLINT NOT NULL,
+            RefererHash BIGINT NOT NULL,
+            URLHash BIGINT NOT NULL,
+            CLID INT NOT NULL
+        )  
+        DUPLICATE KEY (CounterID, EventDate, UserID, EventTime, WatchID) 
+        DISTRIBUTED BY HASH(UserID) BUCKETS 16
+        PROPERTIES (\"replication_num\"=\"1\");
+        "
+        ${clt} -D"${DB}" -e"${ddl}"
+        echo "#### load data"
+        if [[ ! -d "${data_home}" ]]; then mkdir -p "${data_home}"; fi
+        if [[ ! -f "${data_home}"/hits_0.orc ]] || [[ $(wc -c "${data_home}"/hits_0.orc | awk '{print $1}') != '1101869774' ]]; then
+            cd "${data_home}"
+            wget --continue 'https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/ClickBench/hits_0.orc'
+            cd -
+        fi
+        ret=$(
+            curl --location-trusted \
+                -u root: \
+                -T "${data_home}/hits_0.orc" \
+                -H "format:orc" \
+                -H "label:hits_0_orc" \
+                -H "columns: watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid" \
+                "http://${FE_HOST:-127.0.0.1}:${http_port}/api/${DB}/hits_orc/_stream_load"
+        )
+        sleep 5
+        if [[ $(${clt} -D"${DB}" -e"select count(*) from hits_orc" | sed -n '2p') != 8800160 ]]; then echo "check load fail..." && return 1; fi
+
+        echo "#### record load test result"
+        stream_load_orc_size=$(echo "${ret}" | jq '.LoadBytes')
+        stream_load_orc_time=$(printf "%.0f" "$(echo "scale=1;$(echo "${ret}" | jq '.LoadTimeMs')/1000" | bc)")
+        stream_load_orc_speed=$(echo "${stream_load_orc_size} / 1024 / 1024/ ${stream_load_orc_time}" | bc)
+        export stream_load_orc_size
+        export stream_load_orc_time
+        export stream_load_orc_speed
+    }
+
+    stream_load_parquet() {
+        echo "#### create table"
+        ddl="
+        CREATE TABLE IF NOT EXISTS  hits_parquet (
+            CounterID INT NOT NULL, 
+            EventDate INT NOT NULL, 
+            UserID BIGINT NOT NULL, 
+            EventTime INT NOT NULL, 
+            WatchID BIGINT NOT NULL, 
+            JavaEnable SMALLINT NOT NULL,
+            Title STRING NOT NULL,
+            GoodEvent SMALLINT NOT NULL,
+            ClientIP INT NOT NULL,
+            RegionID INT NOT NULL,
+            CounterClass SMALLINT NOT NULL,
+            OS SMALLINT NOT NULL,
+            UserAgent SMALLINT NOT NULL,
+            URL STRING NOT NULL,
+            Referer STRING NOT NULL,
+            IsRefresh SMALLINT NOT NULL,
+            RefererCategoryID SMALLINT NOT NULL,
+            RefererRegionID INT NOT NULL,
+            URLCategoryID SMALLINT NOT NULL,
+            URLRegionID INT NOT NULL,
+            ResolutionWidth SMALLINT NOT NULL,
+            ResolutionHeight SMALLINT NOT NULL,
+            ResolutionDepth SMALLINT NOT NULL,
+            FlashMajor SMALLINT NOT NULL,
+            FlashMinor SMALLINT NOT NULL,
+            FlashMinor2 STRING NOT NULL,
+            NetMajor SMALLINT NOT NULL,
+            NetMinor SMALLINT NOT NULL,
+            UserAgentMajor SMALLINT NOT NULL,
+            UserAgentMinor VARCHAR(255) NOT NULL,
+            CookieEnable SMALLINT NOT NULL,
+            JavascriptEnable SMALLINT NOT NULL,
+            IsMobile SMALLINT NOT NULL,
+            MobilePhone SMALLINT NOT NULL,
+            MobilePhoneModel STRING NOT NULL,
+            Params STRING NOT NULL,
+            IPNetworkID INT NOT NULL,
+            TraficSourceID SMALLINT NOT NULL,
+            SearchEngineID SMALLINT NOT NULL,
+            SearchPhrase STRING NOT NULL,
+            AdvEngineID SMALLINT NOT NULL,
+            IsArtifical SMALLINT NOT NULL,
+            WindowClientWidth SMALLINT NOT NULL,
+            WindowClientHeight SMALLINT NOT NULL,
+            ClientTimeZone SMALLINT NOT NULL,
+            ClientEventTime INT NOT NULL,
+            SilverlightVersion1 SMALLINT NOT NULL,
+            SilverlightVersion2 SMALLINT NOT NULL,
+            SilverlightVersion3 INT NOT NULL,
+            SilverlightVersion4 SMALLINT NOT NULL,
+            PageCharset STRING NOT NULL,
+            CodeVersion INT NOT NULL,
+            IsLink SMALLINT NOT NULL,
+            IsDownload SMALLINT NOT NULL,
+            IsNotBounce SMALLINT NOT NULL,
+            FUniqID BIGINT NOT NULL,
+            OriginalURL STRING NOT NULL,
+            HID INT NOT NULL,
+            IsOldCounter SMALLINT NOT NULL,
+            IsEvent SMALLINT NOT NULL,
+            IsParameter SMALLINT NOT NULL,
+            DontCountHits SMALLINT NOT NULL,
+            WithHash SMALLINT NOT NULL,
+            HitColor CHAR NOT NULL,
+            LocalEventTime INT NOT NULL,
+            Age SMALLINT NOT NULL,
+            Sex SMALLINT NOT NULL,
+            Income SMALLINT NOT NULL,
+            Interests SMALLINT NOT NULL,
+            Robotness SMALLINT NOT NULL,
+            RemoteIP INT NOT NULL,
+            WindowName INT NOT NULL,
+            OpenerName INT NOT NULL,
+            HistoryLength SMALLINT NOT NULL,
+            BrowserLanguage STRING NOT NULL,
+            BrowserCountry STRING NOT NULL,
+            SocialNetwork STRING NOT NULL,
+            SocialAction STRING NOT NULL,
+            HTTPError SMALLINT NOT NULL,
+            SendTiming INT NOT NULL,
+            DNSTiming INT NOT NULL,
+            ConnectTiming INT NOT NULL,
+            ResponseStartTiming INT NOT NULL,
+            ResponseEndTiming INT NOT NULL,
+            FetchTiming INT NOT NULL,
+            SocialSourceNetworkID SMALLINT NOT NULL,
+            SocialSourcePage STRING NOT NULL,
+            ParamPrice BIGINT NOT NULL,
+            ParamOrderID STRING NOT NULL,
+            ParamCurrency STRING NOT NULL,
+            ParamCurrencyID SMALLINT NOT NULL,
+            OpenstatServiceName STRING NOT NULL,
+            OpenstatCampaignID STRING NOT NULL,
+            OpenstatAdID STRING NOT NULL,
+            OpenstatSourceID STRING NOT NULL,
+            UTMSource STRING NOT NULL,
+            UTMMedium STRING NOT NULL,
+            UTMCampaign STRING NOT NULL,
+            UTMContent STRING NOT NULL,
+            UTMTerm STRING NOT NULL,
+            FromTag STRING NOT NULL,
+            HasGCLID SMALLINT NOT NULL,
+            RefererHash BIGINT NOT NULL,
+            URLHash BIGINT NOT NULL,
+            CLID INT NOT NULL
+        )
+        DUPLICATE KEY (CounterID, EventDate, UserID, EventTime, WatchID) 
+        DISTRIBUTED BY HASH(UserID) BUCKETS 16
+        PROPERTIES (\"replication_num\"=\"1\");
+        "
+        ${clt} -D"${DB}" -e"${ddl}"
+        echo "#### load data"
+        stream_load_parquet_size=0
+        stream_load_parquet_time=0
+        if [[ ! -d "${data_home}" ]]; then mkdir -p "${data_home}"; fi
+        declare -A file_sizes=(['hits_0.parquet']=122446530 ['hits_1.parquet']=174965044 ['hits_2.parquet']=230595491 ['hits_3.parquet']=192507052 ['hits_4.parquet']=140929275)
+        for file_name in ${!file_sizes[*]}; do
+            size_expect=${file_sizes[${file_name}]}
+            if [[ ! -f "${data_home}/${file_name}" ]] || [[ $(wc -c "${data_home}/${file_name}" | awk '{print $1}') != "${size_expect}" ]]; then
+                cd "${data_home}" && rm -f "${file_name}" && wget --continue "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/ClickBench/${file_name}" && cd - || exit
+            fi
+            if ret=$(
+                curl --location-trusted \
+                    -u root: \
+                    -T "${data_home}/${file_name}" \
+                    -H "format:parquet" \
+                    -H "label:${file_name//./_}" \
+                    -H "columns: watchid,javaenable,title,goodevent,eventtime,eventdate,counterid,clientip,regionid,userid,counterclass,os,useragent,url,referer,isrefresh,referercategoryid,refererregionid,urlcategoryid,urlregionid,resolutionwidth,resolutionheight,resolutiondepth,flashmajor,flashminor,flashminor2,netmajor,netminor,useragentmajor,useragentminor,cookieenable,javascriptenable,ismobile,mobilephone,mobilephonemodel,params,ipnetworkid,traficsourceid,searchengineid,searchphrase,advengineid,isartifical,windowclientwidth,windowclientheight,clienttimezone,clienteventtime,silverlightversion1,silverlightversion2,silverlightversion3,silverlightversion4,pagecharset,codeversion,islink,isdownload,isnotbounce,funiqid,originalurl,hid,isoldcounter,isevent,isparameter,dontcounthits,withhash,hitcolor,localeventtime,age,sex,income,interests,robotness,remoteip,windowname,openername,historylength,browserlanguage,browsercountry,socialnetwork,socialaction,httperror,sendtiming,dnstiming,connecttiming,responsestarttiming,responseendtiming,fetchtiming,socialsourcenetworkid,socialsourcepage,paramprice,paramorderid,paramcurrency,paramcurrencyid,openstatservicename,openstatcampaignid,openstatadid,openstatsourceid,utmsource,utmmedium,utmcampaign,utmcontent,utmterm,fromtag,hasgclid,refererhash,urlhash,clid" \
+                    "http://${FE_HOST:-127.0.0.1}:${http_port}/api/${DB}/hits_parquet/_stream_load"
+            ); then
+                _stream_load_parquet_size=$(echo "${ret}" | jq '.LoadBytes')
+                _stream_load_parquet_time=$(printf "%.0f" "$(echo "scale=1;$(echo "${ret}" | jq '.LoadTimeMs')/1000" | bc)")
+                stream_load_parquet_size=$((stream_load_parquet_size + _stream_load_parquet_size))
+                stream_load_parquet_time=$((stream_load_parquet_time + _stream_load_parquet_time))
+            fi
+        done
+        sleep 5
+        if [[ $(${clt} -D"${DB}" -e"select count(*) from hits_parquet" | sed -n '2p') != 5000000 ]]; then echo "check load fail..." && return 1; fi
+
+        echo "#### record load test result"
+        stream_load_parquet_speed=$(echo "${stream_load_parquet_size} / 1024 / 1024/ ${stream_load_parquet_time}" | bc)
+        export stream_load_parquet_size
+        export stream_load_parquet_time
+        export stream_load_parquet_speed
+    }
+
+    insert_into_select() {
+        echo "#### create table"
+        ddl="
+        CREATE TABLE IF NOT EXISTS  hits_insert_into_select (
+            CounterID INT NOT NULL, 
+            EventDate INT NOT NULL, 
+            UserID BIGINT NOT NULL, 
+            EventTime INT NOT NULL, 
+            WatchID BIGINT NOT NULL, 
+            JavaEnable SMALLINT NOT NULL,
+            Title STRING NOT NULL,
+            GoodEvent SMALLINT NOT NULL,
+            ClientIP INT NOT NULL,
+            RegionID INT NOT NULL,
+            CounterClass SMALLINT NOT NULL,
+            OS SMALLINT NOT NULL,
+            UserAgent SMALLINT NOT NULL,
+            URL STRING NOT NULL,
+            Referer STRING NOT NULL,
+            IsRefresh SMALLINT NOT NULL,
+            RefererCategoryID SMALLINT NOT NULL,
+            RefererRegionID INT NOT NULL,
+            URLCategoryID SMALLINT NOT NULL,
+            URLRegionID INT NOT NULL,
+            ResolutionWidth SMALLINT NOT NULL,
+            ResolutionHeight SMALLINT NOT NULL,
+            ResolutionDepth SMALLINT NOT NULL,
+            FlashMajor SMALLINT NOT NULL,
+            FlashMinor SMALLINT NOT NULL,
+            FlashMinor2 STRING NOT NULL,
+            NetMajor SMALLINT NOT NULL,
+            NetMinor SMALLINT NOT NULL,
+            UserAgentMajor SMALLINT NOT NULL,
+            UserAgentMinor VARCHAR(255) NOT NULL,
+            CookieEnable SMALLINT NOT NULL,
+            JavascriptEnable SMALLINT NOT NULL,
+            IsMobile SMALLINT NOT NULL,
+            MobilePhone SMALLINT NOT NULL,
+            MobilePhoneModel STRING NOT NULL,
+            Params STRING NOT NULL,
+            IPNetworkID INT NOT NULL,
+            TraficSourceID SMALLINT NOT NULL,
+            SearchEngineID SMALLINT NOT NULL,
+            SearchPhrase STRING NOT NULL,
+            AdvEngineID SMALLINT NOT NULL,
+            IsArtifical SMALLINT NOT NULL,
+            WindowClientWidth SMALLINT NOT NULL,
+            WindowClientHeight SMALLINT NOT NULL,
+            ClientTimeZone SMALLINT NOT NULL,
+            ClientEventTime INT NOT NULL,
+            SilverlightVersion1 SMALLINT NOT NULL,
+            SilverlightVersion2 SMALLINT NOT NULL,
+            SilverlightVersion3 INT NOT NULL,
+            SilverlightVersion4 SMALLINT NOT NULL,
+            PageCharset STRING NOT NULL,
+            CodeVersion INT NOT NULL,
+            IsLink SMALLINT NOT NULL,
+            IsDownload SMALLINT NOT NULL,
+            IsNotBounce SMALLINT NOT NULL,
+            FUniqID BIGINT NOT NULL,
+            OriginalURL STRING NOT NULL,
+            HID INT NOT NULL,
+            IsOldCounter SMALLINT NOT NULL,
+            IsEvent SMALLINT NOT NULL,
+            IsParameter SMALLINT NOT NULL,
+            DontCountHits SMALLINT NOT NULL,
+            WithHash SMALLINT NOT NULL,
+            HitColor CHAR NOT NULL,
+            LocalEventTime INT NOT NULL,
+            Age SMALLINT NOT NULL,
+            Sex SMALLINT NOT NULL,
+            Income SMALLINT NOT NULL,
+            Interests SMALLINT NOT NULL,
+            Robotness SMALLINT NOT NULL,
+            RemoteIP INT NOT NULL,
+            WindowName INT NOT NULL,
+            OpenerName INT NOT NULL,
+            HistoryLength SMALLINT NOT NULL,
+            BrowserLanguage STRING NOT NULL,
+            BrowserCountry STRING NOT NULL,
+            SocialNetwork STRING NOT NULL,
+            SocialAction STRING NOT NULL,
+            HTTPError SMALLINT NOT NULL,
+            SendTiming INT NOT NULL,
+            DNSTiming INT NOT NULL,
+            ConnectTiming INT NOT NULL,
+            ResponseStartTiming INT NOT NULL,
+            ResponseEndTiming INT NOT NULL,
+            FetchTiming INT NOT NULL,
+            SocialSourceNetworkID SMALLINT NOT NULL,
+            SocialSourcePage STRING NOT NULL,
+            ParamPrice BIGINT NOT NULL,
+            ParamOrderID STRING NOT NULL,
+            ParamCurrency STRING NOT NULL,
+            ParamCurrencyID SMALLINT NOT NULL,
+            OpenstatServiceName STRING NOT NULL,
+            OpenstatCampaignID STRING NOT NULL,
+            OpenstatAdID STRING NOT NULL,
+            OpenstatSourceID STRING NOT NULL,
+            UTMSource STRING NOT NULL,
+            UTMMedium STRING NOT NULL,
+            UTMCampaign STRING NOT NULL,
+            UTMContent STRING NOT NULL,
+            UTMTerm STRING NOT NULL,
+            FromTag STRING NOT NULL,
+            HasGCLID SMALLINT NOT NULL,
+            RefererHash BIGINT NOT NULL,
+            URLHash BIGINT NOT NULL,
+            CLID INT NOT NULL
+        )
+        DUPLICATE KEY (CounterID, EventDate, UserID, EventTime, WatchID) 
+        DISTRIBUTED BY HASH(UserID) BUCKETS 16
+        PROPERTIES (\"replication_num\"=\"1\");
+        "
+        ${clt} -D"${DB}" -e"${ddl}"
+
+        echo "#### load data by INSERT INTO SELECT"
+        insert_into_select_time=0
+        insert_into_select_rows=10000000
+        start=$(date +%s%3N)
+        if ${clt} -e"insert into ${DB}.hits_insert_into_select select * from clickbench.hits limit ${insert_into_select_rows};"; then
+            end=$(date +%s%3N)
+            insert_into_select_time=$(echo "scale=1; (${end} - ${start})/1000" | bc)
+        else
+            echo "ERROR: failed to insert into ${DB}.hits_insert_into_select select * from clickbench.hits limit ${insert_into_select_rows};"
+            return 1
+        fi
+        sleep 5
+        if [[ $(${clt} -D"${DB}" -e"select count(*) from hits_insert_into_select" | sed -n '2p') != "${insert_into_select_rows}" ]]; then echo "check load fail..." && return 1; fi
+
+        echo "#### record load test result"
+        insert_into_select_speed=$(echo "${insert_into_select_rows} / 1000 / ${insert_into_select_time}" | bc)
+        export insert_into_select_rows
+        export insert_into_select_time
+        export insert_into_select_speed
+    }
+
+    echo "#### 1. Restart doris"
+    if ! restart_doris; then echo "ERROR: Restart doris failed" && exit 1; fi
+
+    echo "#### 3. run streamload test"
+    set_session_variable runtime_filter_mode global
+    ${clt} -e "DROP DATABASE IF EXISTS ${DB}" && sleep 1
+    ${clt} -e "CREATE DATABASE IF NOT EXISTS ${DB}" && sleep 5
+    if ! stream_load_json; then exit 1; fi
+    if ! stream_load_orc; then exit 1; fi
+    if ! stream_load_parquet; then exit 1; fi
+    if ! insert_into_select; then exit 1; fi
+    if ! check_load_performance; then exit 1; fi
+
+    echo "#### 4. comment result on tpch"
+    comment_body="Load test result on commit ${commit_id:-} with default conf and session variables"
+    if [[ -n ${stream_load_json_time} ]]; then comment_body="${comment_body}\n stream load json:         ${stream_load_json_time} seconds loaded ${stream_load_json_size} Bytes, about ${stream_load_json_speed} MB/s"; fi
+    if [[ -n ${stream_load_orc_time} ]]; then comment_body="${comment_body}\n stream load orc:          ${stream_load_orc_time} seconds loaded ${stream_load_orc_size} Bytes, about ${stream_load_orc_speed} MB/s"; fi
+    if [[ -n ${stream_load_parquet_time} ]]; then comment_body="${comment_body}\n stream load parquet:          ${stream_load_parquet_time} seconds loaded ${stream_load_parquet_size} Bytes, about ${stream_load_parquet_speed} MB/s"; fi
+    if [[ -n ${insert_into_select_time} ]]; then comment_body="${comment_body}\n insert into select:          ${insert_into_select_time} seconds inserted ${insert_into_select_rows} Rows, about ${insert_into_select_speed}K ops/s"; fi
+
+    comment_body=$(echo "${comment_body}" | sed -e ':a;N;$!ba;s/\t/\\t/g;s/\n/\\n/g') # 将所有的 Tab字符替换为\t 换行符替换为\n
+    create_an_issue_comment_tpch "${pull_request_num:-}" "${comment_body}"
+)
+exit_flag="$?"
+
+echo "#### 5. check if need backup doris logs"
+if [[ ${exit_flag} != "0" ]]; then
+    print_doris_fe_log
+    print_doris_be_log
+    if file_name=$(archive_doris_logs "${pull_request_num}_${commit_id}_doris_logs.tar.gz"); then
+        upload_doris_log_to_oss "${file_name}"
+    fi
+fi
+
+exit "${exit_flag}"
diff --git a/regression-test/pipeline/performance/run-tpcds.sh b/regression-test/pipeline/performance/run-tpcds.sh
new file mode 100644
index 00000000000000..0684454ca49b98
--- /dev/null
+++ b/regression-test/pipeline/performance/run-tpcds.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+export DEBUG=true
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/run-tpcds.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
+    bash -x run-tpcds.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/run-tpcds.sh" && exit 1
+fi
+EOF
+
+#####################################################################################
+## run-tpcds.sh content ##
+
+# shellcheck source=/dev/null
+# check_tpcds_table_rows, restart_doris, set_session_variable, check_tpcds_result
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+# shellcheck source=/dev/null
+# create_an_issue_comment
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh
+# shellcheck source=/dev/null
+# upload_doris_log_to_oss
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh
+
+if ${DEBUG:-false}; then
+    pull_request_num="28431"
+    commit_id="5f5c4c80564c76ff4267fc4ce6a5408498ed1ab5"
+    SF="1"
+fi
+echo "#### Check env"
+if [[ -z "${teamcity_build_checkoutDir}" ||
+    -z "${pull_request_num}" ||
+    -z "${commit_id}" ]]; then
+    echo "ERROR: env teamcity_build_checkoutDir or pull_request_num or commit_id not set"
+    exit 1
+fi
+
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+
+echo "#### Run tpcds test on Doris ####"
+DORIS_HOME="${teamcity_build_checkoutDir}/output"
+export DORIS_HOME
+cold_run_time_threshold=${cold_run_time_threshold:-600000} # ms
+hot_run_time_threshold=${hot_run_time_threshold:-240000}   # ms
+exit_flag=0
+
+(
+    set -e
+    shopt -s inherit_errexit
+
+    echo "#### 1. Restart doris"
+    if ! restart_doris; then echo "ERROR: Restart doris failed" && exit 1; fi
+
+    echo "#### 2. check if need to load data"
+    SF=${SF:-"100"}                                                                      # SCALE FACTOR
+    TPCDS_DATA_DIR="/data/tpcds/sf_${SF}"                                                # no / at the end
+    TPCDS_DATA_DIR_LINK="${teamcity_build_checkoutDir}"/tools/tpcds-tools/bin/tpcds-data # no / at the end
+    db_name="tpcds_sf${SF}"
+    sed -i "s|^export DB=.*$|export DB='${db_name}'|g" \
+        "${teamcity_build_checkoutDir}"/tools/tpcds-tools/conf/doris-cluster.conf
+    if ! check_tpcds_table_rows "${db_name}" "${SF}"; then
+        echo "INFO: need to load tpcds-sf${SF} data"
+        if ${force_load_data:-false}; then echo "INFO: force_load_data is true"; else echo "ERROR: force_load_data is false" && exit 1; fi
+        # prepare data
+        mkdir -p "${TPCDS_DATA_DIR}"
+        (
+            cd "${TPCDS_DATA_DIR}" || exit 1
+            declare -A table_file_count
+            if [[ ${SF} == "1" ]]; then
+                table_file_count=(['income_band']=1 ['ship_mode']=1 ['warehouse']=1 ['reason']=1 ['web_site']=1 ['call_center']=1 ['store']=1 ['promotion']=1 ['household_demographics']=1 ['web_page']=1 ['catalog_page']=1 ['time_dim']=1 ['date_dim']=1 ['item']=1 ['customer_demographics']=10 ['customer_address']=1 ['customer']=1 ['web_returns']=1 ['catalog_returns']=1 ['store_returns']=1 ['inventory']=10 ['web_sales']=1 ['catalog_sales']=1 ['store_sales']=1)
+            elif [[ ${SF} == "100" ]]; then
+                table_file_count=(['income_band']=1 ['ship_mode']=1 ['warehouse']=1 ['reason']=1 ['web_site']=1 ['call_center']=1 ['store']=1 ['promotion']=1 ['household_demographics']=1 ['web_page']=1 ['catalog_page']=1 ['time_dim']=1 ['date_dim']=1 ['item']=1 ['customer_demographics']=10 ['customer_address']=10 ['customer']=10 ['web_returns']=10 ['catalog_returns']=10 ['store_returns']=10 ['inventory']=10 ['web_sales']=10 ['catalog_sales']=10 ['store_sales']=10)
+            fi
+            for table_name in ${!table_file_count[*]}; do
+                if [[ ${table_file_count[${table_name}]} -eq 1 ]]; then
+                    url="https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/tpcds/sf${SF}/${table_name}_1_10.dat.gz"
+                    if [[ -f ${table_name}_1_10.dat ]]; then continue; fi
+                    if ! wget --continue -t3 -q "${url}"; then echo "ERROR: wget --continue ${url}" && exit 1; fi
+                    if ! gzip -d "${table_name}_1_10.dat.gz"; then echo "ERROR: gzip -d ${table_name}_1_10.dat.gz" && exit 1; fi
+                elif [[ ${table_file_count[${table_name}]} -eq 10 ]]; then
+                    (
+                        for i in {1..10}; do
+                            url="https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/tpcds/sf${SF}/${table_name}_${i}_10.dat.gz"
+                            if [[ -f ${table_name}_${i}_10.dat ]]; then continue; fi
+                            if ! wget --continue -t3 -q "${url}"; then echo "ERROR: wget --continue ${url}" && exit 1; fi
+                            if ! gzip -d "${table_name}_${i}_10.dat.gz"; then echo "ERROR: gzip -d ${table_name}_${i}_10.dat.gz" && exit 1; fi
+                        done
+                    ) &
+                    wait
+                fi
+            done
+        )
+        # create table and load data
+        bash "${teamcity_build_checkoutDir}"/tools/tpcds-tools/bin/create-tpcds-tables.sh -s "${SF}"
+        rm -rf "${TPCDS_DATA_DIR_LINK}"
+        ln -s "${TPCDS_DATA_DIR}" "${TPCDS_DATA_DIR_LINK}"
+        bash "${teamcity_build_checkoutDir}"/tools/tpcds-tools/bin/load-tpcds-data.sh -c 10
+        if ! check_tpcds_table_rows "${db_name}" "${SF}"; then
+            exit 1
+        fi
+        echo "INFO: sleep 10min to wait compaction done"
+        if ${DEBUG:-false}; then sleep 10s; else sleep 10m; fi
+        data_reload="true"
+    fi
+
+    echo "#### 3. run tpcds-sf${SF} query"
+    set_session_variable runtime_filter_mode global
+    bash "${teamcity_build_checkoutDir}"/tools/tpcds-tools/bin/run-tpcds-queries.sh -s "${SF}" | tee "${teamcity_build_checkoutDir}"/run-tpcds-queries.log
+    if ! check_tpcds_result "${teamcity_build_checkoutDir}"/run-tpcds-queries.log; then exit 1; fi
+    line_end=$(sed -n '/^Total hot run time/=' "${teamcity_build_checkoutDir}"/run-tpcds-queries.log)
+    line_begin=$((line_end - 100))
+    comment_body="TPC-DS sf${SF} test result on commit ${commit_id:-}, data reload: ${data_reload:-"false"}
+
+run tpcds-sf${SF} query with default conf and session variables
+$(sed -n "${line_begin},${line_end}p" "${teamcity_build_checkoutDir}"/run-tpcds-queries.log)"
+
+    echo "#### 4. comment result on tpcds"
+    comment_body=$(echo "${comment_body}" | sed -e ':a;N;$!ba;s/\t/\\t/g;s/\n/\\n/g') # 将所有的 Tab字符替换为\t 换行符替换为\n
+    create_an_issue_comment_tpcds "${pull_request_num:-}" "${comment_body}"
+    rm -f result.csv
+)
+exit_flag="$?"
+
+echo "#### 5. check if need backup doris logs"
+if [[ ${exit_flag} != "0" ]]; then
+    stop_doris
+    print_doris_fe_log
+    print_doris_be_log
+    if file_name=$(archive_doris_logs "${pull_request_num}_${commit_id}_doris_logs.tar.gz"); then
+        upload_doris_log_to_oss "${file_name}"
+    fi
+fi
+
+exit "${exit_flag}"
diff --git a/regression-test/pipeline/performance/run-tpch.sh b/regression-test/pipeline/performance/run-tpch.sh
new file mode 100644
index 00000000000000..d3e408eef869fd
--- /dev/null
+++ b/regression-test/pipeline/performance/run-tpch.sh
@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Build Step: Command Line
+: <<EOF
+#!/bin/bash
+export DEBUG=true
+
+if [[ -f "${teamcity_build_checkoutDir:-}"/regression-test/pipeline/performance/run-tpch.sh ]]; then
+    cd "${teamcity_build_checkoutDir}"/regression-test/pipeline/performance/
+    bash -x run-tpch.sh
+else
+    echo "Build Step file missing: regression-test/pipeline/performance/run-tpch.sh" && exit 1
+fi
+EOF
+
+#####################################################################################
+## run-tpch.sh content ##
+
+# shellcheck source=/dev/null
+# check_tpch_table_rows, restart_doris, set_session_variable, check_tpch_result
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/doris-utils.sh
+# shellcheck source=/dev/null
+# create_an_issue_comment
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh
+# shellcheck source=/dev/null
+# upload_doris_log_to_oss
+source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/oss-utils.sh
+
+if ${DEBUG:-false}; then
+    pull_request_num="28431"
+    commit_id="5f5c4c80564c76ff4267fc4ce6a5408498ed1ab5"
+    SF="1"
+fi
+echo "#### Check env"
+if [[ -z "${teamcity_build_checkoutDir}" ||
+    -z "${pull_request_num}" ||
+    -z "${commit_id}" ]]; then
+    echo "ERROR: env teamcity_build_checkoutDir or pull_request_num or commit_id not set"
+    exit 1
+fi
+
+# shellcheck source=/dev/null
+source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
+if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
+
+echo "#### Run tpch test on Doris ####"
+DORIS_HOME="${teamcity_build_checkoutDir}/output"
+export DORIS_HOME
+cold_run_time_threshold=${cold_run_time_threshold:-200000} # ms
+hot_run_time_threshold=${hot_run_time_threshold:-50000}    # ms
+exit_flag=0
+
+(
+    set -e
+    shopt -s inherit_errexit
+
+    echo "#### 1. Restart doris"
+    if ! restart_doris; then echo "ERROR: Restart doris failed" && exit 1; fi
+
+    echo "#### 2. check if need to load data"
+    SF=${SF:-"100"}                                                                   # SCALE FACTOR
+    TPCH_DATA_DIR="/data/tpch/sf_${SF}"                                               # no / at the end
+    TPCH_DATA_DIR_LINK="${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/tpch-data # no / at the end
+    db_name="tpch_sf${SF}"
+    sed -i "s|^export DB=.*$|export DB='${db_name}'|g" \
+        "${teamcity_build_checkoutDir}"/tools/tpch-tools/conf/doris-cluster.conf
+    if ! check_tpch_table_rows "${db_name}" "${SF}"; then
+        echo "INFO: need to load tpch-sf${SF} data"
+        if ${force_load_data:-false}; then echo "INFO: force_load_data is true"; else echo "ERROR: force_load_data is false" && exit 1; fi
+        # prepare data
+        mkdir -p "${TPCH_DATA_DIR}"
+        (
+            cd "${TPCH_DATA_DIR}" || exit 1
+            declare -A table_file_count
+            table_file_count=(['region']=1 ['nation']=1 ['supplier']=1 ['customer']=1 ['part']=1 ['partsupp']=10 ['orders']=10 ['lineitem']=10)
+            for table_name in ${!table_file_count[*]}; do
+                if [[ ${table_file_count[${table_name}]} -eq 1 ]]; then
+                    url="https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/tpch/sf${SF}/${table_name}.tbl"
+                    if ! wget --continue -t3 -q "${url}"; then echo "ERROR: wget --continue ${url}" && exit 1; fi
+                elif [[ ${table_file_count[${table_name}]} -eq 10 ]]; then
+                    (
+                        for i in {1..10}; do
+                            url="https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/tpch/sf${SF}/${table_name}.tbl.${i}"
+                            if ! wget --continue -t3 -q "${url}"; then echo "ERROR: wget --continue ${url}" && exit 1; fi
+                        done
+                    ) &
+                    wait
+                fi
+            done
+        )
+        # create table and load data
+        bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/create-tpch-tables.sh -s "${SF}"
+        rm -rf "${TPCH_DATA_DIR_LINK}"
+        ln -s "${TPCH_DATA_DIR}" "${TPCH_DATA_DIR_LINK}"
+        bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/load-tpch-data.sh -c 2
+        if ! check_tpch_table_rows "${db_name}" "${SF}"; then
+            exit 1
+        fi
+        echo "INFO: sleep 10min to wait compaction done"
+        if ${DEBUG:-false}; then sleep 10s; else sleep 10m; fi
+        data_reload="true"
+    fi
+
+    echo "#### 3. run tpch-sf${SF} query"
+    set_session_variable runtime_filter_mode global
+    bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh -s "${SF}" | tee "${teamcity_build_checkoutDir}"/run-tpch-queries.log
+    if ! check_tpch_result "${teamcity_build_checkoutDir}"/run-tpch-queries.log; then exit 1; fi
+    line_end=$(sed -n '/^Total hot run time/=' "${teamcity_build_checkoutDir}"/run-tpch-queries.log)
+    line_begin=$((line_end - 23))
+    comment_body="Tpch sf${SF} test result on commit ${commit_id:-}, data reload: ${data_reload:-"false"}
+
+------ Round 1 ----------------------------------
+$(sed -n "${line_begin},${line_end}p" "${teamcity_build_checkoutDir}"/run-tpch-queries.log)"
+
+    echo "#### 4. run tpch-sf${SF} query with runtime_filter_mode=off"
+    set_session_variable runtime_filter_mode off
+    bash "${teamcity_build_checkoutDir}"/tools/tpch-tools/bin/run-tpch-queries.sh | tee "${teamcity_build_checkoutDir}"/run-tpch-queries.log
+    if ! grep '^Total hot run time' "${teamcity_build_checkoutDir}"/run-tpch-queries.log >/dev/null; then exit 1; fi
+    line_end=$(sed -n '/^Total hot run time/=' "${teamcity_build_checkoutDir}"/run-tpch-queries.log)
+    line_begin=$((line_end - 23))
+    comment_body="${comment_body}
+
+----- Round 2, with runtime_filter_mode=off -----
+$(sed -n "${line_begin},${line_end}p" "${teamcity_build_checkoutDir}"/run-tpch-queries.log)"
+
+    echo "#### 5. comment result on tpch"
+    comment_body=$(echo "${comment_body}" | sed -e ':a;N;$!ba;s/\t/\\t/g;s/\n/\\n/g') # 将所有的 Tab字符替换为\t 换行符替换为\n
+    create_an_issue_comment_tpch "${pull_request_num:-}" "${comment_body}"
+    rm -f result.csv
+)
+exit_flag="$?"
+
+echo "#### 5. check if need backup doris logs"
+if [[ ${exit_flag} != "0" ]]; then
+    stop_doris
+    print_doris_fe_log
+    print_doris_be_log
+    if file_name=$(archive_doris_logs "${pull_request_num}_${commit_id}_doris_logs.tar.gz"); then
+        upload_doris_log_to_oss "${file_name}"
+    fi
+fi
+
+exit "${exit_flag}"
diff --git a/tools/clickbench-tools/run-clickbench-queries.sh b/tools/clickbench-tools/run-clickbench-queries.sh
index 831879b7643e86..0d798e51a00374 100755
--- a/tools/clickbench-tools/run-clickbench-queries.sh
+++ b/tools/clickbench-tools/run-clickbench-queries.sh
@@ -96,19 +96,30 @@ echo "USER: $USER"
 echo "PASSWORD: $PASSWORD"
 echo "DB: $DB"
 
-pre_set() {
+run_sql() {
   echo $@
   mysql -h$FE_HOST -u$USER -P$FE_QUERY_PORT -D$DB -e "$@"
 }
 
-pre_set "set global parallel_fragment_exec_instance_num=8;"
-pre_set "set global exec_mem_limit=32G;"
-pre_set "set global query_timeout=900;"
+get_session_variable() {
+  k="$1"
+  v=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" -e"show variables like '${k}'\G" | grep " Value: ")
+  echo "${v/*Value: /}"
+}
+
+_parallel_fragment_exec_instance_num="$(get_session_variable parallel_fragment_exec_instance_num)"
+_exec_mem_limit="$(get_session_variable exec_mem_limit)"
+_query_timeout="$(get_session_variable query_timeout)"
 
 echo '============================================'
-pre_set "show variables"
+echo "Optimize session variables"
+run_sql "set global parallel_fragment_exec_instance_num=16;"
+run_sql "set global exec_mem_limit=32G;"
+run_sql "set global query_timeout=900;"
+echo '============================================'
+run_sql "show variables"
 echo '============================================'
-pre_set "analyze table hits with sync;"
+run_sql "analyze table hits with sync;"
 
 TRIES=3
 QUERY_NUM=1
@@ -122,7 +133,7 @@ cat ${QUERIES_FILE} | while read query; do
   sync
   echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
 
-  echo -n "query${QUERY_NUM}: " | tee -a result.csv
+  echo -n "query${QUERY_NUM}," | tee -a result.csv
   for i in $(seq 1 $TRIES); do
     RES=$(mysql -vvv -h$FE_HOST -u$USER -P$FE_QUERY_PORT -D$DB -e "${query}" | perl -nle 'print $1 if /\((\d+\.\d+)+ sec\)/' || :)
 
@@ -133,3 +144,14 @@ cat ${QUERIES_FILE} | while read query; do
 
   QUERY_NUM=$((QUERY_NUM + 1))
 done
+
+cold_run_sum=$(awk -F ',' '{sum+=$2} END {print sum}' result.csv)
+best_hot_run_sum=$(awk -F ',' '{if($3<$4){sum+=$3}else{sum+=$4}} END {print sum}' result.csv)
+echo "Total cold run time: ${cold_run_sum} ms"
+echo "Total hot run time: ${best_hot_run_sum} ms"
+echo 'Finish ClickBench queries.'
+
+echo "Restore session variables"
+run_sql "set global parallel_fragment_exec_instance_num=${_parallel_fragment_exec_instance_num};"
+run_sql "set global exec_mem_limit=${_exec_mem_limit};"
+run_sql "set global query_timeout=${_query_timeout};"
diff --git a/tools/tpcds-tools/bin/run-tpcds-queries.sh b/tools/tpcds-tools/bin/run-tpcds-queries.sh
index 9f6542da6d5fec..f3d8e58bffd00c 100755
--- a/tools/tpcds-tools/bin/run-tpcds-queries.sh
+++ b/tools/tpcds-tools/bin/run-tpcds-queries.sh
@@ -123,8 +123,24 @@ run_sql() {
     echo "$*"
     mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" -e "$*"
 }
+get_session_variable() {
+    k="$1"
+    v=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" -e"show variables like '${k}'\G" | grep " Value: ")
+    echo "${v/*Value: /}"
+}
+backup_session_variables_file="${CURDIR}/../conf/opt/backup_session_variables.sql"
+backup_session_variables() {
+    while IFS= read -r line; do
+        k="${line/set global /}"
+        k="${k%=*}"
+        v=$(get_session_variable "${k}")
+        echo "set global ${k}=${v};" >>"${backup_session_variables_file}"
+    done < <(grep -v '^ *#' <"${TPCDS_OPT_CONF}")
+}
+backup_session_variables
 
 echo '============================================'
+echo "Optimize session variables"
 run_sql "source ${TPCDS_OPT_CONF};"
 echo '============================================'
 run_sql "show variables;"
@@ -182,3 +198,7 @@ done
 echo "Total cold run time: ${cold_run_sum} ms"
 echo "Total hot run time: ${best_hot_run_sum} ms"
 echo 'Finish tpcds queries.'
+
+echo "Restore session variables"
+run_sql "source ${backup_session_variables_file};"
+rm -f "${backup_session_variables_file}"
diff --git a/tools/tpch-tools/bin/run-tpch-queries.sh b/tools/tpch-tools/bin/run-tpch-queries.sh
index d88062183cdf11..c2e5350a27bbb7 100755
--- a/tools/tpch-tools/bin/run-tpch-queries.sh
+++ b/tools/tpch-tools/bin/run-tpch-queries.sh
@@ -123,8 +123,24 @@ run_sql() {
     echo "$*"
     mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" -e "$*"
 }
+get_session_variable() {
+    k="$1"
+    v=$(mysql -h"${FE_HOST}" -u"${USER}" -P"${FE_QUERY_PORT}" -D"${DB}" -e"show variables like '${k}'\G" | grep " Value: ")
+    echo "${v/*Value: /}"
+}
+backup_session_variables_file="${CURDIR}/../conf/opt/backup_session_variables.sql"
+backup_session_variables() {
+    while IFS= read -r line; do
+        k="${line/set global /}"
+        k="${k%=*}"
+        v=$(get_session_variable "${k}")
+        echo "set global ${k}=${v};" >>"${backup_session_variables_file}"
+    done < <(grep -v '^ *#' <"${TPCH_OPT_CONF}")
+}
+backup_session_variables
 
 echo '============================================'
+echo "Optimize session variables"
 run_sql "source ${TPCH_OPT_CONF};"
 echo '============================================'
 run_sql "show variables;"
@@ -183,3 +199,7 @@ echo "Total cold run time: ${cold_run_sum} ms"
 # tpch 流水线依赖这个'Total hot run time'字符串
 echo "Total hot run time: ${best_hot_run_sum} ms"
 echo 'Finish tpch queries.'
+
+echo "Restore session variables"
+run_sql "source ${backup_session_variables_file};"
+rm -f "${backup_session_variables_file}"

From d2dc12bed5f78bc29da1fd4ba5be72fd2c3ef00b Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Fri, 29 Dec 2023 09:45:20 +0800
Subject: [PATCH 145/299] [fix](nereids)exists subquery should handle top level
 scarlar agg correctly (#29135)

---
 .../rules/analysis/SubqueryToApply.java       | 51 +++++++-----
 .../sub_query_correlated.out                  | 81 ++++++++++++++++++
 .../sub_query_correlated.groovy               | 83 +++++++++++++++++++
 3 files changed, 193 insertions(+), 22 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
index 5f55f86cc91257..675575ad298b38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SubqueryToApply.java
@@ -325,7 +325,10 @@ private LogicalPlan subqueryToApply(List<SubqueryExpr> subqueryExprs, LogicalPla
         LogicalPlan tmpPlan = childPlan;
         for (int i = 0; i < subqueryExprs.size(); ++i) {
             SubqueryExpr subqueryExpr = subqueryExprs.get(i);
-            if (nonMarkJoinExistsWithAgg(subqueryExpr, subqueryToMarkJoinSlot)) {
+            if (subqueryExpr instanceof Exists && hasTopLevelScalarAgg(subqueryExpr.getQueryPlan())) {
+                // because top level scalar agg always returns a value or null(for empty input)
+                // so Exists and Not Exists conjunct are always evaluated to True and false literals respectively
+                // we don't create apply node for it
                 continue;
             }
 
@@ -338,18 +341,11 @@ private LogicalPlan subqueryToApply(List<SubqueryExpr> subqueryExprs, LogicalPla
         return tmpPlan;
     }
 
-    private boolean nonMarkJoinExistsWithAgg(SubqueryExpr exists,
-                 Map<SubqueryExpr, Optional<MarkJoinSlotReference>> subqueryToMarkJoinSlot) {
-        return exists instanceof Exists
-                && !subqueryToMarkJoinSlot.get(exists).isPresent()
-                && hasTopLevelAggWithoutGroupBy(exists.getQueryPlan());
-    }
-
-    private static boolean hasTopLevelAggWithoutGroupBy(Plan plan) {
+    private static boolean hasTopLevelScalarAgg(Plan plan) {
         if (plan instanceof LogicalAggregate) {
             return ((LogicalAggregate) plan).getGroupByExpressions().isEmpty();
         } else if (plan instanceof LogicalProject || plan instanceof LogicalSort) {
-            return hasTopLevelAggWithoutGroupBy(plan.child(0));
+            return hasTopLevelScalarAgg(plan.child(0));
         }
         return false;
     }
@@ -427,19 +423,30 @@ public Expression visitExistsSubquery(Exists exists, SubqueryContext context) {
             // The result set when NULL is specified in the subquery and still evaluates to TRUE by using EXISTS
             // When the number of rows returned is empty, agg will return null, so if there is more agg,
             // it will always consider the returned result to be true
-            boolean needCreateMarkJoinSlot = isMarkJoin || shouldOutputMarkJoinSlot;
-            MarkJoinSlotReference markJoinSlotReference = null;
-            if (hasTopLevelAggWithoutGroupBy(exists.getQueryPlan()) && needCreateMarkJoinSlot) {
-                markJoinSlotReference =
-                        new MarkJoinSlotReference(statementContext.generateColumnName(), true);
-            } else if (needCreateMarkJoinSlot) {
-                markJoinSlotReference =
-                        new MarkJoinSlotReference(statementContext.generateColumnName());
-            }
-            if (needCreateMarkJoinSlot) {
-                context.setSubqueryToMarkJoinSlot(exists, Optional.of(markJoinSlotReference));
+            if (hasTopLevelScalarAgg(exists.getQueryPlan())) {
+                /*
+                top level scalar agg and always return a value or null for empty input
+                so Exists and Not Exists conjunct are always evaluated to True and False literals respectively
+                    SELECT *
+                    FROM t1
+                    WHERE EXISTS (
+                            SELECT SUM(a)
+                            FROM t2
+                            WHERE t1.a = t2.b and t1.a = 1;
+                        );
+                 */
+                return exists.isNot() ? BooleanLiteral.FALSE : BooleanLiteral.TRUE;
+            } else {
+                boolean needCreateMarkJoinSlot = isMarkJoin || shouldOutputMarkJoinSlot;
+                if (needCreateMarkJoinSlot) {
+                    MarkJoinSlotReference markJoinSlotReference =
+                            new MarkJoinSlotReference(statementContext.generateColumnName());
+                    context.setSubqueryToMarkJoinSlot(exists, Optional.of(markJoinSlotReference));
+                    return markJoinSlotReference;
+                } else {
+                    return BooleanLiteral.TRUE;
+                }
             }
-            return needCreateMarkJoinSlot ? markJoinSlotReference : BooleanLiteral.TRUE;
         }
 
         @Override
diff --git a/regression-test/data/nereids_syntax_p0/sub_query_correlated.out b/regression-test/data/nereids_syntax_p0/sub_query_correlated.out
index 0564d1884d9c05..e7758d02a5f660 100644
--- a/regression-test/data/nereids_syntax_p0/sub_query_correlated.out
+++ b/regression-test/data/nereids_syntax_p0/sub_query_correlated.out
@@ -489,3 +489,84 @@ true
 -- !doris_7643 --
 3	3
 
+-- !select_exists1 --
+\N	\N	0
+\N	\N	100
+
+-- !select_exists2 --
+
+-- !select_exists3 --
+\N	\N	0
+\N	\N	100
+
+-- !select_exists4 --
+\N	\N	100
+
+-- !select_exists5 --
+true
+true
+
+-- !select_exists6 --
+false
+false
+
+-- !select_exists7 --
+true
+true
+
+-- !select_exists8 --
+false
+true
+
+-- !select_exists9 --
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+
+-- !select_exists10 --
+
+-- !select_exists11 --
+\N	\N	0
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+\N	\N	100
+
+-- !select_exists12 --
+\N	\N	0
+
diff --git a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
index ac87bbc813fe3e..09ba6c31becc8c 100644
--- a/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
+++ b/regression-test/suites/nereids_syntax_p0/sub_query_correlated.groovy
@@ -679,4 +679,87 @@ suite ("sub_query_correlated") {
     // order_qt_doris_6937_2 """
     //     select * from sub_query_correlated_subquery1 where sub_query_correlated_subquery1.k1 not in (select sub_query_correlated_subquery3.k3 from sub_query_correlated_subquery3 where sub_query_correlated_subquery3.v2 > sub_query_correlated_subquery1.k2) or k1 < 10 order by k1, k2;
     // """
+    sql """drop table if exists table_21_undef_partitions2_keys3;"""
+    sql """drop table if exists table_1_undef_partitions2_keys3;"""
+    sql """create table table_21_undef_partitions2_keys3 (
+            `col_int_undef_signed` int   ,
+            `col_varchar_10__undef_signed` varchar(10)   ,
+            `pk` int
+            ) engine=olap
+            distributed by hash(pk) buckets 10
+            properties('replication_num' = '1');"""
+    sql """create table table_1_undef_partitions2_keys3 (
+            `col_int_undef_signed` int   ,
+            `col_varchar_10__undef_signed` varchar(10)   ,
+            `pk` int
+            ) engine=olap
+            distributed by hash(pk) buckets 10
+            properties('replication_num' = '1');"""
+    sql """insert into table_21_undef_partitions2_keys3(pk,col_int_undef_signed,col_varchar_10__undef_signed) values (0,null,null),(1,6,'c'),(2,7,'m'),(3,null,null),(4,null,'b'),(5,4,null),(6,3,null),(7,0,'z'),(8,null,"me"),(9,6,null),(10,0,""),(11,null,'e'),(12,null,"up"),(13,null,""),(14,7,'s'),(15,null,""),(16,3,null),(17,null,""),(18,null,""),(19,4,""),(20,7,null);"""
+    sql """insert into table_1_undef_partitions2_keys3(pk,col_int_undef_signed,col_varchar_10__undef_signed) values (0,null,null),(100,null,null);"""
+
+    qt_select_exists1 """SELECT *
+                            FROM table_1_undef_partitions2_keys3 AS t1
+                            WHERE EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9
+                                ) order by t1.pk;"""
+    qt_select_exists2 """SELECT *
+                            FROM table_1_undef_partitions2_keys3 AS t1
+                            WHERE not EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9
+                                ) order by t1.pk;"""
+    qt_select_exists3 """SELECT *
+                            FROM table_1_undef_partitions2_keys3 AS t1
+                            WHERE EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = 9
+                                ) or t1.pk = 100 order by t1.pk;"""
+    qt_select_exists4 """SELECT *
+                            FROM table_1_undef_partitions2_keys3 AS t1
+                            WHERE not EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = 9
+                                ) or t1.pk = 100 order by t1.pk;"""
+    qt_select_exists5 """select EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9
+                                ) from table_1_undef_partitions2_keys3 AS t1;"""
+    qt_select_exists6 """select not EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9
+                                ) from table_1_undef_partitions2_keys3 AS t1 order by t1.pk;"""
+    qt_select_exists7 """select EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9
+                                ) or t1.pk = 100 from table_1_undef_partitions2_keys3 AS t1 order by t1.pk;"""
+    qt_select_exists8 """select EXISTS (
+                                    SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9
+                                ) and t1.pk = 100 from table_1_undef_partitions2_keys3 AS t1 order by t1.pk;"""
+    qt_select_exists9 """select t1.* from table_1_undef_partitions2_keys3 AS t1 join table_21_undef_partitions2_keys3 AS t2 
+                                on t1.pk = t2.pk and not exists ( SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9 ) or t1.pk = 100 order by t1.pk;"""
+    qt_select_exists10 """select t1.* from table_1_undef_partitions2_keys3 AS t1 join table_21_undef_partitions2_keys3 AS t2 
+                                on t1.pk = t2.pk and not exists ( SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9 ) or t1.pk > 100 order by t1.pk;"""
+    qt_select_exists11 """select t1.* from table_1_undef_partitions2_keys3 AS t1  join table_21_undef_partitions2_keys3 AS t2 
+                                on t1.pk = t2.pk and exists ( SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9 ) or t1.pk = 100 order by t1.pk;"""
+    qt_select_exists12 """select t1.* from table_1_undef_partitions2_keys3 AS t1  join table_21_undef_partitions2_keys3 AS t2 
+                                on t1.pk = t2.pk and exists ( SELECT SUM(`pk`)
+                                    FROM table_21_undef_partitions2_keys3 AS t2
+                                    WHERE t1.pk = t2.pk and t1.pk = 9 ) or t1.pk > 100 order by t1.pk;"""                                                                                                                                            
 }

From a525d5c5a3bd13e651f47d4ae066f10f8195f8fc Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Fri, 29 Dec 2023 10:11:44 +0800
Subject: [PATCH 146/299] [refactor](decimal) change type name Decimal128 to
 Decimal128V2, Decimal128I to Decimal128V3 to avoid confusion (#29265)

change type name Decimal128 to Decimal128V2, Decimal128I to Decimal128V3 to avoid confusion
---
 be/src/exec/schema_scanner.cpp                |  4 +-
 be/src/runtime/primitive_type.h               |  6 +-
 be/src/runtime/runtime_predicate.h            |  8 +-
 be/src/runtime/type_limit.h                   | 12 +--
 be/src/util/arrow/block_convertor.cpp         |  4 +-
 .../aggregate_function_avg.h                  |  8 +-
 .../aggregate_function_histogram.h            |  2 +-
 .../aggregate_function_product.h              | 28 +++---
 .../aggregate_function_stddev.h               |  6 +-
 .../aggregate_function_uniq.cpp               |  8 +-
 be/src/vec/aggregate_functions/helpers.h      |  4 +-
 be/src/vec/columns/column_array.h             |  8 +-
 be/src/vec/columns/column_decimal.cpp         |  8 +-
 be/src/vec/columns/columns_number.h           |  4 +-
 be/src/vec/common/field_visitors.h            |  8 +-
 be/src/vec/core/call_on_type_index.h          | 32 +++----
 be/src/vec/core/field.cpp                     | 18 ++--
 be/src/vec/core/field.h                       | 96 +++++++++----------
 be/src/vec/core/types.h                       | 70 +++++++-------
 .../vec/data_types/convert_field_to_type.cpp  |  8 +-
 be/src/vec/data_types/data_type.cpp           |  4 +-
 be/src/vec/data_types/data_type.h             |  8 +-
 be/src/vec/data_types/data_type_decimal.cpp   | 22 ++---
 be/src/vec/data_types/data_type_decimal.h     | 34 +++----
 be/src/vec/data_types/data_type_factory.cpp   | 26 ++---
 be/src/vec/data_types/get_least_supertype.cpp | 14 +--
 .../serde/data_type_decimal_serde.cpp         | 19 ++--
 .../serde/data_type_decimal_serde.h           | 20 ++--
 be/src/vec/exec/format/format_common.h        |  4 +-
 be/src/vec/exec/format/orc/vorc_reader.cpp    | 12 +--
 .../format/parquet/parquet_column_convert.h   | 14 +--
 .../exec/format/parquet/parquet_pred_cmp.h    | 20 ++--
 be/src/vec/exec/jni_connector.cpp             | 38 ++++----
 .../array/function_array_aggregation.cpp      |  4 +-
 .../functions/array/function_array_apply.cpp  | 84 ++++++++--------
 .../array/function_array_cum_sum.cpp          | 30 +++---
 .../array/function_array_difference.h         | 12 +--
 .../functions/array/function_array_distinct.h | 12 +--
 .../functions/array/function_array_element.h  | 12 +--
 .../array/function_array_enumerate_uniq.cpp   |  8 +-
 .../functions/array/function_array_index.h    |  8 +-
 .../vec/functions/array/function_array_join.h | 15 +--
 .../functions/array/function_array_remove.h   | 12 +--
 .../functions/array/function_arrays_overlap.h | 16 ++--
 be/src/vec/functions/function.h               | 10 +-
 .../functions/function_binary_arithmetic.h    | 10 +-
 be/src/vec/functions/function_cast.h          | 30 +++---
 be/src/vec/functions/function_string.h        |  8 +-
 .../vec/functions/function_unary_arithmetic.h |  2 +-
 .../vec/functions/function_width_bucket.cpp   | 12 +--
 be/src/vec/functions/least_greast.cpp         |  8 +-
 be/src/vec/functions/multiply.cpp             |  6 +-
 be/src/vec/io/io_helper.h                     |  2 +-
 be/src/vec/olap/olap_data_convertor.cpp       |  8 +-
 be/src/vec/sink/vmysql_result_writer.cpp      |  8 +-
 be/src/vec/sink/vtablet_block_convertor.cpp   |  8 +-
 .../vec/sink/writer/vmysql_table_writer.cpp   |  2 +-
 .../segment_v2/column_reader_writer_test.cpp  |  2 +-
 .../aggregate_functions/agg_collect_test.cpp  |  8 +-
 .../agg_histogram_test.cpp                    |  4 +-
 .../aggregate_functions/agg_min_max_test.cpp  |  8 +-
 .../aggregate_functions/agg_replace_test.cpp  | 20 ++--
 be/test/vec/columns/column_hash_func_test.cpp |  2 +-
 be/test/vec/columns/column_resize_test.cpp    |  2 +-
 be/test/vec/exprs/vexpr_test.cpp              |  2 +-
 .../function/function_array_element_test.cpp  |  8 +-
 .../function/function_array_index_test.cpp    | 10 +-
 .../function/function_arrays_overlap_test.cpp |  6 +-
 be/test/vec/function/function_math_test.cpp   |  2 +-
 be/test/vec/function/function_test_util.cpp   |  6 +-
 be/test/vec/function/function_test_util.h     |  8 +-
 be/test/vec/function/table_function_test.cpp  |  4 +-
 .../arrow_column_to_doris_column_test.cpp     |  2 +-
 73 files changed, 501 insertions(+), 497 deletions(-)

diff --git a/be/src/exec/schema_scanner.cpp b/be/src/exec/schema_scanner.cpp
index 8d640341bfb9a6..a905f69336de27 100644
--- a/be/src/exec/schema_scanner.cpp
+++ b/be/src/exec/schema_scanner.cpp
@@ -275,13 +275,13 @@ Status SchemaScanner::fill_dest_column_for_range(vectorized::Block* block, size_
 
         case TYPE_DECIMALV2: {
             const vectorized::Int128 num = (reinterpret_cast<PackedInt128*>(data))->value;
-            reinterpret_cast<vectorized::ColumnDecimal128*>(col_ptr)->insert_data(
+            reinterpret_cast<vectorized::ColumnDecimal128V2*>(col_ptr)->insert_data(
                     reinterpret_cast<const char*>(&num), 0);
             break;
         }
         case TYPE_DECIMAL128I: {
             const vectorized::Int128 num = (reinterpret_cast<PackedInt128*>(data))->value;
-            reinterpret_cast<vectorized::ColumnDecimal128I*>(col_ptr)->insert_data(
+            reinterpret_cast<vectorized::ColumnDecimal128V3*>(col_ptr)->insert_data(
                     reinterpret_cast<const char*>(&num), 0);
             break;
         }
diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h
index 4c36ac21bff296..6040f73388b465 100644
--- a/be/src/runtime/primitive_type.h
+++ b/be/src/runtime/primitive_type.h
@@ -208,7 +208,7 @@ struct PrimitiveTypeTraits<TYPE_DECIMALV2> {
     using CppType = DecimalV2Value;
     /// Different with compute layer, the DecimalV1 was stored as decimal12_t(12 bytes).
     using StorageFieldType = decimal12_t;
-    using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal128>;
+    using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal128V2>;
 };
 template <>
 struct PrimitiveTypeTraits<TYPE_DECIMAL32> {
@@ -224,9 +224,9 @@ struct PrimitiveTypeTraits<TYPE_DECIMAL64> {
 };
 template <>
 struct PrimitiveTypeTraits<TYPE_DECIMAL128I> {
-    using CppType = vectorized::Decimal128I;
+    using CppType = vectorized::Decimal128V3;
     using StorageFieldType = vectorized::Int128;
-    using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal128I>;
+    using ColumnType = vectorized::ColumnDecimal<vectorized::Decimal128V3>;
 };
 template <>
 struct PrimitiveTypeTraits<TYPE_DECIMAL256> {
diff --git a/be/src/runtime/runtime_predicate.h b/be/src/runtime/runtime_predicate.h
index 9e3bce17d6e744..80742fa66dcedf 100644
--- a/be/src/runtime/runtime_predicate.h
+++ b/be/src/runtime/runtime_predicate.h
@@ -148,9 +148,9 @@ class RuntimePredicate {
 
     static std::string get_decimalv2_value(const Field& field) {
         // can NOT use PrimitiveTypeTraits<TYPE_DECIMALV2>::CppType since
-        //   it is DecimalV2Value and Decimal128 can not convert to it implicitly
-        using ValueType = Decimal128::NativeType;
-        auto v = field.get<DecimalField<Decimal128>>();
+        //   it is DecimalV2Value and Decimal128V2 can not convert to it implicitly
+        using ValueType = Decimal128V2::NativeType;
+        auto v = field.get<DecimalField<Decimal128V2>>();
         // use TYPE_DECIMAL128I instead of TYPE_DECIMALV2 since v.get_scale()
         //   is always 9 for DECIMALV2
         return cast_to_string<TYPE_DECIMAL128I, ValueType>(v.get_value(), v.get_scale());
@@ -170,7 +170,7 @@ class RuntimePredicate {
 
     static std::string get_decimal128_value(const Field& field) {
         using ValueType = typename PrimitiveTypeTraits<TYPE_DECIMAL128I>::CppType;
-        auto v = field.get<DecimalField<Decimal128I>>();
+        auto v = field.get<DecimalField<Decimal128V3>>();
         return cast_to_string<TYPE_DECIMAL128I, ValueType>(v.get_value(), v.get_scale());
     }
 
diff --git a/be/src/runtime/type_limit.h b/be/src/runtime/type_limit.h
index be3b2186d12024..f1f08caf43f3a9 100644
--- a/be/src/runtime/type_limit.h
+++ b/be/src/runtime/type_limit.h
@@ -54,18 +54,18 @@ struct type_limit<vectorized::Decimal64> {
 };
 
 template <>
-struct type_limit<vectorized::Decimal128I> {
-    static vectorized::Decimal128I max() {
+struct type_limit<vectorized::Decimal128V3> {
+    static vectorized::Decimal128V3 max() {
         return (static_cast<int128_t>(999999999999999999ll) * 100000000000000000ll * 1000ll +
                 static_cast<int128_t>(99999999999999999ll) * 1000ll + 999ll);
     }
-    static vectorized::Decimal128I min() { return -max(); }
+    static vectorized::Decimal128V3 min() { return -max(); }
 };
 
 template <>
-struct type_limit<vectorized::Decimal128> {
-    static vectorized::Decimal128 max() { return DecimalV2Value::get_max_decimal().value(); }
-    static vectorized::Decimal128 min() { return -max(); }
+struct type_limit<vectorized::Decimal128V2> {
+    static vectorized::Decimal128V2 max() { return DecimalV2Value::get_max_decimal().value(); }
+    static vectorized::Decimal128V2 min() { return -max(); }
 };
 static wide::Int256 MAX_DECIMAL256_INT({18446744073709551615ul, 8607968719199866879ul,
                                         532749306367912313ul, 1593091911132452277ul});
diff --git a/be/src/util/arrow/block_convertor.cpp b/be/src/util/arrow/block_convertor.cpp
index 2c605c9c96869b..009306d425e06a 100644
--- a/be/src/util/arrow/block_convertor.cpp
+++ b/be/src/util/arrow/block_convertor.cpp
@@ -178,7 +178,7 @@ class FromBlockConverter : public arrow::TypeVisitor {
         size_t start = _cur_start;
         size_t num_rows = _cur_rows;
         if (auto* decimalv2_column = vectorized::check_and_get_column<
-                    vectorized::ColumnDecimal<vectorized::Decimal128>>(
+                    vectorized::ColumnDecimal<vectorized::Decimal128V2>>(
                     *vectorized::remove_nullable(_cur_col))) {
             std::shared_ptr<arrow::DataType> s_decimal_ptr =
                     std::make_shared<arrow::Decimal128Type>(27, 9);
@@ -198,7 +198,7 @@ class FromBlockConverter : public arrow::TypeVisitor {
             }
             return arrow::Status::OK();
         } else if (auto* decimal128_column = vectorized::check_and_get_column<
-                           vectorized::ColumnDecimal<vectorized::Decimal128I>>(
+                           vectorized::ColumnDecimal<vectorized::Decimal128V3>>(
                            *vectorized::remove_nullable(_cur_col))) {
             std::shared_ptr<arrow::DataType> s_decimal_ptr =
                     std::make_shared<arrow::Decimal128Type>(38, decimal128_column->get_scale());
diff --git a/be/src/vec/aggregate_functions/aggregate_function_avg.h b/be/src/vec/aggregate_functions/aggregate_function_avg.h
index b2d95db679826f..61eb04bb13bb44 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_avg.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_avg.h
@@ -85,7 +85,7 @@ struct AggregateFunctionAvgData {
             DecimalV2Value decimal_val_count(count, 0);
             DecimalV2Value decimal_val_sum(sum);
             DecimalV2Value cal_ret = decimal_val_sum / decimal_val_count;
-            Decimal128 ret(cal_ret.value());
+            Decimal128V2 ret(cal_ret.value());
             return ret;
         } else {
             if constexpr (IsDecimal256<T>) {
@@ -113,15 +113,15 @@ class AggregateFunctionAvg final
         : public IAggregateFunctionDataHelper<Data, AggregateFunctionAvg<T, Data>> {
 public:
     using ResultType = std::conditional_t<
-            IsDecimalV2<T>, Decimal128,
+            IsDecimalV2<T>, Decimal128V2,
             std::conditional_t<IsDecimalNumber<T>, typename Data::ResultType, Float64>>;
     using ResultDataType = std::conditional_t<
-            IsDecimalV2<T>, DataTypeDecimal<Decimal128>,
+            IsDecimalV2<T>, DataTypeDecimal<Decimal128V2>,
             std::conditional_t<IsDecimalNumber<T>, DataTypeDecimal<typename Data::ResultType>,
                                DataTypeNumber<Float64>>>;
     using ColVecType = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<T>, ColumnVector<T>>;
     using ColVecResult = std::conditional_t<
-            IsDecimalV2<T>, ColumnDecimal<Decimal128>,
+            IsDecimalV2<T>, ColumnDecimal<Decimal128V2>,
             std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<typename Data::ResultType>,
                                ColumnVector<Float64>>>;
 
diff --git a/be/src/vec/aggregate_functions/aggregate_function_histogram.h b/be/src/vec/aggregate_functions/aggregate_function_histogram.h
index a1a8a346e9c405..295a063bc3044d 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_histogram.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_histogram.h
@@ -56,7 +56,7 @@ namespace doris::vectorized {
 template <typename T>
 struct AggregateFunctionHistogramData {
     using ColVecType =
-            std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<Decimal128>, ColumnVector<T>>;
+            std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<Decimal128V2>, ColumnVector<T>>;
 
     void set_parameters(int input_max_num_buckets) {
         if (input_max_num_buckets > 0) {
diff --git a/be/src/vec/aggregate_functions/aggregate_function_product.h b/be/src/vec/aggregate_functions/aggregate_function_product.h
index 8a13ad0b034027..4b0365a1b6d12b 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_product.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_product.h
@@ -51,43 +51,43 @@ struct AggregateFunctionProductData {
 };
 
 template <>
-struct AggregateFunctionProductData<Decimal128> {
-    Decimal128 product {};
+struct AggregateFunctionProductData<Decimal128V2> {
+    Decimal128V2 product {};
 
-    void add(Decimal128 value, Decimal128) {
+    void add(Decimal128V2 value, Decimal128V2) {
         DecimalV2Value decimal_product(product);
         DecimalV2Value decimal_value(value);
         DecimalV2Value ret = decimal_product * decimal_value;
-        memcpy(&product, &ret, sizeof(Decimal128));
+        memcpy(&product, &ret, sizeof(Decimal128V2));
     }
 
-    void merge(const AggregateFunctionProductData& other, Decimal128) {
+    void merge(const AggregateFunctionProductData& other, Decimal128V2) {
         DecimalV2Value decimal_product(product);
         DecimalV2Value decimal_value(other.product);
         DecimalV2Value ret = decimal_product * decimal_value;
-        memcpy(&product, &ret, sizeof(Decimal128));
+        memcpy(&product, &ret, sizeof(Decimal128V2));
     }
 
     void write(BufferWritable& buffer) const { write_binary(product, buffer); }
 
     void read(BufferReadable& buffer) { read_binary(product, buffer); }
 
-    Decimal128 get() const { return product; }
+    Decimal128V2 get() const { return product; }
 
-    void reset(Decimal128 value) { product = std::move(value); }
+    void reset(Decimal128V2 value) { product = std::move(value); }
 };
 
 template <>
-struct AggregateFunctionProductData<Decimal128I> {
-    Decimal128I product {};
+struct AggregateFunctionProductData<Decimal128V3> {
+    Decimal128V3 product {};
 
     template <typename NestedType>
-    void add(Decimal<NestedType> value, Decimal128I multiplier) {
+    void add(Decimal<NestedType> value, Decimal128V3 multiplier) {
         product *= value;
         product /= multiplier;
     }
 
-    void merge(const AggregateFunctionProductData& other, Decimal128I multiplier) {
+    void merge(const AggregateFunctionProductData& other, Decimal128V3 multiplier) {
         product *= other.product;
         product /= multiplier;
     }
@@ -96,9 +96,9 @@ struct AggregateFunctionProductData<Decimal128I> {
 
     void read(BufferReadable& buffer) { read_binary(product, buffer); }
 
-    Decimal128 get() const { return product; }
+    Decimal128V2 get() const { return product; }
 
-    void reset(Decimal128 value) { product = value; }
+    void reset(Decimal128V2 value) { product = value; }
 };
 
 template <typename T, typename TResult, typename Data>
diff --git a/be/src/vec/aggregate_functions/aggregate_function_stddev.h b/be/src/vec/aggregate_functions/aggregate_function_stddev.h
index 9578b37fbca855..c84e67a7d6d442 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_stddev.h
+++ b/be/src/vec/aggregate_functions/aggregate_function_stddev.h
@@ -214,7 +214,7 @@ struct BaseDatadecimal {
     }
 
     static DataTypePtr get_return_type() {
-        return std::make_shared<DataTypeDecimal<Decimal128>>(27, 9);
+        return std::make_shared<DataTypeDecimal<Decimal128V2>>(27, 9);
     }
 
     DecimalV2Value mean;
@@ -224,7 +224,7 @@ struct BaseDatadecimal {
 
 template <typename T, typename Data>
 struct PopData : Data {
-    using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<Decimal128>,
+    using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<Decimal128V2>,
                                             ColumnVector<Float64>>;
     void insert_result_into(IColumn& to) const {
         auto& col = assert_cast<ColVecResult&>(to);
@@ -258,7 +258,7 @@ struct StddevSampName : Data {
 
 template <typename T, typename Data>
 struct SampData : Data {
-    using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<Decimal128>,
+    using ColVecResult = std::conditional_t<IsDecimalNumber<T>, ColumnDecimal<Decimal128V2>,
                                             ColumnVector<Float64>>;
     void insert_result_into(IColumn& to) const {
         ColumnNullable& nullable_column = assert_cast<ColumnNullable&>(to);
diff --git a/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
index 97b0fba623cc91..735b8a737c24ec 100644
--- a/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
+++ b/be/src/vec/aggregate_functions/aggregate_function_uniq.cpp
@@ -48,11 +48,11 @@ AggregateFunctionPtr create_aggregate_function_uniq(const std::string& name,
         } else if (which.is_decimal64()) {
             return creator_without_type::create<AggregateFunctionUniq<Decimal64, Data<Int64>>>(
                     argument_types, result_is_nullable);
-        } else if (which.is_decimal128i()) {
-            return creator_without_type::create<AggregateFunctionUniq<Decimal128I, Data<Int128>>>(
+        } else if (which.is_decimal128v3()) {
+            return creator_without_type::create<AggregateFunctionUniq<Decimal128V3, Data<Int128>>>(
                     argument_types, result_is_nullable);
-        } else if (which.is_decimal128() || which.is_decimal128i()) {
-            return creator_without_type::create<AggregateFunctionUniq<Decimal128, Data<Int128>>>(
+        } else if (which.is_decimal128v2() || which.is_decimal128v3()) {
+            return creator_without_type::create<AggregateFunctionUniq<Decimal128V2, Data<Int128>>>(
                     argument_types, result_is_nullable);
         } else if (which.is_string_or_fixed_string()) {
             return creator_without_type::create<AggregateFunctionUniq<String, Data<String>>>(
diff --git a/be/src/vec/aggregate_functions/helpers.h b/be/src/vec/aggregate_functions/helpers.h
index 63b814e3e6f0f2..9bea4e745368bb 100644
--- a/be/src/vec/aggregate_functions/helpers.h
+++ b/be/src/vec/aggregate_functions/helpers.h
@@ -44,8 +44,8 @@
 #define FOR_DECIMAL_TYPES(M) \
     M(Decimal32)             \
     M(Decimal64)             \
-    M(Decimal128)            \
-    M(Decimal128I)           \
+    M(Decimal128V2)          \
+    M(Decimal128V3)          \
     M(Decimal256)
 
 /** If the serialized type is not the default type(string),
diff --git a/be/src/vec/columns/column_array.h b/be/src/vec/columns/column_array.h
index 222e8a911c470f..ad8edbf75657f0 100644
--- a/be/src/vec/columns/column_array.h
+++ b/be/src/vec/columns/column_array.h
@@ -51,10 +51,10 @@ class Arena;
 } // namespace doris
 
 //TODO: use marcos below to decouple array function calls
-#define ALL_COLUMNS_NUMBER                                                                        \
-    ColumnUInt8, ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64, ColumnInt128, ColumnFloat32,  \
-            ColumnFloat64, ColumnDecimal32, ColumnDecimal64, ColumnDecimal128I, ColumnDecimal128, \
-            ColumnDecimal256
+#define ALL_COLUMNS_NUMBER                                                                       \
+    ColumnUInt8, ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64, ColumnInt128, ColumnFloat32, \
+            ColumnFloat64, ColumnDecimal32, ColumnDecimal64, ColumnDecimal128V3,                 \
+            ColumnDecimal128V2, ColumnDecimal256
 #define ALL_COLUMNS_TIME ColumnDate, ColumnDateTime, ColumnDateV2, ColumnDateTimeV2
 #define ALL_COLUMNS_NUMERIC ALL_COLUMNS_NUMBER, ALL_COLUMNS_TIME
 #define ALL_COLUMNS_SIMPLE ALL_COLUMNS_NUMERIC, ColumnString
diff --git a/be/src/vec/columns/column_decimal.cpp b/be/src/vec/columns/column_decimal.cpp
index 1158e5b0d639f9..ed76ce6a113308 100644
--- a/be/src/vec/columns/column_decimal.cpp
+++ b/be/src/vec/columns/column_decimal.cpp
@@ -494,12 +494,12 @@ Decimal64 ColumnDecimal<Decimal64>::get_scale_multiplier() const {
 }
 
 template <>
-Decimal128 ColumnDecimal<Decimal128>::get_scale_multiplier() const {
+Decimal128V2 ColumnDecimal<Decimal128V2>::get_scale_multiplier() const {
     return common::exp10_i128(scale);
 }
 
 template <>
-Decimal128I ColumnDecimal<Decimal128I>::get_scale_multiplier() const {
+Decimal128V3 ColumnDecimal<Decimal128V3>::get_scale_multiplier() const {
     return common::exp10_i128(scale);
 }
 
@@ -529,7 +529,7 @@ void ColumnDecimal<T>::replace_column_null_data(const uint8_t* __restrict null_m
 
 template class ColumnDecimal<Decimal32>;
 template class ColumnDecimal<Decimal64>;
-template class ColumnDecimal<Decimal128>;
-template class ColumnDecimal<Decimal128I>;
+template class ColumnDecimal<Decimal128V2>;
+template class ColumnDecimal<Decimal128V3>;
 template class ColumnDecimal<Decimal256>;
 } // namespace doris::vectorized
diff --git a/be/src/vec/columns/columns_number.h b/be/src/vec/columns/columns_number.h
index 926db2d47cfb7f..33dbe4c8c63ba1 100644
--- a/be/src/vec/columns/columns_number.h
+++ b/be/src/vec/columns/columns_number.h
@@ -53,8 +53,8 @@ using ColumnFloat64 = ColumnVector<Float64>;
 
 using ColumnDecimal32 = ColumnDecimal<Decimal32>;
 using ColumnDecimal64 = ColumnDecimal<Decimal64>;
-using ColumnDecimal128 = ColumnDecimal<Decimal128>;
-using ColumnDecimal128I = ColumnDecimal<Decimal128I>;
+using ColumnDecimal128V2 = ColumnDecimal<Decimal128V2>;
+using ColumnDecimal128V3 = ColumnDecimal<Decimal128V3>;
 using ColumnDecimal256 = ColumnDecimal<Decimal256>;
 
 using ColumnIPv4 = ColumnVector<IPv4>;
diff --git a/be/src/vec/common/field_visitors.h b/be/src/vec/common/field_visitors.h
index 2066a94960a3d5..9e41be40ac3413 100644
--- a/be/src/vec/common/field_visitors.h
+++ b/be/src/vec/common/field_visitors.h
@@ -59,10 +59,10 @@ typename std::decay_t<Visitor>::ResultType apply_visitor(Visitor&& visitor, F&&
         return visitor(field.template get<DecimalField<Decimal32>>());
     case Field::Types::Decimal64:
         return visitor(field.template get<DecimalField<Decimal64>>());
-    case Field::Types::Decimal128:
-        return visitor(field.template get<DecimalField<Decimal128>>());
-    case Field::Types::Decimal128I:
-        return visitor(field.template get<DecimalField<Decimal128I>>());
+    case Field::Types::Decimal128V2:
+        return visitor(field.template get<DecimalField<Decimal128V2>>());
+    case Field::Types::Decimal128V3:
+        return visitor(field.template get<DecimalField<Decimal128V3>>());
     case Field::Types::Decimal256:
         return visitor(field.template get<DecimalField<Decimal256>>());
     case Field::Types::JSONB:
diff --git a/be/src/vec/core/call_on_type_index.h b/be/src/vec/core/call_on_type_index.h
index 30ad31f3560286..b651fe2ec47502 100644
--- a/be/src/vec/core/call_on_type_index.h
+++ b/be/src/vec/core/call_on_type_index.h
@@ -68,10 +68,10 @@ bool call_on_basic_type(TypeIndex number, F&& f) {
             return f(TypePair<T, Decimal32>());
         case TypeIndex::Decimal64:
             return f(TypePair<T, Decimal64>());
-        case TypeIndex::Decimal128:
-            return f(TypePair<T, Decimal128>());
-        case TypeIndex::Decimal128I:
-            return f(TypePair<T, Decimal128I>());
+        case TypeIndex::Decimal128V2:
+            return f(TypePair<T, Decimal128V2>());
+        case TypeIndex::Decimal128V3:
+            return f(TypePair<T, Decimal128V3>());
         case TypeIndex::Decimal256:
             return f(TypePair<T, Decimal256>());
         default:
@@ -139,11 +139,11 @@ bool call_on_basic_types(TypeIndex type_num1, TypeIndex type_num2, F&& f) {
         case TypeIndex::Decimal64:
             return call_on_basic_type<Decimal64, _int, _float, _decimal, _datetime>(
                     type_num2, std::forward<F>(f));
-        case TypeIndex::Decimal128:
-            return call_on_basic_type<Decimal128, _int, _float, _decimal, _datetime>(
+        case TypeIndex::Decimal128V2:
+            return call_on_basic_type<Decimal128V2, _int, _float, _decimal, _datetime>(
                     type_num2, std::forward<F>(f));
-        case TypeIndex::Decimal128I:
-            return call_on_basic_type<Decimal128I, _int, _float, _decimal, _datetime>(
+        case TypeIndex::Decimal128V3:
+            return call_on_basic_type<Decimal128V3, _int, _float, _decimal, _datetime>(
                     type_num2, std::forward<F>(f));
         case TypeIndex::Decimal256:
             return call_on_basic_type<Decimal256, _int, _float, _decimal, _datetime>(
@@ -218,10 +218,10 @@ bool call_on_index_and_data_type(TypeIndex number, F&& f) {
         return f(TypePair<DataTypeDecimal<Decimal32>, T>());
     case TypeIndex::Decimal64:
         return f(TypePair<DataTypeDecimal<Decimal64>, T>());
-    case TypeIndex::Decimal128:
-        return f(TypePair<DataTypeDecimal<Decimal128>, T>());
-    case TypeIndex::Decimal128I:
-        return f(TypePair<DataTypeDecimal<Decimal128I>, T>());
+    case TypeIndex::Decimal128V2:
+        return f(TypePair<DataTypeDecimal<Decimal128V2>, T>());
+    case TypeIndex::Decimal128V3:
+        return f(TypePair<DataTypeDecimal<Decimal128V3>, T>());
     case TypeIndex::Decimal256:
         return f(TypePair<DataTypeDecimal<Decimal256>, T>());
 
@@ -280,10 +280,10 @@ bool call_on_index_and_number_data_type(TypeIndex number, F&& f) {
         return f(TypePair<DataTypeDecimal<Decimal32>, T>());
     case TypeIndex::Decimal64:
         return f(TypePair<DataTypeDecimal<Decimal64>, T>());
-    case TypeIndex::Decimal128:
-        return f(TypePair<DataTypeDecimal<Decimal128>, T>());
-    case TypeIndex::Decimal128I:
-        return f(TypePair<DataTypeDecimal<Decimal128I>, T>());
+    case TypeIndex::Decimal128V2:
+        return f(TypePair<DataTypeDecimal<Decimal128V2>, T>());
+    case TypeIndex::Decimal128V3:
+        return f(TypePair<DataTypeDecimal<Decimal128V3>, T>());
     case TypeIndex::Decimal256:
         return f(TypePair<DataTypeDecimal<Decimal256>, T>());
     default:
diff --git a/be/src/vec/core/field.cpp b/be/src/vec/core/field.cpp
index 337c2c395f201d..8cb07f27c7c416 100644
--- a/be/src/vec/core/field.cpp
+++ b/be/src/vec/core/field.cpp
@@ -127,8 +127,8 @@ void write_binary(const Array& x, BufferWritable& buf) {
 }
 
 template <>
-Decimal128I DecimalField<Decimal128I>::get_scale_multiplier() const {
-    return DataTypeDecimal<Decimal128I>::get_scale_multiplier(scale);
+Decimal128V3 DecimalField<Decimal128V3>::get_scale_multiplier() const {
+    return DataTypeDecimal<Decimal128V3>::get_scale_multiplier(scale);
 }
 
 template <typename T>
@@ -169,19 +169,19 @@ bool dec_less_or_equal(T x, T y, UInt32 x_scale, UInt32 y_scale) {
 
 DECLARE_DECIMAL_COMPARISON(Decimal32)
 DECLARE_DECIMAL_COMPARISON(Decimal64)
-DECLARE_DECIMAL_COMPARISON(Decimal128)
+DECLARE_DECIMAL_COMPARISON(Decimal128V2)
 DECLARE_DECIMAL_COMPARISON(Decimal256)
 
 template <>
-bool decimal_equal(Decimal128I x, Decimal128I y, UInt32 xs, UInt32 ys) {
-    return dec_equal(Decimal128(x.value), Decimal128(y.value), xs, ys);
+bool decimal_equal(Decimal128V3 x, Decimal128V3 y, UInt32 xs, UInt32 ys) {
+    return dec_equal(Decimal128V2(x.value), Decimal128V2(y.value), xs, ys);
 }
 template <>
-bool decimal_less(Decimal128I x, Decimal128I y, UInt32 xs, UInt32 ys) {
-    return dec_less(Decimal128(x.value), Decimal128(y.value), xs, ys);
+bool decimal_less(Decimal128V3 x, Decimal128V3 y, UInt32 xs, UInt32 ys) {
+    return dec_less(Decimal128V2(x.value), Decimal128V2(y.value), xs, ys);
 }
 template <>
-bool decimal_less_or_equal(Decimal128I x, Decimal128I y, UInt32 xs, UInt32 ys) {
-    return dec_less_or_equal(Decimal128(x.value), Decimal128(y.value), xs, ys);
+bool decimal_less_or_equal(Decimal128V3 x, Decimal128V3 y, UInt32 xs, UInt32 ys) {
+    return dec_less_or_equal(Decimal128V2(x.value), Decimal128V2(y.value), xs, ys);
 }
 } // namespace doris::vectorized
diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h
index 752e8356ab5519..15de84589f2c2b 100644
--- a/be/src/vec/core/field.h
+++ b/be/src/vec/core/field.h
@@ -71,22 +71,22 @@ struct AvgNearestFieldTypeTrait {
 
 template <>
 struct AvgNearestFieldTypeTrait<Decimal32> {
-    using Type = Decimal128I;
+    using Type = Decimal128V3;
 };
 
 template <>
 struct AvgNearestFieldTypeTrait<Decimal64> {
-    using Type = Decimal128I;
+    using Type = Decimal128V3;
 };
 
 template <>
-struct AvgNearestFieldTypeTrait<Decimal128> {
-    using Type = Decimal128;
+struct AvgNearestFieldTypeTrait<Decimal128V2> {
+    using Type = Decimal128V2;
 };
 
 template <>
-struct AvgNearestFieldTypeTrait<Decimal128I> {
-    using Type = Decimal128I;
+struct AvgNearestFieldTypeTrait<Decimal128V3> {
+    using Type = Decimal128V3;
 };
 
 template <>
@@ -115,12 +115,12 @@ struct AvgNearestFieldTypeTrait256<Decimal64> {
 };
 
 template <>
-struct AvgNearestFieldTypeTrait256<Decimal128> {
-    using Type = Decimal128;
+struct AvgNearestFieldTypeTrait256<Decimal128V2> {
+    using Type = Decimal128V2;
 };
 
 template <>
-struct AvgNearestFieldTypeTrait256<Decimal128I> {
+struct AvgNearestFieldTypeTrait256<Decimal128V3> {
     using Type = Decimal256;
 };
 
@@ -349,10 +349,10 @@ class Field {
             Tuple = 18,
             Decimal32 = 19,
             Decimal64 = 20,
-            Decimal128 = 21,
+            Decimal128V2 = 21,
             AggregateFunctionState = 22,
             JSONB = 23,
-            Decimal128I = 24,
+            Decimal128V3 = 24,
             Map = 25,
             VariantMap = 26,
             Bitmap = 27,
@@ -392,10 +392,10 @@ class Field {
                 return "Decimal32";
             case Decimal64:
                 return "Decimal64";
-            case Decimal128:
-                return "Decimal128";
-            case Decimal128I:
-                return "Decimal128I";
+            case Decimal128V2:
+                return "Decimal128V2";
+            case Decimal128V3:
+                return "Decimal128V3";
             case Decimal256:
                 return "Decimal256";
             case FixedLengthObject:
@@ -422,8 +422,8 @@ class Field {
     struct EnumToType;
 
     static bool is_decimal(Types::Which which) {
-        return (which >= Types::Decimal32 && which <= Types::Decimal128) ||
-               which == Types::Decimal128I || which == Types::Decimal256;
+        return (which >= Types::Decimal32 && which <= Types::Decimal128V2) ||
+               which == Types::Decimal128V3 || which == Types::Decimal256;
     }
 
     Field() : which(Types::Null) {}
@@ -590,10 +590,10 @@ class Field {
             return get<Decimal32>() <=> rhs.get<Decimal32>();
         case Types::Decimal64:
             return get<Decimal64>() <=> rhs.get<Decimal64>();
-        case Types::Decimal128:
-            return get<Decimal128>() <=> rhs.get<Decimal128>();
-        case Types::Decimal128I:
-            return get<Decimal128I>() <=> rhs.get<Decimal128I>();
+        case Types::Decimal128V2:
+            return get<Decimal128V2>() <=> rhs.get<Decimal128V2>();
+        case Types::Decimal128V3:
+            return get<Decimal128V3>() <=> rhs.get<Decimal128V3>();
         case Types::Decimal256:
             return get<Decimal256>() <=> rhs.get<Decimal256>();
         default:
@@ -646,11 +646,11 @@ class Field {
         case Types::Decimal64:
             f(field.template get<DecimalField<Decimal64>>());
             return;
-        case Types::Decimal128:
-            f(field.template get<DecimalField<Decimal128>>());
+        case Types::Decimal128V2:
+            f(field.template get<DecimalField<Decimal128V2>>());
             return;
-        case Types::Decimal128I:
-            f(field.template get<DecimalField<Decimal128I>>());
+        case Types::Decimal128V3:
+            f(field.template get<DecimalField<Decimal128V3>>());
             return;
         case Types::Decimal256:
             f(field.template get<DecimalField<Decimal256>>());
@@ -676,9 +676,9 @@ class Field {
 private:
     std::aligned_union_t<DBMS_MIN_FIELD_SIZE - sizeof(Types::Which), Null, UInt64, UInt128, Int64,
                          Int128, Float64, String, JsonbField, Array, Tuple, Map, VariantMap,
-                         DecimalField<Decimal32>, DecimalField<Decimal64>, DecimalField<Decimal128>,
-                         DecimalField<Decimal128I>, DecimalField<Decimal256>, BitmapValue,
-                         HyperLogLog, QuantileState>
+                         DecimalField<Decimal32>, DecimalField<Decimal64>,
+                         DecimalField<Decimal128V2>, DecimalField<Decimal128V3>,
+                         DecimalField<Decimal256>, BitmapValue, HyperLogLog, QuantileState>
             storage;
 
     Types::Which which;
@@ -795,12 +795,12 @@ struct TypeId<DecimalField<Decimal64>> {
     static constexpr const TypeIndex value = TypeIndex::Decimal64;
 };
 template <>
-struct TypeId<DecimalField<Decimal128>> {
-    static constexpr const TypeIndex value = TypeIndex::Decimal128;
+struct TypeId<DecimalField<Decimal128V2>> {
+    static constexpr const TypeIndex value = TypeIndex::Decimal128V2;
 };
 template <>
-struct TypeId<DecimalField<Decimal128I>> {
-    static constexpr const TypeIndex value = TypeIndex::Decimal128I;
+struct TypeId<DecimalField<Decimal128V3>> {
+    static constexpr const TypeIndex value = TypeIndex::Decimal128V3;
 };
 template <>
 struct TypeId<DecimalField<Decimal256>> {
@@ -863,12 +863,12 @@ struct Field::TypeToEnum<DecimalField<Decimal64>> {
     static constexpr Types::Which value = Types::Decimal64;
 };
 template <>
-struct Field::TypeToEnum<DecimalField<Decimal128>> {
-    static constexpr Types::Which value = Types::Decimal128;
+struct Field::TypeToEnum<DecimalField<Decimal128V2>> {
+    static constexpr Types::Which value = Types::Decimal128V2;
 };
 template <>
-struct Field::TypeToEnum<DecimalField<Decimal128I>> {
-    static constexpr Types::Which value = Types::Decimal128I;
+struct Field::TypeToEnum<DecimalField<Decimal128V3>> {
+    static constexpr Types::Which value = Types::Decimal128V3;
 };
 template <>
 struct Field::TypeToEnum<DecimalField<Decimal256>> {
@@ -947,12 +947,12 @@ struct Field::EnumToType<Field::Types::Decimal64> {
     using Type = DecimalField<Decimal64>;
 };
 template <>
-struct Field::EnumToType<Field::Types::Decimal128> {
-    using Type = DecimalField<Decimal128>;
+struct Field::EnumToType<Field::Types::Decimal128V2> {
+    using Type = DecimalField<Decimal128V2>;
 };
 template <>
-struct Field::EnumToType<Field::Types::Decimal128I> {
-    using Type = DecimalField<Decimal128I>;
+struct Field::EnumToType<Field::Types::Decimal128V3> {
+    using Type = DecimalField<Decimal128V3>;
 };
 template <>
 struct Field::EnumToType<Field::Types::Decimal256> {
@@ -1051,12 +1051,12 @@ struct NearestFieldTypeImpl<Decimal64> {
     using Type = DecimalField<Decimal64>;
 };
 template <>
-struct NearestFieldTypeImpl<Decimal128> {
-    using Type = DecimalField<Decimal128>;
+struct NearestFieldTypeImpl<Decimal128V2> {
+    using Type = DecimalField<Decimal128V2>;
 };
 template <>
-struct NearestFieldTypeImpl<Decimal128I> {
-    using Type = DecimalField<Decimal128I>;
+struct NearestFieldTypeImpl<Decimal128V3> {
+    using Type = DecimalField<Decimal128V3>;
 };
 template <>
 struct NearestFieldTypeImpl<Decimal256> {
@@ -1071,12 +1071,12 @@ struct NearestFieldTypeImpl<DecimalField<Decimal64>> {
     using Type = DecimalField<Decimal64>;
 };
 template <>
-struct NearestFieldTypeImpl<DecimalField<Decimal128>> {
-    using Type = DecimalField<Decimal128>;
+struct NearestFieldTypeImpl<DecimalField<Decimal128V2>> {
+    using Type = DecimalField<Decimal128V2>;
 };
 template <>
-struct NearestFieldTypeImpl<DecimalField<Decimal128I>> {
-    using Type = DecimalField<Decimal128I>;
+struct NearestFieldTypeImpl<DecimalField<Decimal128V3>> {
+    using Type = DecimalField<Decimal128V3>;
 };
 template <>
 struct NearestFieldTypeImpl<DecimalField<Decimal256>> {
diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h
index aca679c9109cbf..82bc67a8fc5646 100644
--- a/be/src/vec/core/types.h
+++ b/be/src/vec/core/types.h
@@ -72,7 +72,7 @@ enum class TypeIndex {
     Enum16 = 18,
     Decimal32 = 19,
     Decimal64 = 20,
-    Decimal128 = 21,
+    Decimal128V2 = 21,
     UUID = 22,
     Array = 23,
     Tuple = 24,
@@ -89,7 +89,7 @@ enum class TypeIndex {
     TimeV2 = 35,
     FixedLengthObject = 36,
     JSONB = 37,
-    Decimal128I = 38,
+    Decimal128V3 = 38,
     Map = 39,
     Struct = 40,
     VARIANT = 41,
@@ -650,11 +650,11 @@ inline Decimal<T> operator%(const Decimal<T>& x, const Decimal<T>& y) {
     return Decimal<T>(x.value % y.value);
 }
 
-struct Decimal128I : public Decimal<Int128> {
-    Decimal128I() = default;
+struct Decimal128V3 : public Decimal<Int128> {
+    Decimal128V3() = default;
 
 #define DECLARE_NUMERIC_CTOR(TYPE) \
-    Decimal128I(const TYPE& value_) : Decimal<Int128>(value_) {}
+    Decimal128V3(const TYPE& value_) : Decimal<Int128>(value_) {}
 
     DECLARE_NUMERIC_CTOR(wide::Int256)
     DECLARE_NUMERIC_CTOR(Int128)
@@ -667,14 +667,14 @@ struct Decimal128I : public Decimal<Int128> {
 #undef DECLARE_NUMERIC_CTOR
 
     template <typename U>
-    Decimal128I(const Decimal<U>& x) {
+    Decimal128V3(const Decimal<U>& x) {
         value = x;
     }
 };
 
 using Decimal32 = Decimal<Int32>;
 using Decimal64 = Decimal<Int64>;
-using Decimal128 = Decimal<Int128>;
+using Decimal128V2 = Decimal<Int128>;
 using Decimal256 = Decimal<wide::Int256>;
 
 inline bool operator<(const Decimal256& x, const Decimal256& y) {
@@ -699,12 +699,12 @@ struct TypeName<Decimal64> {
     static const char* get() { return "Decimal64"; }
 };
 template <>
-struct TypeName<Decimal128> {
-    static const char* get() { return "Decimal128"; }
+struct TypeName<Decimal128V2> {
+    static const char* get() { return "Decimal128V2"; }
 };
 template <>
-struct TypeName<Decimal128I> {
-    static const char* get() { return "Decimal128I"; }
+struct TypeName<Decimal128V3> {
+    static const char* get() { return "Decimal128V3"; }
 };
 
 template <>
@@ -721,12 +721,12 @@ struct TypeId<Decimal64> {
     static constexpr const TypeIndex value = TypeIndex::Decimal64;
 };
 template <>
-struct TypeId<Decimal128> {
-    static constexpr const TypeIndex value = TypeIndex::Decimal128;
+struct TypeId<Decimal128V2> {
+    static constexpr const TypeIndex value = TypeIndex::Decimal128V2;
 };
 template <>
-struct TypeId<Decimal128I> {
-    static constexpr const TypeIndex value = TypeIndex::Decimal128I;
+struct TypeId<Decimal128V3> {
+    static constexpr const TypeIndex value = TypeIndex::Decimal128V3;
 };
 template <>
 struct TypeId<Decimal256> {
@@ -740,21 +740,21 @@ inline constexpr bool IsDecimalNumber<Decimal32> = true;
 template <>
 inline constexpr bool IsDecimalNumber<Decimal64> = true;
 template <>
-inline constexpr bool IsDecimalNumber<Decimal128> = true;
+inline constexpr bool IsDecimalNumber<Decimal128V2> = true;
 template <>
-inline constexpr bool IsDecimalNumber<Decimal128I> = true;
+inline constexpr bool IsDecimalNumber<Decimal128V3> = true;
 template <>
 inline constexpr bool IsDecimalNumber<Decimal256> = true;
 
 template <typename T>
-constexpr bool IsDecimal128 = false;
+constexpr bool IsDecimal128V2 = false;
 template <>
-inline constexpr bool IsDecimal128<Decimal128> = true;
+inline constexpr bool IsDecimal128V2<Decimal128V2> = true;
 
 template <typename T>
-constexpr bool IsDecimal128I = false;
+constexpr bool IsDecimal128V3 = false;
 template <>
-inline constexpr bool IsDecimal128I<Decimal128I> = true;
+inline constexpr bool IsDecimal128V3<Decimal128V3> = true;
 
 template <typename T>
 constexpr bool IsDecimal256 = false;
@@ -762,14 +762,14 @@ template <>
 inline constexpr bool IsDecimal256<Decimal256> = true;
 
 template <typename T>
-constexpr bool IsDecimalV2 = IsDecimal128<T> && !IsDecimal128I<T>;
+constexpr bool IsDecimalV2 = IsDecimal128V2<T> && !IsDecimal128V3<T>;
 
 template <typename T, typename U>
-using DisposeDecimal = std::conditional_t<IsDecimalV2<T>, Decimal128,
-                                          std::conditional_t<IsDecimalNumber<T>, Decimal128I, U>>;
+using DisposeDecimal = std::conditional_t<IsDecimalV2<T>, Decimal128V2,
+                                          std::conditional_t<IsDecimalNumber<T>, Decimal128V3, U>>;
 
 template <typename T, typename U>
-using DisposeDecimal256 = std::conditional_t<IsDecimalV2<T>, Decimal128,
+using DisposeDecimal256 = std::conditional_t<IsDecimalV2<T>, Decimal128V2,
                                              std::conditional_t<IsDecimalNumber<T>, Decimal256, U>>;
 
 template <typename T>
@@ -792,11 +792,11 @@ struct NativeType<Decimal64> {
     using Type = Int64;
 };
 template <>
-struct NativeType<Decimal128> {
+struct NativeType<Decimal128V2> {
     using Type = Int128;
 };
 template <>
-struct NativeType<Decimal128I> {
+struct NativeType<Decimal128V3> {
     using Type = Int128;
 };
 template <>
@@ -860,10 +860,10 @@ inline const char* getTypeName(TypeIndex idx) {
         return TypeName<Decimal32>::get();
     case TypeIndex::Decimal64:
         return TypeName<Decimal64>::get();
-    case TypeIndex::Decimal128:
-        return TypeName<Decimal128>::get();
-    case TypeIndex::Decimal128I:
-        return TypeName<Decimal128I>::get();
+    case TypeIndex::Decimal128V2:
+        return TypeName<Decimal128V2>::get();
+    case TypeIndex::Decimal128V3:
+        return TypeName<Decimal128V3>::get();
     case TypeIndex::Decimal256:
         return TypeName<Decimal256>::get();
     case TypeIndex::UUID:
@@ -919,8 +919,8 @@ struct std::hash<doris::vectorized::Decimal<T>> {
 };
 
 template <>
-struct std::hash<doris::vectorized::Decimal128> {
-    size_t operator()(const doris::vectorized::Decimal128& x) const {
+struct std::hash<doris::vectorized::Decimal128V2> {
+    size_t operator()(const doris::vectorized::Decimal128V2& x) const {
         return std::hash<doris::vectorized::Int64>()(x.value >> 64) ^
                std::hash<doris::vectorized::Int64>()(
                        x.value & std::numeric_limits<doris::vectorized::UInt64>::max());
@@ -928,8 +928,8 @@ struct std::hash<doris::vectorized::Decimal128> {
 };
 
 template <>
-struct std::hash<doris::vectorized::Decimal128I> {
-    size_t operator()(const doris::vectorized::Decimal128I& x) const {
+struct std::hash<doris::vectorized::Decimal128V3> {
+    size_t operator()(const doris::vectorized::Decimal128V3& x) const {
         return std::hash<doris::vectorized::Int64>()(x.value >> 64) ^
                std::hash<doris::vectorized::Int64>()(
                        x.value & std::numeric_limits<doris::vectorized::UInt64>::max());
diff --git a/be/src/vec/data_types/convert_field_to_type.cpp b/be/src/vec/data_types/convert_field_to_type.cpp
index 13e1ce62924e52..0196d5ede76a55 100644
--- a/be/src/vec/data_types/convert_field_to_type.cpp
+++ b/be/src/vec/data_types/convert_field_to_type.cpp
@@ -89,11 +89,11 @@ class FieldVisitorToStringSimple : public StaticVisitor<String> {
         LOG(FATAL) << "not implemeted";
         __builtin_unreachable();
     }
-    [[noreturn]] String operator()(const DecimalField<Decimal128>& x) const {
+    [[noreturn]] String operator()(const DecimalField<Decimal128V2>& x) const {
         LOG(FATAL) << "not implemeted";
         __builtin_unreachable();
     }
-    [[noreturn]] String operator()(const DecimalField<Decimal128I>& x) const {
+    [[noreturn]] String operator()(const DecimalField<Decimal128V3>& x) const {
         LOG(FATAL) << "not implemeted";
         __builtin_unreachable();
     }
@@ -135,10 +135,10 @@ class FieldVisitorToJsonb : public StaticVisitor<void> {
     void operator()(const DecimalField<Decimal64>& x, JsonbWriter* writer) const {
         throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR, "Not implemeted");
     }
-    void operator()(const DecimalField<Decimal128>& x, JsonbWriter* writer) const {
+    void operator()(const DecimalField<Decimal128V2>& x, JsonbWriter* writer) const {
         throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR, "Not implemeted");
     }
-    void operator()(const DecimalField<Decimal128I>& x, JsonbWriter* writer) const {
+    void operator()(const DecimalField<Decimal128V3>& x, JsonbWriter* writer) const {
         throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR, "Not implemeted");
     }
     void operator()(const DecimalField<Decimal256>& x, JsonbWriter* writer) const {
diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp
index 0db189262b58e5..e1c12d429fffd5 100644
--- a/be/src/vec/data_types/data_type.cpp
+++ b/be/src/vec/data_types/data_type.cpp
@@ -139,9 +139,9 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
         return PGenericType::DECIMAL32;
     case TypeIndex::Decimal64:
         return PGenericType::DECIMAL64;
-    case TypeIndex::Decimal128:
+    case TypeIndex::Decimal128V2:
         return PGenericType::DECIMAL128;
-    case TypeIndex::Decimal128I:
+    case TypeIndex::Decimal128V3:
         return PGenericType::DECIMAL128I;
     case TypeIndex::Decimal256:
         return PGenericType::DECIMAL256;
diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
index 3bcc9b064f8dc6..06082f0095af9a 100644
--- a/be/src/vec/data_types/data_type.h
+++ b/be/src/vec/data_types/data_type.h
@@ -266,11 +266,11 @@ struct WhichDataType {
 
     bool is_decimal32() const { return idx == TypeIndex::Decimal32; }
     bool is_decimal64() const { return idx == TypeIndex::Decimal64; }
-    bool is_decimal128() const { return idx == TypeIndex::Decimal128; }
-    bool is_decimal128i() const { return idx == TypeIndex::Decimal128I; }
+    bool is_decimal128v2() const { return idx == TypeIndex::Decimal128V2; }
+    bool is_decimal128v3() const { return idx == TypeIndex::Decimal128V3; }
     bool is_decimal256() const { return idx == TypeIndex::Decimal256; }
     bool is_decimal() const {
-        return is_decimal32() || is_decimal64() || is_decimal128() || is_decimal128i() ||
+        return is_decimal32() || is_decimal64() || is_decimal128v2() || is_decimal128v3() ||
                is_decimal256();
     }
 
@@ -326,7 +326,7 @@ inline bool is_decimal(const DataTypePtr& data_type) {
     return WhichDataType(data_type).is_decimal();
 }
 inline bool is_decimal_v2(const DataTypePtr& data_type) {
-    return WhichDataType(data_type).is_decimal128();
+    return WhichDataType(data_type).is_decimal128v2();
 }
 inline bool is_tuple(const DataTypePtr& data_type) {
     return WhichDataType(data_type).is_tuple();
diff --git a/be/src/vec/data_types/data_type_decimal.cpp b/be/src/vec/data_types/data_type_decimal.cpp
index 20e444fb0e2106..bec9847d0a6c4c 100644
--- a/be/src/vec/data_types/data_type_decimal.cpp
+++ b/be/src/vec/data_types/data_type_decimal.cpp
@@ -150,7 +150,7 @@ Field DataTypeDecimal<T>::get_default() const {
 template <typename T>
 MutableColumnPtr DataTypeDecimal<T>::create_column() const {
     if constexpr (IsDecimalV2<T>) {
-        auto col = ColumnDecimal128::create(0, scale);
+        auto col = ColumnDecimal128V2::create(0, scale);
         return col;
     } else {
         return ColumnType::create(0, scale);
@@ -167,7 +167,7 @@ bool DataTypeDecimal<T>::parse_from_string(const std::string& str, T* res) const
 
 DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value, bool use_v2) {
     auto max_precision =
-            use_v2 ? max_decimal_precision<Decimal128>() : max_decimal_precision<Decimal256>();
+            use_v2 ? max_decimal_precision<Decimal128V2>() : max_decimal_precision<Decimal256>();
     if (precision_value < min_decimal_precision() || precision_value > max_precision) {
         throw doris::Exception(doris::ErrorCode::NOT_IMPLEMENTED_ERROR,
                                "Wrong precision {}, min: {}, max: {}", precision_value,
@@ -182,15 +182,15 @@ DataTypePtr create_decimal(UInt64 precision_value, UInt64 scale_value, bool use_
     }
 
     if (use_v2) {
-        return std::make_shared<DataTypeDecimal<Decimal128>>(precision_value, scale_value);
+        return std::make_shared<DataTypeDecimal<Decimal128V2>>(precision_value, scale_value);
     }
 
     if (precision_value <= max_decimal_precision<Decimal32>()) {
         return std::make_shared<DataTypeDecimal<Decimal32>>(precision_value, scale_value);
     } else if (precision_value <= max_decimal_precision<Decimal64>()) {
         return std::make_shared<DataTypeDecimal<Decimal64>>(precision_value, scale_value);
-    } else if (precision_value <= max_decimal_precision<Decimal128I>()) {
-        return std::make_shared<DataTypeDecimal<Decimal128I>>(precision_value, scale_value);
+    } else if (precision_value <= max_decimal_precision<Decimal128V3>()) {
+        return std::make_shared<DataTypeDecimal<Decimal128V3>>(precision_value, scale_value);
     }
     return std::make_shared<DataTypeDecimal<Decimal256>>(precision_value, scale_value);
 }
@@ -206,12 +206,12 @@ Decimal64 DataTypeDecimal<Decimal64>::get_scale_multiplier(UInt32 scale) {
 }
 
 template <>
-Decimal128 DataTypeDecimal<Decimal128>::get_scale_multiplier(UInt32 scale) {
+Decimal128V2 DataTypeDecimal<Decimal128V2>::get_scale_multiplier(UInt32 scale) {
     return common::exp10_i128(scale);
 }
 
 template <>
-Decimal128I DataTypeDecimal<Decimal128I>::get_scale_multiplier(UInt32 scale) {
+Decimal128V3 DataTypeDecimal<Decimal128V3>::get_scale_multiplier(UInt32 scale) {
     return common::exp10_i128(scale);
 }
 
@@ -229,11 +229,11 @@ Decimal64 DataTypeDecimal<Decimal64>::get_max_digits_number(UInt32 digit_count)
     return common::max_i64(digit_count);
 }
 template <>
-Decimal128 DataTypeDecimal<Decimal128>::get_max_digits_number(UInt32 digit_count) {
+Decimal128V2 DataTypeDecimal<Decimal128V2>::get_max_digits_number(UInt32 digit_count) {
     return common::max_i128(digit_count);
 }
 template <>
-Decimal128I DataTypeDecimal<Decimal128I>::get_max_digits_number(UInt32 digit_count) {
+Decimal128V3 DataTypeDecimal<Decimal128V3>::get_max_digits_number(UInt32 digit_count) {
     return common::max_i128(digit_count);
 }
 template <>
@@ -244,8 +244,8 @@ Decimal256 DataTypeDecimal<Decimal256>::get_max_digits_number(UInt32 digit_count
 /// Explicit template instantiations.
 template class DataTypeDecimal<Decimal32>;
 template class DataTypeDecimal<Decimal64>;
-template class DataTypeDecimal<Decimal128>;
-template class DataTypeDecimal<Decimal128I>;
+template class DataTypeDecimal<Decimal128V2>;
+template class DataTypeDecimal<Decimal128V3>;
 template class DataTypeDecimal<Decimal256>;
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h
index 68d48e393b06af..234078ad4c14b7 100644
--- a/be/src/vec/data_types/data_type_decimal.h
+++ b/be/src/vec/data_types/data_type_decimal.h
@@ -86,11 +86,11 @@ constexpr size_t max_decimal_precision<Decimal64>() {
     return BeConsts::MAX_DECIMAL64_PRECISION;
 }
 template <>
-constexpr size_t max_decimal_precision<Decimal128>() {
+constexpr size_t max_decimal_precision<Decimal128V2>() {
     return BeConsts::MAX_DECIMALV2_PRECISION;
 }
 template <>
-constexpr size_t max_decimal_precision<Decimal128I>() {
+constexpr size_t max_decimal_precision<Decimal128V3>() {
     return BeConsts::MAX_DECIMAL128_PRECISION;
 }
 template <>
@@ -160,7 +160,7 @@ class DataTypeDecimal final : public IDataType {
             desc = TypeDescriptor(TYPE_DECIMAL32);
         } else if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal64>>) {
             desc = TypeDescriptor(TYPE_DECIMAL64);
-        } else if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128I>>) {
+        } else if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128V3>>) {
             desc = TypeDescriptor(TYPE_DECIMAL128I);
         } else if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal256>>) {
             desc = TypeDescriptor(TYPE_DECIMAL256);
@@ -179,7 +179,7 @@ class DataTypeDecimal final : public IDataType {
         if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal64>>) {
             return doris::FieldType::OLAP_FIELD_TYPE_DECIMAL64;
         }
-        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128I>>) {
+        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128V3>>) {
             return doris::FieldType::OLAP_FIELD_TYPE_DECIMAL128I;
         }
         if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal256>>) {
@@ -201,11 +201,11 @@ class DataTypeDecimal final : public IDataType {
         DCHECK_EQ(node.node_type, TExprNodeType::DECIMAL_LITERAL);
         DCHECK(node.__isset.decimal_literal);
         // decimalv2
-        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128>>) {
+        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128V2>>) {
             DecimalV2Value value;
             if (value.parse_from_str(node.decimal_literal.value.c_str(),
                                      node.decimal_literal.value.size()) == E_DEC_OK) {
-                return DecimalField<Decimal128>(value.value(), value.scale());
+                return DecimalField<Decimal128V2>(value.value(), value.scale());
             } else {
                 throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
                                        "Invalid decimal(scale: {}) value: {}", value.scale(),
@@ -364,10 +364,10 @@ inline UInt32 get_decimal_scale(const IDataType& data_type, UInt32 default_value
     if (auto* decimal_type = check_decimal<Decimal64>(data_type)) {
         return decimal_type->get_scale();
     }
-    if (auto* decimal_type = check_decimal<Decimal128>(data_type)) {
+    if (auto* decimal_type = check_decimal<Decimal128V2>(data_type)) {
         return decimal_type->get_scale();
     }
-    if (auto* decimal_type = check_decimal<Decimal128I>(data_type)) {
+    if (auto* decimal_type = check_decimal<Decimal128V3>(data_type)) {
         return decimal_type->get_scale();
     }
     if (auto* decimal_type = check_decimal<Decimal256>(data_type)) {
@@ -385,21 +385,21 @@ inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal32>> = true;
 template <>
 inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal64>> = true;
 template <>
-inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal128>> = true;
+inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal128V2>> = true;
 template <>
-inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal128I>> = true;
+inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal128V3>> = true;
 template <>
 inline constexpr bool IsDataTypeDecimal<DataTypeDecimal<Decimal256>> = true;
 
 template <typename DataType>
 constexpr bool IsDataTypeDecimalV2 = false;
 template <>
-inline constexpr bool IsDataTypeDecimalV2<DataTypeDecimal<Decimal128>> = true;
+inline constexpr bool IsDataTypeDecimalV2<DataTypeDecimal<Decimal128V2>> = true;
 
 template <typename DataType>
-constexpr bool IsDataTypeDecimal128I = false;
+constexpr bool IsDataTypeDecimal128V3 = false;
 template <>
-inline constexpr bool IsDataTypeDecimal128I<DataTypeDecimal<Decimal128I>> = true;
+inline constexpr bool IsDataTypeDecimal128V3<DataTypeDecimal<Decimal128V3>> = true;
 
 template <typename DataType>
 constexpr bool IsDataTypeDecimal256 = false;
@@ -499,9 +499,9 @@ void convert_decimal_cols(
     using ToFieldType = typename ToDataType::FieldType;
     using MaxFieldType =
             std::conditional_t<(sizeof(FromFieldType) == sizeof(ToFieldType)) &&
-                                       (std::is_same_v<ToFieldType, Decimal128I> ||
-                                        std::is_same_v<FromFieldType, Decimal128I>),
-                               Decimal128I,
+                                       (std::is_same_v<ToFieldType, Decimal128V3> ||
+                                        std::is_same_v<FromFieldType, Decimal128V3>),
+                               Decimal128V3,
                                std::conditional_t<(sizeof(FromFieldType) > sizeof(ToFieldType)),
                                                   FromFieldType, ToFieldType>>;
     using MaxNativeType = typename MaxFieldType::NativeType;
@@ -657,7 +657,7 @@ void convert_to_decimal(typename ToDataType::FieldType* dst,
         }
     } else {
         using DecimalFrom =
-                std::conditional_t<std::is_same_v<FromFieldType, Int128>, Decimal128,
+                std::conditional_t<std::is_same_v<FromFieldType, Int128>, Decimal128V2,
                                    std::conditional_t<std::is_same_v<FromFieldType, wide::Int256>,
                                                       Decimal256, Decimal64>>;
         convert_to_decimals<DataTypeDecimal<DecimalFrom>, ToDataType, multiply_may_overflow,
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index fc317128177e62..d349fffe99a727 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -188,7 +188,7 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
         nested = std::make_shared<vectorized::DataTypeBitMap>();
         break;
     case TYPE_DECIMALV2:
-        nested = std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128>>(27, 9);
+        nested = std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128V2>>(27, 9);
         break;
     case TYPE_QUANTILE_STATE:
         nested = std::make_shared<vectorized::DataTypeQuantileState>();
@@ -311,13 +311,13 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeIndex& type_index, bool
     case TypeIndex::Decimal64:
         nested = std::make_shared<DataTypeDecimal<Decimal64>>(BeConsts::MAX_DECIMAL64_PRECISION, 0);
         break;
-    case TypeIndex::Decimal128:
-        nested = std::make_shared<DataTypeDecimal<Decimal128>>(BeConsts::MAX_DECIMAL128_PRECISION,
-                                                               0);
+    case TypeIndex::Decimal128V2:
+        nested = std::make_shared<DataTypeDecimal<Decimal128V2>>(BeConsts::MAX_DECIMALV2_PRECISION,
+                                                                 0);
         break;
-    case TypeIndex::Decimal128I:
-        nested = std::make_shared<DataTypeDecimal<Decimal128I>>(BeConsts::MAX_DECIMAL128_PRECISION,
-                                                                0);
+    case TypeIndex::Decimal128V3:
+        nested = std::make_shared<DataTypeDecimal<Decimal128V3>>(BeConsts::MAX_DECIMAL128_PRECISION,
+                                                                 0);
         break;
     case TypeIndex::Decimal256:
         nested = std::make_shared<DataTypeDecimal<Decimal256>>(BeConsts::MAX_DECIMAL256_PRECISION,
@@ -414,7 +414,7 @@ DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type,
         result = std::make_shared<vectorized::DataTypeBitMap>();
         break;
     case FieldType::OLAP_FIELD_TYPE_DECIMAL:
-        result = std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128>>(27, 9);
+        result = std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128V2>>(27, 9);
         break;
     case FieldType::OLAP_FIELD_TYPE_QUANTILE_STATE:
         result = std::make_shared<vectorized::DataTypeQuantileState>();
@@ -508,12 +508,12 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
                                                               pcolumn.decimal_param().scale());
         break;
     case PGenericType::DECIMAL128:
-        nested = std::make_shared<DataTypeDecimal<Decimal128>>(pcolumn.decimal_param().precision(),
-                                                               pcolumn.decimal_param().scale());
+        nested = std::make_shared<DataTypeDecimal<Decimal128V2>>(
+                pcolumn.decimal_param().precision(), pcolumn.decimal_param().scale());
         break;
     case PGenericType::DECIMAL128I:
-        nested = std::make_shared<DataTypeDecimal<Decimal128I>>(pcolumn.decimal_param().precision(),
-                                                                pcolumn.decimal_param().scale());
+        nested = std::make_shared<DataTypeDecimal<Decimal128V3>>(
+                pcolumn.decimal_param().precision(), pcolumn.decimal_param().scale());
         break;
     case PGenericType::DECIMAL256:
         nested = std::make_shared<DataTypeDecimal<Decimal256>>(pcolumn.decimal_param().precision(),
@@ -667,7 +667,7 @@ DataTypePtr DataTypeFactory::create_data_type(const arrow::DataType* type, bool
         nested = std::make_shared<vectorized::DataTypeString>();
         break;
     case ::arrow::Type::DECIMAL:
-        nested = std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128>>();
+        nested = std::make_shared<vectorized::DataTypeDecimal<vectorized::Decimal128V2>>();
         break;
     case ::arrow::Type::LIST:
         DCHECK(type->num_fields() == 1);
diff --git a/be/src/vec/data_types/get_least_supertype.cpp b/be/src/vec/data_types/get_least_supertype.cpp
index 1baf463b052453..49c5bacce10f77 100644
--- a/be/src/vec/data_types/get_least_supertype.cpp
+++ b/be/src/vec/data_types/get_least_supertype.cpp
@@ -394,8 +394,8 @@ void get_least_supertype(const DataTypes& types, DataTypePtr* type) {
     {
         UInt32 have_decimal32 = type_ids.count(TypeIndex::Decimal32);
         UInt32 have_decimal64 = type_ids.count(TypeIndex::Decimal64);
-        UInt32 have_decimal128 = type_ids.count(TypeIndex::Decimal128);
-        UInt32 have_decimal128i = type_ids.count(TypeIndex::Decimal128I);
+        UInt32 have_decimal128 = type_ids.count(TypeIndex::Decimal128V2);
+        UInt32 have_decimal128i = type_ids.count(TypeIndex::Decimal128V3);
         UInt32 have_decimal256 = type_ids.count(TypeIndex::Decimal256);
 
         if (have_decimal32 || have_decimal64 || have_decimal128 || have_decimal128i ||
@@ -452,19 +452,19 @@ void get_least_supertype(const DataTypes& types, DataTypePtr* type) {
                         type);
             }
 
-            if (have_decimal256 || min_precision > DataTypeDecimal<Decimal128I>::max_precision()) {
+            if (have_decimal256 || min_precision > DataTypeDecimal<Decimal128V3>::max_precision()) {
                 *type = std::make_shared<DataTypeDecimal<Decimal256>>(
                         DataTypeDecimal<Decimal256>::max_precision(), max_scale);
                 return;
             }
             if (have_decimal128 || min_precision > DataTypeDecimal<Decimal64>::max_precision()) {
-                *type = std::make_shared<DataTypeDecimal<Decimal128>>(
-                        DataTypeDecimal<Decimal128>::max_precision(), max_scale);
+                *type = std::make_shared<DataTypeDecimal<Decimal128V2>>(
+                        DataTypeDecimal<Decimal128V2>::max_precision(), max_scale);
                 return;
             }
             if (have_decimal128i || min_precision > DataTypeDecimal<Decimal64>::max_precision()) {
-                *type = std::make_shared<DataTypeDecimal<Decimal128I>>(
-                        DataTypeDecimal<Decimal128I>::max_precision(), max_scale);
+                *type = std::make_shared<DataTypeDecimal<Decimal128V3>>(
+                        DataTypeDecimal<Decimal128V3>::max_precision(), max_scale);
                 return;
             }
             if (have_decimal64 || min_precision > DataTypeDecimal<Decimal32>::max_precision()) {
diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp
index 65b504474f32ef..3d5c10ff01d8f2 100644
--- a/be/src/vec/data_types/serde/data_type_decimal_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_decimal_serde.cpp
@@ -107,7 +107,7 @@ void DataTypeDecimalSerDe<T>::write_column_to_arrow(const IColumn& column, const
                              array_builder->type()->name());
         }
         // TODO: decimal256
-    } else if constexpr (std::is_same_v<T, Decimal128I>) {
+    } else if constexpr (std::is_same_v<T, Decimal128V3>) {
         std::shared_ptr<arrow::DataType> s_decimal_ptr =
                 std::make_shared<arrow::Decimal128Type>(38, col.get_scale());
         for (size_t i = start; i < end; ++i) {
@@ -172,18 +172,19 @@ void DataTypeDecimalSerDe<T>::read_column_from_arrow(IColumn& column,
     if constexpr (std::is_same_v<T, Decimal<Int128>>) {
         // TODO check precision
         for (size_t value_i = start; value_i < end; ++value_i) {
-            auto value = *reinterpret_cast<const vectorized::Decimal128*>(
+            auto value = *reinterpret_cast<const vectorized::Decimal128V2*>(
                     concrete_array->Value(value_i));
             // convert scale to 9;
             if (9 > arrow_scale) {
-                using MaxNativeType = typename Decimal128::NativeType;
+                using MaxNativeType = typename Decimal128V2::NativeType;
                 MaxNativeType converted_value = common::exp10_i128(9 - arrow_scale);
                 if (common::mul_overflow(static_cast<MaxNativeType>(value), converted_value,
                                          converted_value)) {
                     VLOG_DEBUG << "Decimal convert overflow";
                     value = converted_value < 0
-                                    ? std::numeric_limits<typename Decimal128 ::NativeType>::min()
-                                    : std::numeric_limits<typename Decimal128 ::NativeType>::max();
+                                    ? std::numeric_limits<typename Decimal128V2 ::NativeType>::min()
+                                    : std::numeric_limits<
+                                              typename Decimal128V2 ::NativeType>::max();
                 } else {
                     value = converted_value;
                 }
@@ -192,7 +193,7 @@ void DataTypeDecimalSerDe<T>::read_column_from_arrow(IColumn& column,
             }
             column_data.emplace_back(value);
         }
-    } else if constexpr (std::is_same_v<T, Decimal128I> || std::is_same_v<T, Decimal64> ||
+    } else if constexpr (std::is_same_v<T, Decimal128V3> || std::is_same_v<T, Decimal64> ||
                          std::is_same_v<T, Decimal32>) {
         for (size_t value_i = start; value_i < end; ++value_i) {
             column_data.emplace_back(*reinterpret_cast<const T*>(concrete_array->Value(value_i)));
@@ -246,7 +247,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_orc(const std::string& timezone,
                                                     std::vector<StringRef>& buffer_list) const {
     auto& col_data = assert_cast<const ColumnDecimal<T>&>(column).get_data();
 
-    if constexpr (IsDecimal128<T> || IsDecimal128I<T>) {
+    if constexpr (IsDecimal128V2<T> || IsDecimal128V3<T>) {
         orc::Decimal128VectorBatch* cur_batch =
                 dynamic_cast<orc::Decimal128VectorBatch*>(orc_col_batch);
 
@@ -274,8 +275,8 @@ Status DataTypeDecimalSerDe<T>::write_column_to_orc(const std::string& timezone,
 
 template class DataTypeDecimalSerDe<Decimal32>;
 template class DataTypeDecimalSerDe<Decimal64>;
-template class DataTypeDecimalSerDe<Decimal128>;
-template class DataTypeDecimalSerDe<Decimal128I>;
+template class DataTypeDecimalSerDe<Decimal128V2>;
+template class DataTypeDecimalSerDe<Decimal128V3>;
 template class DataTypeDecimalSerDe<Decimal256>;
 } // namespace vectorized
 } // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.h b/be/src/vec/data_types/serde/data_type_decimal_serde.h
index 57cbc306518ad1..347e17ada144e5 100644
--- a/be/src/vec/data_types/serde/data_type_decimal_serde.h
+++ b/be/src/vec/data_types/serde/data_type_decimal_serde.h
@@ -55,10 +55,10 @@ class DataTypeDecimalSerDe : public DataTypeSerDe {
         if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal64>>) {
             return TYPE_DECIMAL64;
         }
-        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128I>>) {
+        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128V3>>) {
             return TYPE_DECIMAL128I;
         }
-        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128>>) {
+        if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal128V2>>) {
             return TYPE_DECIMALV2;
         }
         if constexpr (std::is_same_v<TypeId<T>, TypeId<Decimal256>>) {
@@ -130,7 +130,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_pb(const IColumn& column, PValue
     auto ptype = result.mutable_type();
     if constexpr (std::is_same_v<T, Decimal<Int128>>) {
         ptype->set_id(PGenericType::DECIMAL128);
-    } else if constexpr (std::is_same_v<T, Decimal128I>) {
+    } else if constexpr (std::is_same_v<T, Decimal128V3>) {
         ptype->set_id(PGenericType::DECIMAL128I);
     } else if constexpr (std::is_same_v<T, Decimal256>) {
         ptype->set_id(PGenericType::DECIMAL256);
@@ -152,7 +152,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_pb(const IColumn& column, PValue
 // TODO: decimal256
 template <typename T>
 Status DataTypeDecimalSerDe<T>::read_column_from_pb(IColumn& column, const PValues& arg) const {
-    if constexpr (std::is_same_v<T, Decimal<Int128>> || std::is_same_v<T, Decimal128I> ||
+    if constexpr (std::is_same_v<T, Decimal<Int128>> || std::is_same_v<T, Decimal128V3> ||
                   std::is_same_v<T, Decimal256> || std::is_same_v<T, Decimal<Int32>>) {
         column.resize(arg.bytes_value_size());
         auto& data = reinterpret_cast<ColumnDecimal<T>&>(column).get_data();
@@ -173,12 +173,12 @@ void DataTypeDecimalSerDe<T>::write_one_cell_to_jsonb(const IColumn& column, Jso
     result.writeKey(col_id);
     // TODO: decimal256
     if constexpr (std::is_same_v<T, Decimal<Int128>>) {
-        Decimal128::NativeType val =
-                *reinterpret_cast<const Decimal128::NativeType*>(data_ref.data);
+        Decimal128V2::NativeType val =
+                *reinterpret_cast<const Decimal128V2::NativeType*>(data_ref.data);
         result.writeInt128(val);
-    } else if constexpr (std::is_same_v<T, Decimal128I>) {
-        Decimal128I::NativeType val =
-                *reinterpret_cast<const Decimal128I::NativeType*>(data_ref.data);
+    } else if constexpr (std::is_same_v<T, Decimal128V3>) {
+        Decimal128V3::NativeType val =
+                *reinterpret_cast<const Decimal128V3::NativeType*>(data_ref.data);
         result.writeInt128(val);
     } else if constexpr (std::is_same_v<T, Decimal<Int32>>) {
         Decimal32::NativeType val = *reinterpret_cast<const Decimal32::NativeType*>(data_ref.data);
@@ -199,7 +199,7 @@ void DataTypeDecimalSerDe<T>::read_one_cell_from_jsonb(IColumn& column,
     // TODO: decimal256
     if constexpr (std::is_same_v<T, Decimal<Int128>>) {
         col.insert_value(static_cast<const JsonbInt128Val*>(arg)->val());
-    } else if constexpr (std::is_same_v<T, Decimal128I>) {
+    } else if constexpr (std::is_same_v<T, Decimal128V3>) {
         col.insert_value(static_cast<const JsonbInt128Val*>(arg)->val());
     } else if constexpr (std::is_same_v<T, Decimal<Int32>>) {
         col.insert_value(static_cast<const JsonbInt32Val*>(arg)->val());
diff --git a/be/src/vec/exec/format/format_common.h b/be/src/vec/exec/format/format_common.h
index 9e8ce4fe238c35..24664d7769c037 100644
--- a/be/src/vec/exec/format/format_common.h
+++ b/be/src/vec/exec/format/format_common.h
@@ -38,9 +38,9 @@ struct DecimalScaleParams {
             return common::exp10_i32(n);
         } else if constexpr (std::is_same_v<DecimalPrimitiveType, Decimal64>) {
             return common::exp10_i64(n);
-        } else if constexpr (std::is_same_v<DecimalPrimitiveType, Decimal128>) {
+        } else if constexpr (std::is_same_v<DecimalPrimitiveType, Decimal128V2>) {
             return common::exp10_i128(n);
-        } else if constexpr (std::is_same_v<DecimalPrimitiveType, Decimal128I>) {
+        } else if constexpr (std::is_same_v<DecimalPrimitiveType, Decimal128V3>) {
             return common::exp10_i128(n);
         } else {
             static_assert(!sizeof(DecimalPrimitiveType),
diff --git a/be/src/vec/exec/format/orc/vorc_reader.cpp b/be/src/vec/exec/format/orc/vorc_reader.cpp
index 12a889ec23ed2f..943b9d1e4d6066 100644
--- a/be/src/vec/exec/format/orc/vorc_reader.cpp
+++ b/be/src/vec/exec/format/orc/vorc_reader.cpp
@@ -1297,12 +1297,12 @@ Status OrcReader::_orc_column_to_doris_column(const std::string& col_name,
     case TypeIndex::Decimal64:
         return _decode_decimal_column<Decimal64, is_filter>(col_name, data_column, data_type, cvb,
                                                             num_values);
-    case TypeIndex::Decimal128:
-        return _decode_decimal_column<Decimal128, is_filter>(col_name, data_column, data_type, cvb,
-                                                             num_values);
-    case TypeIndex::Decimal128I:
-        return _decode_decimal_column<Decimal128I, is_filter>(col_name, data_column, data_type, cvb,
-                                                              num_values);
+    case TypeIndex::Decimal128V2:
+        return _decode_decimal_column<Decimal128V2, is_filter>(col_name, data_column, data_type,
+                                                               cvb, num_values);
+    case TypeIndex::Decimal128V3:
+        return _decode_decimal_column<Decimal128V3, is_filter>(col_name, data_column, data_type,
+                                                               cvb, num_values);
     case TypeIndex::Date:
         return _decode_time_column<VecDateTimeValue, Int64, orc::LongVectorBatch, is_filter>(
                 col_name, data_column, cvb, num_values);
diff --git a/be/src/vec/exec/format/parquet/parquet_column_convert.h b/be/src/vec/exec/format/parquet/parquet_column_convert.h
index 65f5270face454..1b876a86cecb27 100644
--- a/be/src/vec/exec/format/parquet/parquet_column_convert.h
+++ b/be/src/vec/exec/format/parquet/parquet_column_convert.h
@@ -108,11 +108,11 @@ struct PhysicalTypeTraits<tparquet::Type::INT96> {
     M(TypeIndex::Float32, Float32, Float32) \
     M(TypeIndex::Float64, Float64, Float64)
 
-#define FOR_LOGICAL_DECIMAL_TYPES(M)                 \
-    M(TypeIndex::Decimal32, Decimal32, Decimal32)    \
-    M(TypeIndex::Decimal64, Decimal64, Decimal64)    \
-    M(TypeIndex::Decimal128, Decimal128, Decimal128) \
-    M(TypeIndex::Decimal128I, Decimal128I, Decimal128I)
+#define FOR_LOGICAL_DECIMAL_TYPES(M)                       \
+    M(TypeIndex::Decimal32, Decimal32, Decimal32)          \
+    M(TypeIndex::Decimal64, Decimal64, Decimal64)          \
+    M(TypeIndex::Decimal128V2, Decimal128V2, Decimal128V2) \
+    M(TypeIndex::Decimal128V3, Decimal128V3, Decimal128V3)
 
 struct ConvertParams {
     // schema.logicalType.TIMESTAMP.isAdjustedToUTC == false
@@ -537,10 +537,10 @@ inline Status get_converter(tparquet::Type::type parquet_physical_type, Primitiv
                 *converter = std::make_unique<StringToDecimalString<Decimal64, Int64>>();
                 break;
             } else if (show_type == PrimitiveType::TYPE_DECIMALV2) {
-                *converter = std::make_unique<StringToDecimalString<Decimal128, Int128>>();
+                *converter = std::make_unique<StringToDecimalString<Decimal128V2, Int128>>();
                 break;
             } else if (show_type == PrimitiveType::TYPE_DECIMAL128I) {
-                *converter = std::make_unique<StringToDecimalString<Decimal128, Int128>>();
+                *converter = std::make_unique<StringToDecimalString<Decimal128V2, Int128>>();
                 break;
             }
 
diff --git a/be/src/vec/exec/format/parquet/parquet_pred_cmp.h b/be/src/vec/exec/format/parquet/parquet_pred_cmp.h
index 8158fb9e9f116e..ae0eea44fff86a 100644
--- a/be/src/vec/exec/format/parquet/parquet_pred_cmp.h
+++ b/be/src/vec/exec/format/parquet/parquet_pred_cmp.h
@@ -163,7 +163,7 @@ class ParquetPredicate {
             break;
         case TYPE_DECIMALV2:
             if constexpr (std::is_same_v<CppType, DecimalV2Value>) {
-                size_t max_precision = max_decimal_precision<Decimal128>();
+                size_t max_precision = max_decimal_precision<Decimal128V2>();
                 if (col_schema->parquet_schema.precision < 1 ||
                     col_schema->parquet_schema.precision > max_precision ||
                     col_schema->parquet_schema.scale > max_precision) {
@@ -171,19 +171,19 @@ class ParquetPredicate {
                 }
                 int v2_scale = DecimalV2Value::SCALE;
                 if (physical_type == tparquet::Type::FIXED_LEN_BYTE_ARRAY) {
-                    min_value = DecimalV2Value(
-                            _decode_binary_decimal<Decimal128>(col_schema, encoded_min, v2_scale));
-                    max_value = DecimalV2Value(
-                            _decode_binary_decimal<Decimal128>(col_schema, encoded_max, v2_scale));
+                    min_value = DecimalV2Value(_decode_binary_decimal<Decimal128V2>(
+                            col_schema, encoded_min, v2_scale));
+                    max_value = DecimalV2Value(_decode_binary_decimal<Decimal128V2>(
+                            col_schema, encoded_max, v2_scale));
                 } else if (physical_type == tparquet::Type::INT32) {
-                    min_value = DecimalV2Value(_decode_primitive_decimal<Decimal128, Int32>(
+                    min_value = DecimalV2Value(_decode_primitive_decimal<Decimal128V2, Int32>(
                             col_schema, encoded_min, v2_scale));
-                    max_value = DecimalV2Value(_decode_primitive_decimal<Decimal128, Int32>(
+                    max_value = DecimalV2Value(_decode_primitive_decimal<Decimal128V2, Int32>(
                             col_schema, encoded_max, v2_scale));
                 } else if (physical_type == tparquet::Type::INT64) {
-                    min_value = DecimalV2Value(_decode_primitive_decimal<Decimal128, Int64>(
+                    min_value = DecimalV2Value(_decode_primitive_decimal<Decimal128V2, Int64>(
                             col_schema, encoded_min, v2_scale));
-                    max_value = DecimalV2Value(_decode_primitive_decimal<Decimal128, Int64>(
+                    max_value = DecimalV2Value(_decode_primitive_decimal<Decimal128V2, Int64>(
                             col_schema, encoded_max, v2_scale));
                 } else {
                     return false;
@@ -199,7 +199,7 @@ class ParquetPredicate {
         case TYPE_DECIMAL128I:
             if constexpr (std::is_same_v<CppType, Decimal32> ||
                           std::is_same_v<CppType, Decimal64> ||
-                          std::is_same_v<CppType, Decimal128I>) {
+                          std::is_same_v<CppType, Decimal128V3>) {
                 size_t max_precision = max_decimal_precision<CppType>();
                 if (col_schema->parquet_schema.precision < 1 ||
                     col_schema->parquet_schema.precision > max_precision ||
diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp
index 719e035f1a1e75..59ad1015976e5e 100644
--- a/be/src/vec/exec/jni_connector.cpp
+++ b/be/src/vec/exec/jni_connector.cpp
@@ -44,25 +44,25 @@ class RuntimeProfile;
 
 namespace doris::vectorized {
 
-#define FOR_FIXED_LENGTH_TYPES(M)                                     \
-    M(TypeIndex::Int8, ColumnVector<Int8>, Int8)                      \
-    M(TypeIndex::UInt8, ColumnVector<UInt8>, UInt8)                   \
-    M(TypeIndex::Int16, ColumnVector<Int16>, Int16)                   \
-    M(TypeIndex::UInt16, ColumnVector<UInt16>, UInt16)                \
-    M(TypeIndex::Int32, ColumnVector<Int32>, Int32)                   \
-    M(TypeIndex::UInt32, ColumnVector<UInt32>, UInt32)                \
-    M(TypeIndex::Int64, ColumnVector<Int64>, Int64)                   \
-    M(TypeIndex::UInt64, ColumnVector<UInt64>, UInt64)                \
-    M(TypeIndex::Int128, ColumnVector<Int128>, Int128)                \
-    M(TypeIndex::Float32, ColumnVector<Float32>, Float32)             \
-    M(TypeIndex::Float64, ColumnVector<Float64>, Float64)             \
-    M(TypeIndex::Decimal128, ColumnDecimal<Decimal<Int128>>, Int128)  \
-    M(TypeIndex::Decimal128I, ColumnDecimal<Decimal<Int128>>, Int128) \
-    M(TypeIndex::Decimal32, ColumnDecimal<Decimal<Int32>>, Int32)     \
-    M(TypeIndex::Decimal64, ColumnDecimal<Decimal<Int64>>, Int64)     \
-    M(TypeIndex::Date, ColumnVector<Int64>, Int64)                    \
-    M(TypeIndex::DateV2, ColumnVector<UInt32>, UInt32)                \
-    M(TypeIndex::DateTime, ColumnVector<Int64>, Int64)                \
+#define FOR_FIXED_LENGTH_TYPES(M)                                      \
+    M(TypeIndex::Int8, ColumnVector<Int8>, Int8)                       \
+    M(TypeIndex::UInt8, ColumnVector<UInt8>, UInt8)                    \
+    M(TypeIndex::Int16, ColumnVector<Int16>, Int16)                    \
+    M(TypeIndex::UInt16, ColumnVector<UInt16>, UInt16)                 \
+    M(TypeIndex::Int32, ColumnVector<Int32>, Int32)                    \
+    M(TypeIndex::UInt32, ColumnVector<UInt32>, UInt32)                 \
+    M(TypeIndex::Int64, ColumnVector<Int64>, Int64)                    \
+    M(TypeIndex::UInt64, ColumnVector<UInt64>, UInt64)                 \
+    M(TypeIndex::Int128, ColumnVector<Int128>, Int128)                 \
+    M(TypeIndex::Float32, ColumnVector<Float32>, Float32)              \
+    M(TypeIndex::Float64, ColumnVector<Float64>, Float64)              \
+    M(TypeIndex::Decimal128V2, ColumnDecimal<Decimal<Int128>>, Int128) \
+    M(TypeIndex::Decimal128V3, ColumnDecimal<Decimal<Int128>>, Int128) \
+    M(TypeIndex::Decimal32, ColumnDecimal<Decimal<Int32>>, Int32)      \
+    M(TypeIndex::Decimal64, ColumnDecimal<Decimal<Int64>>, Int64)      \
+    M(TypeIndex::Date, ColumnVector<Int64>, Int64)                     \
+    M(TypeIndex::DateV2, ColumnVector<UInt32>, UInt32)                 \
+    M(TypeIndex::DateTime, ColumnVector<Int64>, Int64)                 \
     M(TypeIndex::DateTimeV2, ColumnVector<UInt64>, UInt64)
 
 JniConnector::~JniConnector() {
diff --git a/be/src/vec/functions/array/function_array_aggregation.cpp b/be/src/vec/functions/array/function_array_aggregation.cpp
index 102acb9d3941da..e8a2fd9e952ae8 100644
--- a/be/src/vec/functions/array/function_array_aggregation.cpp
+++ b/be/src/vec/functions/array/function_array_aggregation.cpp
@@ -167,8 +167,8 @@ struct ArrayAggregateImpl {
             execute_type<Float64>(res, type, data, offsets) ||
             execute_type<Decimal32>(res, type, data, offsets) ||
             execute_type<Decimal64>(res, type, data, offsets) ||
-            execute_type<Decimal128>(res, type, data, offsets) ||
-            execute_type<Decimal128I>(res, type, data, offsets) ||
+            execute_type<Decimal128V2>(res, type, data, offsets) ||
+            execute_type<Decimal128V3>(res, type, data, offsets) ||
             execute_type<Date>(res, type, data, offsets) ||
             execute_type<DateTime>(res, type, data, offsets) ||
             execute_type<DateV2>(res, type, data, offsets) ||
diff --git a/be/src/vec/functions/array/function_array_apply.cpp b/be/src/vec/functions/array/function_array_apply.cpp
index 0e9076e65e50b4..2ad680635a73d8 100644
--- a/be/src/vec/functions/array/function_array_apply.cpp
+++ b/be/src/vec/functions/array/function_array_apply.cpp
@@ -173,48 +173,48 @@ class FunctionArrayApply : public IFunction {
     }
 
 // need exception safety
-#define APPLY_ALL_TYPES(src_column, src_offsets, OP, cmp, dst)                     \
-    do {                                                                           \
-        WhichDataType which(remove_nullable(nested_type));                         \
-        if (which.is_uint8()) {                                                    \
-            *dst = _apply_internal<UInt8, OP>(src_column, src_offsets, cmp);       \
-        } else if (which.is_int8()) {                                              \
-            *dst = _apply_internal<Int8, OP>(src_column, src_offsets, cmp);        \
-        } else if (which.is_int16()) {                                             \
-            *dst = _apply_internal<Int16, OP>(src_column, src_offsets, cmp);       \
-        } else if (which.is_int32()) {                                             \
-            *dst = _apply_internal<Int32, OP>(src_column, src_offsets, cmp);       \
-        } else if (which.is_int64()) {                                             \
-            *dst = _apply_internal<Int64, OP>(src_column, src_offsets, cmp);       \
-        } else if (which.is_int128()) {                                            \
-            *dst = _apply_internal<Int128, OP>(src_column, src_offsets, cmp);      \
-        } else if (which.is_float32()) {                                           \
-            *dst = _apply_internal<Float32, OP>(src_column, src_offsets, cmp);     \
-        } else if (which.is_float64()) {                                           \
-            *dst = _apply_internal<Float64, OP>(src_column, src_offsets, cmp);     \
-        } else if (which.is_date()) {                                              \
-            *dst = _apply_internal<Int64, OP>(src_column, src_offsets, cmp);       \
-        } else if (which.is_date_time()) {                                         \
-            *dst = _apply_internal<Int64, OP>(src_column, src_offsets, cmp);       \
-        } else if (which.is_date_v2()) {                                           \
-            *dst = _apply_internal<UInt32, OP>(src_column, src_offsets, cmp);      \
-        } else if (which.is_date_time_v2()) {                                      \
-            *dst = _apply_internal<UInt64, OP>(src_column, src_offsets, cmp);      \
-        } else if (which.is_date_time_v2()) {                                      \
-            *dst = _apply_internal<UInt64, OP>(src_column, src_offsets, cmp);      \
-        } else if (which.is_decimal32()) {                                         \
-            *dst = _apply_internal<Decimal32, OP>(src_column, src_offsets, cmp);   \
-        } else if (which.is_decimal64()) {                                         \
-            *dst = _apply_internal<Decimal64, OP>(src_column, src_offsets, cmp);   \
-        } else if (which.is_decimal128()) {                                        \
-            *dst = _apply_internal<Decimal128, OP>(src_column, src_offsets, cmp);  \
-        } else if (which.is_decimal128i()) {                                       \
-            *dst = _apply_internal<Decimal128I, OP>(src_column, src_offsets, cmp); \
-        } else if (which.is_decimal256()) {                                        \
-            *dst = _apply_internal<Decimal256, OP>(src_column, src_offsets, cmp);  \
-        } else {                                                                   \
-            LOG(FATAL) << "unsupported type " << nested_type->get_name();          \
-        }                                                                          \
+#define APPLY_ALL_TYPES(src_column, src_offsets, OP, cmp, dst)                      \
+    do {                                                                            \
+        WhichDataType which(remove_nullable(nested_type));                          \
+        if (which.is_uint8()) {                                                     \
+            *dst = _apply_internal<UInt8, OP>(src_column, src_offsets, cmp);        \
+        } else if (which.is_int8()) {                                               \
+            *dst = _apply_internal<Int8, OP>(src_column, src_offsets, cmp);         \
+        } else if (which.is_int16()) {                                              \
+            *dst = _apply_internal<Int16, OP>(src_column, src_offsets, cmp);        \
+        } else if (which.is_int32()) {                                              \
+            *dst = _apply_internal<Int32, OP>(src_column, src_offsets, cmp);        \
+        } else if (which.is_int64()) {                                              \
+            *dst = _apply_internal<Int64, OP>(src_column, src_offsets, cmp);        \
+        } else if (which.is_int128()) {                                             \
+            *dst = _apply_internal<Int128, OP>(src_column, src_offsets, cmp);       \
+        } else if (which.is_float32()) {                                            \
+            *dst = _apply_internal<Float32, OP>(src_column, src_offsets, cmp);      \
+        } else if (which.is_float64()) {                                            \
+            *dst = _apply_internal<Float64, OP>(src_column, src_offsets, cmp);      \
+        } else if (which.is_date()) {                                               \
+            *dst = _apply_internal<Int64, OP>(src_column, src_offsets, cmp);        \
+        } else if (which.is_date_time()) {                                          \
+            *dst = _apply_internal<Int64, OP>(src_column, src_offsets, cmp);        \
+        } else if (which.is_date_v2()) {                                            \
+            *dst = _apply_internal<UInt32, OP>(src_column, src_offsets, cmp);       \
+        } else if (which.is_date_time_v2()) {                                       \
+            *dst = _apply_internal<UInt64, OP>(src_column, src_offsets, cmp);       \
+        } else if (which.is_date_time_v2()) {                                       \
+            *dst = _apply_internal<UInt64, OP>(src_column, src_offsets, cmp);       \
+        } else if (which.is_decimal32()) {                                          \
+            *dst = _apply_internal<Decimal32, OP>(src_column, src_offsets, cmp);    \
+        } else if (which.is_decimal64()) {                                          \
+            *dst = _apply_internal<Decimal64, OP>(src_column, src_offsets, cmp);    \
+        } else if (which.is_decimal128v2()) {                                       \
+            *dst = _apply_internal<Decimal128V2, OP>(src_column, src_offsets, cmp); \
+        } else if (which.is_decimal128v3()) {                                       \
+            *dst = _apply_internal<Decimal128V3, OP>(src_column, src_offsets, cmp); \
+        } else if (which.is_decimal256()) {                                         \
+            *dst = _apply_internal<Decimal256, OP>(src_column, src_offsets, cmp);   \
+        } else {                                                                    \
+            LOG(FATAL) << "unsupported type " << nested_type->get_name();           \
+        }                                                                           \
     } while (0)
 
     // need exception safety
diff --git a/be/src/vec/functions/array/function_array_cum_sum.cpp b/be/src/vec/functions/array/function_array_cum_sum.cpp
index b3ddd0a6fe07fd..c4ae12a72d940d 100644
--- a/be/src/vec/functions/array/function_array_cum_sum.cpp
+++ b/be/src/vec/functions/array/function_array_cum_sum.cpp
@@ -70,13 +70,13 @@ class FunctionArrayCumSum : public IFunction {
             return_type = std::make_shared<DataTypeInt128>();
         } else if (which.is_float32() || which.is_float64()) {
             return_type = std::make_shared<DataTypeFloat64>();
-        } else if (which.is_decimal128() && !which.is_decimal128i()) {
+        } else if (which.is_decimal128v2() && !which.is_decimal128v3()) {
             // decimalv2
-            return_type = std::make_shared<DataTypeDecimal<Decimal128>>(
-                    DataTypeDecimal<Decimal128>::max_precision(), scale);
+            return_type = std::make_shared<DataTypeDecimal<Decimal128V2>>(
+                    DataTypeDecimal<Decimal128V2>::max_precision(), scale);
         } else if (which.is_decimal()) {
-            return_type = std::make_shared<DataTypeDecimal<Decimal128I>>(
-                    DataTypeDecimal<Decimal128I>::max_precision(), scale);
+            return_type = std::make_shared<DataTypeDecimal<Decimal128V3>>(
+                    DataTypeDecimal<Decimal128V3>::max_precision(), scale);
         }
         if (return_type) {
             return std::make_shared<DataTypeArray>(make_nullable(return_type));
@@ -161,17 +161,17 @@ class FunctionArrayCumSum : public IFunction {
             res = _execute_number<Float64, Float64>(src_column, src_offsets, src_null_map,
                                                     res_nested_ptr);
         } else if (which.is_decimal32()) {
-            res = _execute_number<Decimal32, Decimal128I>(src_column, src_offsets, src_null_map,
-                                                          res_nested_ptr);
+            res = _execute_number<Decimal32, Decimal128V3>(src_column, src_offsets, src_null_map,
+                                                           res_nested_ptr);
         } else if (which.is_decimal64()) {
-            res = _execute_number<Decimal64, Decimal128I>(src_column, src_offsets, src_null_map,
-                                                          res_nested_ptr);
-        } else if (which.is_decimal128i()) {
-            res = _execute_number<Decimal128I, Decimal128I>(src_column, src_offsets, src_null_map,
-                                                            res_nested_ptr);
-        } else if (which.is_decimal128()) {
-            res = _execute_number<Decimal128, Decimal128>(src_column, src_offsets, src_null_map,
-                                                          res_nested_ptr);
+            res = _execute_number<Decimal64, Decimal128V3>(src_column, src_offsets, src_null_map,
+                                                           res_nested_ptr);
+        } else if (which.is_decimal128v3()) {
+            res = _execute_number<Decimal128V3, Decimal128V3>(src_column, src_offsets, src_null_map,
+                                                              res_nested_ptr);
+        } else if (which.is_decimal128v2()) {
+            res = _execute_number<Decimal128V2, Decimal128V2>(src_column, src_offsets, src_null_map,
+                                                              res_nested_ptr);
         }
 
         return res;
diff --git a/be/src/vec/functions/array/function_array_difference.h b/be/src/vec/functions/array/function_array_difference.h
index cc2efa64bd93f1..b078396a91fea3 100644
--- a/be/src/vec/functions/array/function_array_difference.h
+++ b/be/src/vec/functions/array/function_array_difference.h
@@ -227,12 +227,12 @@ class FunctionArrayDifference : public IFunction {
         } else if (which_type.is_decimal64()) {
             res = _execute_number_expanded<Decimal64, Decimal64>(offsets, *nested_column,
                                                                  nested_null_map);
-        } else if (which_type.is_decimal128i()) {
-            res = _execute_number_expanded<Decimal128I, Decimal128I>(offsets, *nested_column,
-                                                                     nested_null_map);
-        } else if (which_type.is_decimal128()) {
-            res = _execute_number_expanded<Decimal128, Decimal128>(offsets, *nested_column,
-                                                                   nested_null_map);
+        } else if (which_type.is_decimal128v3()) {
+            res = _execute_number_expanded<Decimal128V3, Decimal128V3>(offsets, *nested_column,
+                                                                       nested_null_map);
+        } else if (which_type.is_decimal128v2()) {
+            res = _execute_number_expanded<Decimal128V2, Decimal128V2>(offsets, *nested_column,
+                                                                       nested_null_map);
         } else if (which_type.is_decimal256()) {
             res = _execute_number_expanded<Decimal256, Decimal256>(offsets, *nested_column,
                                                                    nested_null_map);
diff --git a/be/src/vec/functions/array/function_array_distinct.h b/be/src/vec/functions/array/function_array_distinct.h
index 7e5e0a73729299..936db4354474d7 100644
--- a/be/src/vec/functions/array/function_array_distinct.h
+++ b/be/src/vec/functions/array/function_array_distinct.h
@@ -301,15 +301,15 @@ class FunctionArrayDistinct : public IFunction {
         } else if (which.is_decimal64()) {
             res = _execute_number<ColumnDecimal64>(src_column, src_offsets, dest_column,
                                                    dest_offsets, src_null_map, dest_null_map);
-        } else if (which.is_decimal128i()) {
-            res = _execute_number<ColumnDecimal128I>(src_column, src_offsets, dest_column,
-                                                     dest_offsets, src_null_map, dest_null_map);
+        } else if (which.is_decimal128v3()) {
+            res = _execute_number<ColumnDecimal128V3>(src_column, src_offsets, dest_column,
+                                                      dest_offsets, src_null_map, dest_null_map);
         } else if (which.is_decimal256()) {
             res = _execute_number<ColumnDecimal256>(src_column, src_offsets, dest_column,
                                                     dest_offsets, src_null_map, dest_null_map);
-        } else if (which.is_decimal128()) {
-            res = _execute_number<ColumnDecimal128>(src_column, src_offsets, dest_column,
-                                                    dest_offsets, src_null_map, dest_null_map);
+        } else if (which.is_decimal128v2()) {
+            res = _execute_number<ColumnDecimal128V2>(src_column, src_offsets, dest_column,
+                                                      dest_offsets, src_null_map, dest_null_map);
         } else if (which.is_string()) {
             res = _execute_string(src_column, src_offsets, dest_column, dest_offsets, src_null_map,
                                   dest_null_map);
diff --git a/be/src/vec/functions/array/function_array_element.h b/be/src/vec/functions/array/function_array_element.h
index 53c7b9df3d081e..fef7aa3beab32f 100644
--- a/be/src/vec/functions/array/function_array_element.h
+++ b/be/src/vec/functions/array/function_array_element.h
@@ -392,12 +392,12 @@ class FunctionArrayElement : public IFunction {
         } else if (which_type.is_decimal64()) {
             res = _execute_number<ColumnDecimal64>(offsets, *nested_column, src_null_map, *idx_col,
                                                    nested_null_map, dst_null_map);
-        } else if (which_type.is_decimal128i()) {
-            res = _execute_number<ColumnDecimal128I>(offsets, *nested_column, src_null_map,
-                                                     *idx_col, nested_null_map, dst_null_map);
-        } else if (which_type.is_decimal128()) {
-            res = _execute_number<ColumnDecimal128>(offsets, *nested_column, src_null_map, *idx_col,
-                                                    nested_null_map, dst_null_map);
+        } else if (which_type.is_decimal128v3()) {
+            res = _execute_number<ColumnDecimal128V3>(offsets, *nested_column, src_null_map,
+                                                      *idx_col, nested_null_map, dst_null_map);
+        } else if (which_type.is_decimal128v2()) {
+            res = _execute_number<ColumnDecimal128V2>(offsets, *nested_column, src_null_map,
+                                                      *idx_col, nested_null_map, dst_null_map);
         } else if (which_type.is_decimal256()) {
             res = _execute_number<ColumnDecimal256>(offsets, *nested_column, src_null_map, *idx_col,
                                                     nested_null_map, dst_null_map);
diff --git a/be/src/vec/functions/array/function_array_enumerate_uniq.cpp b/be/src/vec/functions/array/function_array_enumerate_uniq.cpp
index f3bbf3c57a8165..9eaa71d08ae20a 100644
--- a/be/src/vec/functions/array/function_array_enumerate_uniq.cpp
+++ b/be/src/vec/functions/array/function_array_enumerate_uniq.cpp
@@ -196,14 +196,14 @@ class FunctionArrayEnumerateUniq : public IFunction {
                 _execute_number<ColumnDecimal32>(data_columns, *offsets, null_map, dst_values);
             } else if (which.is_decimal64()) {
                 _execute_number<ColumnDecimal64>(data_columns, *offsets, null_map, dst_values);
-            } else if (which.is_decimal128i()) {
-                _execute_number<ColumnDecimal128I>(data_columns, *offsets, null_map, dst_values);
+            } else if (which.is_decimal128v3()) {
+                _execute_number<ColumnDecimal128V3>(data_columns, *offsets, null_map, dst_values);
             } else if (which.is_decimal256()) {
                 _execute_number<ColumnDecimal256>(data_columns, *offsets, null_map, dst_values);
             } else if (which.is_date_time_v2()) {
                 _execute_number<ColumnDateTimeV2>(data_columns, *offsets, null_map, dst_values);
-            } else if (which.is_decimal128()) {
-                _execute_number<ColumnDecimal128>(data_columns, *offsets, null_map, dst_values);
+            } else if (which.is_decimal128v2()) {
+                _execute_number<ColumnDecimal128V2>(data_columns, *offsets, null_map, dst_values);
             } else if (which.is_string()) {
                 _execute_string(data_columns, *offsets, null_map, dst_values);
             }
diff --git a/be/src/vec/functions/array/function_array_index.h b/be/src/vec/functions/array/function_array_index.h
index 23f251e68c7698..d4a8fa32962fab 100644
--- a/be/src/vec/functions/array/function_array_index.h
+++ b/be/src/vec/functions/array/function_array_index.h
@@ -349,12 +349,12 @@ class FunctionArrayIndex : public IFunction {
                 return_column = _execute_number_expanded<ColumnDecimal64>(
                         offsets, nested_null_map, *nested_column, *right_column,
                         right_nested_null_map, array_null_map);
-            } else if (left_which_type.is_decimal128i()) {
-                return_column = _execute_number_expanded<ColumnDecimal128I>(
+            } else if (left_which_type.is_decimal128v3()) {
+                return_column = _execute_number_expanded<ColumnDecimal128V3>(
                         offsets, nested_null_map, *nested_column, *right_column,
                         right_nested_null_map, array_null_map);
-            } else if (left_which_type.is_decimal128()) {
-                return_column = _execute_number_expanded<ColumnDecimal128>(
+            } else if (left_which_type.is_decimal128v2()) {
+                return_column = _execute_number_expanded<ColumnDecimal128V2>(
                         offsets, nested_null_map, *nested_column, *right_column,
                         right_nested_null_map, array_null_map);
             } else if (left_which_type.is_decimal256()) {
diff --git a/be/src/vec/functions/array/function_array_join.h b/be/src/vec/functions/array/function_array_join.h
index b9829649148a0b..583738e4319e01 100644
--- a/be/src/vec/functions/array/function_array_join.h
+++ b/be/src/vec/functions/array/function_array_join.h
@@ -247,16 +247,17 @@ struct ArrayJoinImpl {
         } else if (which.is_decimal64()) {
             res = _execute_number<ColumnDecimal64>(src_column, src_offsets, src_null_map, sep_str,
                                                    null_replace_str, nested_type, dest_column_ptr);
-        } else if (which.is_decimal128i()) {
-            res = _execute_number<ColumnDecimal128I>(src_column, src_offsets, src_null_map, sep_str,
-                                                     null_replace_str, nested_type,
-                                                     dest_column_ptr);
+        } else if (which.is_decimal128v3()) {
+            res = _execute_number<ColumnDecimal128V3>(src_column, src_offsets, src_null_map,
+                                                      sep_str, null_replace_str, nested_type,
+                                                      dest_column_ptr);
         } else if (which.is_decimal256()) {
             res = _execute_number<ColumnDecimal256>(src_column, src_offsets, src_null_map, sep_str,
                                                     null_replace_str, nested_type, dest_column_ptr);
-        } else if (which.is_decimal128()) {
-            res = _execute_number<ColumnDecimal128>(src_column, src_offsets, src_null_map, sep_str,
-                                                    null_replace_str, nested_type, dest_column_ptr);
+        } else if (which.is_decimal128v2()) {
+            res = _execute_number<ColumnDecimal128V2>(src_column, src_offsets, src_null_map,
+                                                      sep_str, null_replace_str, nested_type,
+                                                      dest_column_ptr);
         } else if (which.is_string()) {
             res = _execute_string(src_column, src_offsets, src_null_map, sep_str, null_replace_str,
                                   dest_column_ptr);
diff --git a/be/src/vec/functions/array/function_array_remove.h b/be/src/vec/functions/array/function_array_remove.h
index 5a0ee49369da61..2645f87320fabb 100644
--- a/be/src/vec/functions/array/function_array_remove.h
+++ b/be/src/vec/functions/array/function_array_remove.h
@@ -323,12 +323,12 @@ class FunctionArrayRemove : public IFunction {
             } else if (left_which_type.is_decimal64()) {
                 res = _execute_number_expanded<ColumnDecimal64>(offsets, *nested_column,
                                                                 *right_column, nested_null_map);
-            } else if (left_which_type.is_decimal128i()) {
-                res = _execute_number_expanded<ColumnDecimal128I>(offsets, *nested_column,
-                                                                  *right_column, nested_null_map);
-            } else if (left_which_type.is_decimal128()) {
-                res = _execute_number_expanded<ColumnDecimal128>(offsets, *nested_column,
-                                                                 *right_column, nested_null_map);
+            } else if (left_which_type.is_decimal128v3()) {
+                res = _execute_number_expanded<ColumnDecimal128V3>(offsets, *nested_column,
+                                                                   *right_column, nested_null_map);
+            } else if (left_which_type.is_decimal128v2()) {
+                res = _execute_number_expanded<ColumnDecimal128V2>(offsets, *nested_column,
+                                                                   *right_column, nested_null_map);
             } else if (left_which_type.is_decimal256()) {
                 res = _execute_number_expanded<ColumnDecimal256>(offsets, *nested_column,
                                                                  *right_column, nested_null_map);
diff --git a/be/src/vec/functions/array/function_arrays_overlap.h b/be/src/vec/functions/array/function_arrays_overlap.h
index fb138d3edbc1b0..d1bca0ec02e2b4 100644
--- a/be/src/vec/functions/array/function_arrays_overlap.h
+++ b/be/src/vec/functions/array/function_arrays_overlap.h
@@ -217,14 +217,14 @@ class FunctionArraysOverlap : public IFunction {
             ret = _execute_internal<ColumnDecimal64>(left_exec_data, right_exec_data,
                                                      dst_null_map_data,
                                                      dst_nested_col->get_data().data());
-        } else if (left_which_type.is_decimal128i()) {
-            ret = _execute_internal<ColumnDecimal128I>(left_exec_data, right_exec_data,
-                                                       dst_null_map_data,
-                                                       dst_nested_col->get_data().data());
-        } else if (left_which_type.is_decimal128()) {
-            ret = _execute_internal<ColumnDecimal128>(left_exec_data, right_exec_data,
-                                                      dst_null_map_data,
-                                                      dst_nested_col->get_data().data());
+        } else if (left_which_type.is_decimal128v3()) {
+            ret = _execute_internal<ColumnDecimal128V3>(left_exec_data, right_exec_data,
+                                                        dst_null_map_data,
+                                                        dst_nested_col->get_data().data());
+        } else if (left_which_type.is_decimal128v2()) {
+            ret = _execute_internal<ColumnDecimal128V2>(left_exec_data, right_exec_data,
+                                                        dst_null_map_data,
+                                                        dst_nested_col->get_data().data());
         } else if (left_which_type.is_decimal256()) {
             ret = _execute_internal<ColumnDecimal256>(left_exec_data, right_exec_data,
                                                       dst_null_map_data,
diff --git a/be/src/vec/functions/function.h b/be/src/vec/functions/function.h
index 44a629bf248288..1b4c9fe128c32a 100644
--- a/be/src/vec/functions/function.h
+++ b/be/src/vec/functions/function.h
@@ -689,11 +689,11 @@ ColumnPtr wrap_in_nullable(const ColumnPtr& src, const Block& block, const Colum
     M(Float32, ColumnFloat32)          \
     M(Float64, ColumnFloat64)
 
-#define DECIMAL_TYPE_TO_COLUMN_TYPE(M)         \
-    M(Decimal32, ColumnDecimal<Decimal32>)     \
-    M(Decimal64, ColumnDecimal<Decimal64>)     \
-    M(Decimal128, ColumnDecimal<Decimal128>)   \
-    M(Decimal128I, ColumnDecimal<Decimal128I>) \
+#define DECIMAL_TYPE_TO_COLUMN_TYPE(M)           \
+    M(Decimal32, ColumnDecimal<Decimal32>)       \
+    M(Decimal64, ColumnDecimal<Decimal64>)       \
+    M(Decimal128V2, ColumnDecimal<Decimal128V2>) \
+    M(Decimal128V3, ColumnDecimal<Decimal128V3>) \
     M(Decimal256, ColumnDecimal<Decimal256>)
 
 #define STRING_TYPE_TO_COLUMN_TYPE(M) \
diff --git a/be/src/vec/functions/function_binary_arithmetic.h b/be/src/vec/functions/function_binary_arithmetic.h
index 4b69561b14e9e4..8c433783c2d321 100644
--- a/be/src/vec/functions/function_binary_arithmetic.h
+++ b/be/src/vec/functions/function_binary_arithmetic.h
@@ -83,7 +83,7 @@ struct OperationTraits {
             std::is_same_v<Op, DivideIntegralImpl<T, T>>;
     static constexpr bool can_overflow =
             (is_plus_minus || is_multiply) &&
-            (IsDecimalV2<OpA> || IsDecimalV2<OpB> || IsDecimal128I<OpA> || IsDecimal128I<OpB> ||
+            (IsDecimalV2<OpA> || IsDecimalV2<OpB> || IsDecimal128V3<OpA> || IsDecimal128V3<OpB> ||
              IsDecimal256<OpA> || IsDecimal256<OpB>);
     static constexpr bool has_variadic_argument =
             !std::is_void_v<decltype(has_variadic_argument_types(std::declval<Op>()))>;
@@ -658,13 +658,13 @@ template <>
 inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal256>, DataTypeDecimal<Decimal64>> =
         true;
 template <>
-inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal256>, DataTypeDecimal<Decimal128I>> =
+inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal256>, DataTypeDecimal<Decimal128V3>> =
         true;
 template <>
-inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128I>, DataTypeDecimal<Decimal32>> =
+inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128V3>, DataTypeDecimal<Decimal32>> =
         true;
 template <>
-inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128I>, DataTypeDecimal<Decimal64>> =
+inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal128V3>, DataTypeDecimal<Decimal64>> =
         true;
 template <>
 inline constexpr bool UseLeftDecimal<DataTypeDecimal<Decimal64>, DataTypeDecimal<Decimal32>> = true;
@@ -863,7 +863,7 @@ class FunctionBinaryArithmetic : public IFunction {
         return cast_type_to_either<DataTypeUInt8, DataTypeInt8, DataTypeInt16, DataTypeInt32,
                                    DataTypeInt64, DataTypeInt128, DataTypeFloat32, DataTypeFloat64,
                                    DataTypeDecimal<Decimal32>, DataTypeDecimal<Decimal64>,
-                                   DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal128I>,
+                                   DataTypeDecimal<Decimal128V2>, DataTypeDecimal<Decimal128V3>,
                                    DataTypeDecimal<Decimal256>>(type, std::forward<F>(f));
     }
 
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 474851e80d104d..57678b02b6880c 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -929,8 +929,8 @@ struct NameToDecimal64 {
 struct NameToDecimal128 {
     static constexpr auto name = "toDecimal128";
 };
-struct NameToDecimal128I {
-    static constexpr auto name = "toDecimal128I";
+struct NameToDecimal128V3 {
+    static constexpr auto name = "toDecimal128V3";
 };
 struct NameToDecimal256 {
     static constexpr auto name = "toDecimal256";
@@ -1057,7 +1057,7 @@ StringParser::ParseResult try_parse_decimal_impl(typename DataType::FieldType& x
         return try_read_decimal_text<TYPE_DECIMAL64>(x, rb, precision, scale);
     }
 
-    if constexpr (IsDataTypeDecimal128I<DataType>) {
+    if constexpr (IsDataTypeDecimal128V3<DataType>) {
         UInt32 scale = ((PrecisionScaleArg)additions).scale;
         UInt32 precision = ((PrecisionScaleArg)additions).precision;
         return try_read_decimal_text<TYPE_DECIMAL128I>(x, rb, precision, scale);
@@ -1333,9 +1333,9 @@ using FunctionToDecimal32 =
 using FunctionToDecimal64 =
         FunctionConvert<DataTypeDecimal<Decimal64>, NameToDecimal64, UnknownMonotonicity>;
 using FunctionToDecimal128 =
-        FunctionConvert<DataTypeDecimal<Decimal128>, NameToDecimal128, UnknownMonotonicity>;
-using FunctionToDecimal128I =
-        FunctionConvert<DataTypeDecimal<Decimal128I>, NameToDecimal128I, UnknownMonotonicity>;
+        FunctionConvert<DataTypeDecimal<Decimal128V2>, NameToDecimal128, UnknownMonotonicity>;
+using FunctionToDecimal128V3 =
+        FunctionConvert<DataTypeDecimal<Decimal128V3>, NameToDecimal128V3, UnknownMonotonicity>;
 using FunctionToDecimal256 =
         FunctionConvert<DataTypeDecimal<Decimal256>, NameToDecimal256, UnknownMonotonicity>;
 using FunctionToIPv4 = FunctionConvert<DataTypeIPv4, NameToIPv4, UnknownMonotonicity>;
@@ -1402,12 +1402,12 @@ struct FunctionTo<DataTypeDecimal<Decimal64>> {
     using Type = FunctionToDecimal64;
 };
 template <>
-struct FunctionTo<DataTypeDecimal<Decimal128>> {
+struct FunctionTo<DataTypeDecimal<Decimal128V2>> {
     using Type = FunctionToDecimal128;
 };
 template <>
-struct FunctionTo<DataTypeDecimal<Decimal128I>> {
-    using Type = FunctionToDecimal128I;
+struct FunctionTo<DataTypeDecimal<Decimal128V3>> {
+    using Type = FunctionToDecimal128V3;
 };
 template <>
 struct FunctionTo<DataTypeDecimal<Decimal256>> {
@@ -1573,11 +1573,11 @@ template <typename Name>
 struct ConvertImpl<DataTypeString, DataTypeDecimal<Decimal64>, Name>
         : ConvertThroughParsing<DataTypeString, DataTypeDecimal<Decimal64>, Name> {};
 template <typename Name>
-struct ConvertImpl<DataTypeString, DataTypeDecimal<Decimal128>, Name>
-        : ConvertThroughParsing<DataTypeString, DataTypeDecimal<Decimal128>, Name> {};
+struct ConvertImpl<DataTypeString, DataTypeDecimal<Decimal128V2>, Name>
+        : ConvertThroughParsing<DataTypeString, DataTypeDecimal<Decimal128V2>, Name> {};
 template <typename Name>
-struct ConvertImpl<DataTypeString, DataTypeDecimal<Decimal128I>, Name>
-        : ConvertThroughParsing<DataTypeString, DataTypeDecimal<Decimal128I>, Name> {};
+struct ConvertImpl<DataTypeString, DataTypeDecimal<Decimal128V3>, Name>
+        : ConvertThroughParsing<DataTypeString, DataTypeDecimal<Decimal128V3>, Name> {};
 template <typename Name>
 struct ConvertImpl<DataTypeString, DataTypeDecimal<Decimal256>, Name>
         : ConvertThroughParsing<DataTypeString, DataTypeDecimal<Decimal256>, Name> {};
@@ -2414,8 +2414,8 @@ class FunctionCast final : public IFunctionBase {
 
             if constexpr (std::is_same_v<ToDataType, DataTypeDecimal<Decimal32>> ||
                           std::is_same_v<ToDataType, DataTypeDecimal<Decimal64>> ||
-                          std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>> ||
-                          std::is_same_v<ToDataType, DataTypeDecimal<Decimal128I>> ||
+                          std::is_same_v<ToDataType, DataTypeDecimal<Decimal128V2>> ||
+                          std::is_same_v<ToDataType, DataTypeDecimal<Decimal128V3>> ||
                           std::is_same_v<ToDataType, DataTypeDecimal<Decimal256>>) {
                 ret = create_decimal_wrapper(from_type,
                                              check_and_get_data_type<ToDataType>(to_type.get()));
diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h
index 3d8e52203952cd..2b1d5891a663fd 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -2411,12 +2411,12 @@ struct MoneyFormatInt128Impl {
 
 struct MoneyFormatDecimalImpl {
     static DataTypes get_variadic_argument_types() {
-        return {std::make_shared<DataTypeDecimal<Decimal128>>(27, 9)};
+        return {std::make_shared<DataTypeDecimal<Decimal128V2>>(27, 9)};
     }
 
     static void execute(FunctionContext* context, ColumnString* result_column, ColumnPtr col_ptr,
                         size_t input_rows_count) {
-        if (auto* decimalv2_column = check_and_get_column<ColumnDecimal<Decimal128>>(*col_ptr)) {
+        if (auto* decimalv2_column = check_and_get_column<ColumnDecimal<Decimal128V2>>(*col_ptr)) {
             for (size_t i = 0; i < input_rows_count; i++) {
                 DecimalV2Value value = DecimalV2Value(decimalv2_column->get_element(i));
 
@@ -2467,12 +2467,12 @@ struct MoneyFormatDecimalImpl {
                 result_column->insert_data(str.data, str.size);
             }
         } else if (auto* decimal128_column =
-                           check_and_get_column<ColumnDecimal<Decimal128I>>(*col_ptr)) {
+                           check_and_get_column<ColumnDecimal<Decimal128V3>>(*col_ptr)) {
             const UInt32 scale = decimal128_column->get_scale();
             const auto multiplier =
                     scale > 2 ? common::exp10_i32(scale - 2) : common::exp10_i32(2 - scale);
             for (size_t i = 0; i < input_rows_count; i++) {
-                Decimal128I frac_part = decimal128_column->get_fractional_part(i);
+                Decimal128V3 frac_part = decimal128_column->get_fractional_part(i);
                 if (scale > 2) {
                     int delta = ((frac_part % multiplier) << 1) > multiplier;
                     frac_part = frac_part / multiplier + delta;
diff --git a/be/src/vec/functions/function_unary_arithmetic.h b/be/src/vec/functions/function_unary_arithmetic.h
index 8d18129a048a36..723fccad97279e 100644
--- a/be/src/vec/functions/function_unary_arithmetic.h
+++ b/be/src/vec/functions/function_unary_arithmetic.h
@@ -72,7 +72,7 @@ class FunctionUnaryArithmetic : public IFunction {
                                    DataTypeInt8, DataTypeInt16, DataTypeInt32, DataTypeInt64,
                                    DataTypeInt128, DataTypeFloat32, DataTypeFloat64,
                                    DataTypeDecimal<Decimal32>, DataTypeDecimal<Decimal64>,
-                                   DataTypeDecimal<Decimal128>, DataTypeDecimal<Decimal128I>,
+                                   DataTypeDecimal<Decimal128V2>, DataTypeDecimal<Decimal128V3>,
                                    DataTypeDecimal<Decimal256>>(type, std::forward<F>(f));
     }
 
diff --git a/be/src/vec/functions/function_width_bucket.cpp b/be/src/vec/functions/function_width_bucket.cpp
index 40c08a950e2588..193cf832b1ad80 100644
--- a/be/src/vec/functions/function_width_bucket.cpp
+++ b/be/src/vec/functions/function_width_bucket.cpp
@@ -136,18 +136,18 @@ class FunctionWidthBucket : public IFunction {
         } else if (which.is_float64()) {
             _execute<ColumnFloat64>(expr_column, min_value_column, max_value_column, num_buckets,
                                     nested_column_column);
-        } else if (which.is_decimal128()) {
-            _execute<ColumnDecimal128>(expr_column, min_value_column, max_value_column, num_buckets,
-                                       nested_column_column);
+        } else if (which.is_decimal128v2()) {
+            _execute<ColumnDecimal128V2>(expr_column, min_value_column, max_value_column,
+                                         num_buckets, nested_column_column);
         } else if (which.is_decimal32()) {
             _execute<ColumnDecimal32>(expr_column, min_value_column, max_value_column, num_buckets,
                                       nested_column_column);
         } else if (which.is_decimal64()) {
             _execute<ColumnDecimal64>(expr_column, min_value_column, max_value_column, num_buckets,
                                       nested_column_column);
-        } else if (which.is_decimal128i()) {
-            _execute<ColumnDecimal128I>(expr_column, min_value_column, max_value_column,
-                                        num_buckets, nested_column_column);
+        } else if (which.is_decimal128v3()) {
+            _execute<ColumnDecimal128V3>(expr_column, min_value_column, max_value_column,
+                                         num_buckets, nested_column_column);
         } else if (which.is_decimal256()) {
             _execute<ColumnDecimal256>(expr_column, min_value_column, max_value_column, num_buckets,
                                        nested_column_column);
diff --git a/be/src/vec/functions/least_greast.cpp b/be/src/vec/functions/least_greast.cpp
index 4c008da67f934a..9d8cd318b499cf 100644
--- a/be/src/vec/functions/least_greast.cpp
+++ b/be/src/vec/functions/least_greast.cpp
@@ -128,7 +128,7 @@ struct CompareMultiImpl {
         auto* __restrict column_raw_data =
                 reinterpret_cast<const ColumnType*>(argument_column.get())->get_data().data();
 
-        if constexpr (std::is_same_v<ColumnType, ColumnDecimal128>) {
+        if constexpr (std::is_same_v<ColumnType, ColumnDecimal128V2>) {
             for (size_t i = 0; i < input_rows_count; ++i) {
                 result_raw_data[i] =
                         Op<DecimalV2Value, DecimalV2Value>::apply(
@@ -138,7 +138,7 @@ struct CompareMultiImpl {
             }
         } else if constexpr (std::is_same_v<ColumnType, ColumnDecimal32> ||
                              std::is_same_v<ColumnType, ColumnDecimal64> ||
-                             std::is_same_v<ColumnType, ColumnDecimal128I> ||
+                             std::is_same_v<ColumnType, ColumnDecimal128V3> ||
                              std::is_same_v<ColumnType, ColumnDecimal256>) {
             for (size_t i = 0; i < input_rows_count; ++i) {
                 using type = std::decay_t<decltype(result_raw_data[0].value)>;
@@ -237,7 +237,7 @@ struct FunctionFieldImpl {
         auto [argument_column_raw, argument_column_is_const] = unpack_if_const(argument_column);
         const auto& arg_data =
                 assert_cast<const ColumnType&>(*argument_column_raw).get_data().data()[0];
-        if constexpr (std::is_same_v<ColumnType, ColumnDecimal128>) {
+        if constexpr (std::is_same_v<ColumnType, ColumnDecimal128V2>) {
             for (size_t i = 0; i < input_rows_count; ++i) {
                 res_data[i] |= (!res_data[i] *
                                 (EqualsOp<DecimalV2Value, DecimalV2Value>::apply(
@@ -246,7 +246,7 @@ struct FunctionFieldImpl {
             }
         } else if constexpr (std::is_same_v<ColumnType, ColumnDecimal32> ||
                              std::is_same_v<ColumnType, ColumnDecimal64> ||
-                             std::is_same_v<ColumnType, ColumnDecimal128I> ||
+                             std::is_same_v<ColumnType, ColumnDecimal128V3> ||
                              std::is_same_v<ColumnType, ColumnDecimal256>) {
             for (size_t i = 0; i < input_rows_count; ++i) {
                 using type = std::decay_t<decltype(first_raw_data[0].value)>;
diff --git a/be/src/vec/functions/multiply.cpp b/be/src/vec/functions/multiply.cpp
index 0dc9f4a410c9d7..43da8bb56ff6e4 100644
--- a/be/src/vec/functions/multiply.cpp
+++ b/be/src/vec/functions/multiply.cpp
@@ -53,9 +53,9 @@ struct MultiplyImpl {
     999999999999999999999999998000000000.000000000000000001 54 digits
     */
     template <bool check_overflow>
-    static void vector_vector(const ColumnDecimal128::Container::value_type* __restrict a,
-                              const ColumnDecimal128::Container::value_type* __restrict b,
-                              ColumnDecimal128::Container::value_type* c, size_t size) {
+    static void vector_vector(const ColumnDecimal128V2::Container::value_type* __restrict a,
+                              const ColumnDecimal128V2::Container::value_type* __restrict b,
+                              ColumnDecimal128V2::Container::value_type* c, size_t size) {
         auto sng_uptr = std::unique_ptr<int8[]>(new int8[size]);
         int8* sgn = sng_uptr.get();
         auto max = DecimalV2Value::get_max_decimal();
diff --git a/be/src/vec/io/io_helper.h b/be/src/vec/io/io_helper.h
index c9e52fe87a1312..b3c475c947f1aa 100644
--- a/be/src/vec/io/io_helper.h
+++ b/be/src/vec/io/io_helper.h
@@ -389,7 +389,7 @@ template <PrimitiveType P, typename T>
 StringParser::ParseResult read_decimal_text_impl(T& x, ReadBuffer& buf, UInt32 precision,
                                                  UInt32 scale) {
     static_assert(IsDecimalNumber<T>);
-    if constexpr (!std::is_same_v<Decimal128, T>) {
+    if constexpr (!std::is_same_v<Decimal128V2, T>) {
         StringParser::ParseResult result = StringParser::PARSE_SUCCESS;
 
         x.value = StringParser::string_to_decimal<P>((const char*)buf.position(), buf.count(),
diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp
index 2820cca53c6e8e..d2f1d431daa84d 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -131,7 +131,7 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
         return std::make_unique<OlapColumnDataConvertorDecimalV3<Decimal64>>();
     }
     case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: {
-        return std::make_unique<OlapColumnDataConvertorDecimalV3<Decimal128I>>();
+        return std::make_unique<OlapColumnDataConvertorDecimalV3<Decimal128V3>>();
     }
     case FieldType::OLAP_FIELD_TYPE_DECIMAL256: {
         return std::make_unique<OlapColumnDataConvertorDecimalV3<Decimal256>>();
@@ -838,14 +838,14 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDateTime::convert_to_olap(
 
 Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap() {
     assert(_typed_column.column);
-    const vectorized::ColumnDecimal<vectorized::Decimal128>* column_decimal = nullptr;
+    const vectorized::ColumnDecimal<vectorized::Decimal128V2>* column_decimal = nullptr;
     if (_nullmap) {
         auto nullable_column =
                 assert_cast<const vectorized::ColumnNullable*>(_typed_column.column.get());
-        column_decimal = assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128>*>(
+        column_decimal = assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128V2>*>(
                 nullable_column->get_nested_column_ptr().get());
     } else {
-        column_decimal = assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128>*>(
+        column_decimal = assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128V2>*>(
                 _typed_column.column.get());
     }
 
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp
index 336f627fcc3a5f..6a7e16a484122f 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -152,12 +152,12 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
             if (_output_vexpr_ctxs[col_idx]->root()->type().is_decimal_v2_type()) {
                 if (_output_vexpr_ctxs[col_idx]->root()->is_nullable()) {
                     auto nested_serde =
-                            std::make_shared<DataTypeDecimalSerDe<vectorized::Decimal128>>(scale,
-                                                                                           27);
+                            std::make_shared<DataTypeDecimalSerDe<vectorized::Decimal128V2>>(scale,
+                                                                                             27);
                     serde = std::make_shared<DataTypeNullableSerDe>(nested_serde);
                 } else {
-                    serde = std::make_shared<DataTypeDecimalSerDe<vectorized::Decimal128>>(scale,
-                                                                                           27);
+                    serde = std::make_shared<DataTypeDecimalSerDe<vectorized::Decimal128V2>>(scale,
+                                                                                             27);
                 }
             } else {
                 serde = block.get_by_position(col_idx).type->get_serde();
diff --git a/be/src/vec/sink/vtablet_block_convertor.cpp b/be/src/vec/sink/vtablet_block_convertor.cpp
index 5e2358fc371055..ab3060af3a81ca 100644
--- a/be/src/vec/sink/vtablet_block_convertor.cpp
+++ b/be/src/vec/sink/vtablet_block_convertor.cpp
@@ -137,7 +137,7 @@ DecimalType OlapTableBlockConvertor::_get_decimalv3_min_or_max(const TypeDescrip
         pmap = IsMin ? &_min_decimal32_val : &_max_decimal32_val;
     } else if constexpr (std::is_same_v<DecimalType, vectorized::Decimal64>) {
         pmap = IsMin ? &_min_decimal64_val : &_max_decimal64_val;
-    } else if constexpr (std::is_same_v<DecimalType, vectorized::Decimal128I>) {
+    } else if constexpr (std::is_same_v<DecimalType, vectorized::Decimal128V3>) {
         pmap = IsMin ? &_min_decimal128_val : &_max_decimal128_val;
     } else {
         pmap = IsMin ? &_min_decimal256_val : &_max_decimal256_val;
@@ -256,8 +256,8 @@ Status OlapTableBlockConvertor::_validate_column(RuntimeState* state, const Type
         break;
     }
     case TYPE_DECIMALV2: {
-        auto* column_decimal = const_cast<vectorized::ColumnDecimal<vectorized::Decimal128>*>(
-                assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128>*>(
+        auto* column_decimal = const_cast<vectorized::ColumnDecimal<vectorized::Decimal128V2>*>(
+                assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128V2>*>(
                         real_column_ptr.get()));
         const auto& max_decimalv2 = _get_decimalv2_min_or_max<false>(type);
         const auto& min_decimalv2 = _get_decimalv2_min_or_max<true>(type);
@@ -335,7 +335,7 @@ Status OlapTableBlockConvertor::_validate_column(RuntimeState* state, const Type
         break;
     }
     case TYPE_DECIMAL128I: {
-        CHECK_VALIDATION_FOR_DECIMALV3(vectorized::Decimal128I);
+        CHECK_VALIDATION_FOR_DECIMALV3(vectorized::Decimal128V3);
         break;
     }
     case TYPE_DECIMAL256: {
diff --git a/be/src/vec/sink/writer/vmysql_table_writer.cpp b/be/src/vec/sink/writer/vmysql_table_writer.cpp
index fdcedf5996210b..4a3513c80f40d8 100644
--- a/be/src/vec/sink/writer/vmysql_table_writer.cpp
+++ b/be/src/vec/sink/writer/vmysql_table_writer.cpp
@@ -196,7 +196,7 @@ Status VMysqlTableWriter::_insert_row(vectorized::Block& block, size_t row) {
         case TYPE_DECIMALV2: {
             DecimalV2Value value =
                     (DecimalV2Value)
-                            assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128>&>(
+                            assert_cast<const vectorized::ColumnDecimal<vectorized::Decimal128V2>&>(
                                     *column)
                                     .get_data()[row];
             fmt::format_to(_insert_stmt_buffer, "{}", value.to_string());
diff --git a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
index 619e575f96371c..189786a0ec3361 100644
--- a/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/column_reader_writer_test.cpp
@@ -542,7 +542,7 @@ static vectorized::MutableColumnPtr create_vectorized_column_ptr(FieldType type)
     } else if (type == FieldType::OLAP_FIELD_TYPE_DATETIME) {
         return vectorized::DataTypeDateTime().create_column();
     } else if (type == FieldType::OLAP_FIELD_TYPE_DECIMAL) {
-        return vectorized::DataTypeDecimal<vectorized::Decimal128>(27, 9).create_column();
+        return vectorized::DataTypeDecimal<vectorized::Decimal128V2>(27, 9).create_column();
     }
     return vectorized::DataTypeNothing().create_column();
 }
diff --git a/be/test/vec/aggregate_functions/agg_collect_test.cpp b/be/test/vec/aggregate_functions/agg_collect_test.cpp
index cb750f1a0c53e8..d592967cbe6d42 100644
--- a/be/test/vec/aggregate_functions/agg_collect_test.cpp
+++ b/be/test/vec/aggregate_functions/agg_collect_test.cpp
@@ -147,8 +147,8 @@ TEST_F(VAggCollectTest, test_empty) {
     test_agg_collect<DataTypeInt128>("collect_list");
     test_agg_collect<DataTypeInt128>("collect_set");
 
-    test_agg_collect<DataTypeDecimal<Decimal128>>("collect_list");
-    test_agg_collect<DataTypeDecimal<Decimal128>>("collect_set");
+    test_agg_collect<DataTypeDecimal<Decimal128V2>>("collect_list");
+    test_agg_collect<DataTypeDecimal<Decimal128V2>>("collect_set");
 
     test_agg_collect<DataTypeDate>("collect_list");
     test_agg_collect<DataTypeDate>("collect_set");
@@ -163,8 +163,8 @@ TEST_F(VAggCollectTest, test_with_data) {
     test_agg_collect<DataTypeInt128>("collect_list", 20);
     test_agg_collect<DataTypeInt128>("collect_set", 30);
 
-    test_agg_collect<DataTypeDecimal<Decimal128>>("collect_list", 10);
-    test_agg_collect<DataTypeDecimal<Decimal128>>("collect_set", 11);
+    test_agg_collect<DataTypeDecimal<Decimal128V2>>("collect_list", 10);
+    test_agg_collect<DataTypeDecimal<Decimal128V2>>("collect_set", 11);
 
     test_agg_collect<DataTypeDateTime>("collect_list", 5);
     test_agg_collect<DataTypeDateTime>("collect_set", 6);
diff --git a/be/test/vec/aggregate_functions/agg_histogram_test.cpp b/be/test/vec/aggregate_functions/agg_histogram_test.cpp
index 7d7037f6c6b04d..069b8762952149 100644
--- a/be/test/vec/aggregate_functions/agg_histogram_test.cpp
+++ b/be/test/vec/aggregate_functions/agg_histogram_test.cpp
@@ -218,7 +218,7 @@ TEST_F(VAggHistogramTest, test_empty) {
     test_agg_histogram<DataTypeDate>();
     test_agg_histogram<DataTypeDateTime>();
     test_agg_histogram<DataTypeString>();
-    test_agg_histogram<DataTypeDecimal<Decimal128>>();
+    test_agg_histogram<DataTypeDecimal<Decimal128V2>>();
 }
 
 TEST_F(VAggHistogramTest, test_with_data) {
@@ -238,7 +238,7 @@ TEST_F(VAggHistogramTest, test_with_data) {
     test_agg_histogram<DataTypeDateTime>(100, 5);
     test_agg_histogram<DataTypeDateTimeV2>(100, 5);
 
-    test_agg_histogram<DataTypeDecimal<Decimal128>>(100, 5);
+    test_agg_histogram<DataTypeDecimal<Decimal128V2>>(100, 5);
 }
 
 } // namespace doris::vectorized
diff --git a/be/test/vec/aggregate_functions/agg_min_max_test.cpp b/be/test/vec/aggregate_functions/agg_min_max_test.cpp
index aaff93d5e607bb..fef91af4d1f63b 100644
--- a/be/test/vec/aggregate_functions/agg_min_max_test.cpp
+++ b/be/test/vec/aggregate_functions/agg_min_max_test.cpp
@@ -78,12 +78,12 @@ TEST_P(AggMinMaxTest, min_max_test) {
 
 TEST_P(AggMinMaxTest, min_max_decimal_test) {
     std::string min_max_type = GetParam();
-    auto data_type = std::make_shared<DataTypeDecimal<Decimal128>>();
+    auto data_type = std::make_shared<DataTypeDecimal<Decimal128V2>>();
     // Prepare test data.
     auto column_vector_decimal128 = data_type->create_column();
     for (int i = 0; i < agg_test_batch_size; i++) {
         column_vector_decimal128->insert(
-                cast_to_nearest_field_type(DecimalField<Decimal128>(Decimal128(i), 9)));
+                cast_to_nearest_field_type(DecimalField<Decimal128V2>(Decimal128V2(i), 9)));
     }
 
     // Prepare test function and parameters.
@@ -102,7 +102,7 @@ TEST_P(AggMinMaxTest, min_max_decimal_test) {
     }
 
     // Check result.
-    ColumnDecimal128 ans(0, 9);
+    ColumnDecimal128V2 ans(0, 9);
     agg_function->insert_result_into(place, ans);
     EXPECT_EQ(min_max_type == "min" ? 0 : agg_test_batch_size - 1, ans.get_element(0).value);
     agg_function->destroy(place);
@@ -114,7 +114,7 @@ TEST_P(AggMinMaxTest, min_max_decimal_test) {
     AggregateDataPtr places = memory2.get();
     agg_function->deserialize_from_column(places, *dst, nullptr, agg_test_batch_size);
 
-    ColumnDecimal128 result(0, 9);
+    ColumnDecimal128V2 result(0, 9);
     for (size_t i = 0; i != agg_test_batch_size; ++i) {
         agg_function->insert_result_into(places + agg_function->size_of_data() * i, result);
     }
diff --git a/be/test/vec/aggregate_functions/agg_replace_test.cpp b/be/test/vec/aggregate_functions/agg_replace_test.cpp
index 5bc50b149d8a2a..092b7e4fe6a2b8 100644
--- a/be/test/vec/aggregate_functions/agg_replace_test.cpp
+++ b/be/test/vec/aggregate_functions/agg_replace_test.cpp
@@ -364,7 +364,7 @@ TEST_F(VAggReplaceTest, test_basic_data) {
     test_basic_data<DataTypeInt32, ColumnInt32, false>(11);
     test_basic_data<DataTypeInt64, ColumnInt64, false>(11);
     test_basic_data<DataTypeInt128, ColumnInt128, false>(11);
-    test_basic_data<DataTypeDecimal<Decimal128>, ColumnDecimal128, false>(11);
+    test_basic_data<DataTypeDecimal<Decimal128V2>, ColumnDecimal128V2, false>(11);
     test_basic_data<DataTypeString, ColumnString, false>(11);
     test_basic_data<DataTypeInt128, ColumnInt128, false>(11);
     test_basic_data<DataTypeDate, ColumnDate, false>(11);
@@ -377,7 +377,7 @@ TEST_F(VAggReplaceTest, test_array_data) {
     test_array_data<DataTypeInt32, ColumnArray, false>(11);
     test_array_data<DataTypeInt64, ColumnArray, false>(11);
     test_array_data<DataTypeInt128, ColumnArray, false>(11);
-    test_array_data<DataTypeDecimal<Decimal128>, ColumnArray, false>(11);
+    test_array_data<DataTypeDecimal<Decimal128V2>, ColumnArray, false>(11);
     test_array_data<DataTypeString, ColumnArray, false>(11);
     test_array_data<DataTypeInt128, ColumnArray, false>(11);
     test_array_data<DataTypeDate, ColumnArray, false>(11);
@@ -390,7 +390,7 @@ TEST_F(VAggReplaceTest, test_basic_replace_reader) {
     test_agg_replace<DataTypeInt32, false>("replace_reader", 10, 0);
     test_agg_replace<DataTypeInt64, false>("replace_reader", 10, 0);
     test_agg_replace<DataTypeInt128, false>("replace_reader", 10, 0);
-    test_agg_replace<DataTypeDecimal<Decimal128>, false>("replace_reader", 10, 0);
+    test_agg_replace<DataTypeDecimal<Decimal128V2>, false>("replace_reader", 10, 0);
     test_agg_replace<DataTypeString, false>("replace_reader", 10, 0);
     test_agg_replace<DataTypeDate, false>("replace_reader", 10, 0);
     test_agg_replace<DataTypeDateTime, false>("replace_reader", 10, 0);
@@ -403,7 +403,7 @@ TEST_F(VAggReplaceTest, test_basic_replace_reader) {
     test_agg_replace<DataTypeInt32, true>("replace_reader", 10, 0);
     test_agg_replace<DataTypeInt64, true>("replace_reader", 10, 0);
     test_agg_replace<DataTypeInt128, true>("replace_reader", 10, 0);
-    test_agg_replace<DataTypeDecimal<Decimal128>, true>("replace_reader", 10, 0);
+    test_agg_replace<DataTypeDecimal<Decimal128V2>, true>("replace_reader", 10, 0);
     test_agg_replace<DataTypeString, true>("replace_reader", 10, 0);
     test_agg_replace<DataTypeDate, true>("replace_reader", 10, 0);
     test_agg_replace<DataTypeDateTime, true>("replace_reader", 10, 0);
@@ -417,7 +417,7 @@ TEST_F(VAggReplaceTest, test_basic_replace_load) {
     test_agg_replace<DataTypeInt32, false>("replace_load", 10, 9);
     test_agg_replace<DataTypeInt64, false>("replace_load", 10, 9);
     test_agg_replace<DataTypeInt128, false>("replace_load", 10, 9);
-    test_agg_replace<DataTypeDecimal<Decimal128>, false>("replace_load", 10, 9);
+    test_agg_replace<DataTypeDecimal<Decimal128V2>, false>("replace_load", 10, 9);
     test_agg_replace<DataTypeString, false>("replace_load", 10, 9);
     test_agg_replace<DataTypeDate, false>("replace_load", 10, 9);
     test_agg_replace<DataTypeDateTime, false>("replace_load", 10, 9);
@@ -429,7 +429,7 @@ TEST_F(VAggReplaceTest, test_basic_replace_load) {
     test_agg_replace<DataTypeInt32, true>("replace_load", 10, 9);
     test_agg_replace<DataTypeInt64, true>("replace_load", 10, 9);
     test_agg_replace<DataTypeInt128, true>("replace_load", 10, 9);
-    test_agg_replace<DataTypeDecimal<Decimal128>, true>("replace_load", 10, 9);
+    test_agg_replace<DataTypeDecimal<Decimal128V2>, true>("replace_load", 10, 9);
     test_agg_replace<DataTypeString, true>("replace_load", 10, 9);
     test_agg_replace<DataTypeDate, true>("replace_load", 10, 9);
     test_agg_replace<DataTypeDateTime, true>("replace_load", 10, 9);
@@ -443,7 +443,7 @@ TEST_F(VAggReplaceTest, test_array_replace_reader) {
     test_agg_array_replace<DataTypeInt32, false>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeInt64, false>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeInt128, false>("replace_reader", 10, 0);
-    test_agg_array_replace<DataTypeDecimal<Decimal128>, false>("replace_reader", 10, 0);
+    test_agg_array_replace<DataTypeDecimal<Decimal128V2>, false>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeString, false>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeDate, false>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeDateTime, false>("replace_reader", 10, 0);
@@ -453,7 +453,7 @@ TEST_F(VAggReplaceTest, test_array_replace_reader) {
     test_agg_array_replace<DataTypeInt32, true>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeInt64, true>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeInt128, true>("replace_reader", 10, 0);
-    test_agg_array_replace<DataTypeDecimal<Decimal128>, true>("replace_reader", 10, 0);
+    test_agg_array_replace<DataTypeDecimal<Decimal128V2>, true>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeString, true>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeDate, true>("replace_reader", 10, 0);
     test_agg_array_replace<DataTypeDateTime, true>("replace_reader", 10, 0);
@@ -465,7 +465,7 @@ TEST_F(VAggReplaceTest, test_array_replace_load) {
     test_agg_array_replace<DataTypeInt32, false>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeInt64, false>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeInt128, false>("replace_load", 10, 10);
-    test_agg_array_replace<DataTypeDecimal<Decimal128>, false>("replace_load", 10, 10);
+    test_agg_array_replace<DataTypeDecimal<Decimal128V2>, false>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeString, false>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeDate, false>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeDateTime, false>("replace_load", 10, 10);
@@ -475,7 +475,7 @@ TEST_F(VAggReplaceTest, test_array_replace_load) {
     test_agg_array_replace<DataTypeInt32, true>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeInt64, true>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeInt128, true>("replace_load", 10, 10);
-    test_agg_array_replace<DataTypeDecimal<Decimal128>, true>("replace_load", 10, 10);
+    test_agg_array_replace<DataTypeDecimal<Decimal128V2>, true>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeString, true>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeDate, true>("replace_load", 10, 10);
     test_agg_array_replace<DataTypeDateTime, true>("replace_load", 10, 10);
diff --git a/be/test/vec/columns/column_hash_func_test.cpp b/be/test/vec/columns/column_hash_func_test.cpp
index bdde0f33a62e97..a95c7071af5684 100644
--- a/be/test/vec/columns/column_hash_func_test.cpp
+++ b/be/test/vec/columns/column_hash_func_test.cpp
@@ -42,7 +42,7 @@ DataTypes create_scala_data_types() {
     DataTypePtr d = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeDate>());
     DataTypePtr dc = std::make_shared<DataTypeNullable>(vectorized::create_decimal(10, 2, false));
     DataTypePtr dcv2 = std::make_shared<DataTypeNullable>(
-            std::make_shared<DataTypeDecimal<vectorized::Decimal128>>(27, 9));
+            std::make_shared<DataTypeDecimal<vectorized::Decimal128V2>>(27, 9));
     DataTypePtr n3 = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt128>());
     DataTypePtr n1 = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>());
     DataTypePtr s1 = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
diff --git a/be/test/vec/columns/column_resize_test.cpp b/be/test/vec/columns/column_resize_test.cpp
index 809ceb5c5c14e9..dd6f78f3e8d2f0 100644
--- a/be/test/vec/columns/column_resize_test.cpp
+++ b/be/test/vec/columns/column_resize_test.cpp
@@ -44,7 +44,7 @@ DataTypes _create_scala_data_types() {
     DataTypePtr d = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeDate>());
     DataTypePtr dc = std::make_shared<DataTypeNullable>(vectorized::create_decimal(10, 2, false));
     DataTypePtr dcv2 = std::make_shared<DataTypeNullable>(
-            std::make_shared<DataTypeDecimal<vectorized::Decimal128>>(27, 9));
+            std::make_shared<DataTypeDecimal<vectorized::Decimal128V2>>(27, 9));
     DataTypePtr n3 = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt128>());
     DataTypePtr n1 = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>());
     DataTypePtr s1 = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>());
diff --git a/be/test/vec/exprs/vexpr_test.cpp b/be/test/vec/exprs/vexpr_test.cpp
index 81396c9e95be12..94cec355cf4ac7 100644
--- a/be/test/vec/exprs/vexpr_test.cpp
+++ b/be/test/vec/exprs/vexpr_test.cpp
@@ -557,7 +557,7 @@ TEST(TEST_VEXPR, LITERALTEST) {
         int ret = -1;
         static_cast<void>(literal.execute(nullptr, &block, &ret));
         auto ctn = block.safe_get_by_position(ret);
-        auto v = (*ctn.column)[0].get<DecimalField<Decimal128>>();
+        auto v = (*ctn.column)[0].get<DecimalField<Decimal128V2>>();
         EXPECT_FLOAT_EQ(((double)v.get_value()) / (std::pow(10, v.get_scale())), 1234.56);
         EXPECT_EQ("1234.560000000", literal.value());
     }
diff --git a/be/test/vec/function/function_array_element_test.cpp b/be/test/vec/function/function_array_element_test.cpp
index 1b4a55ea3803b7..2d89164f55f498 100644
--- a/be/test/vec/function/function_array_element_test.cpp
+++ b/be/test/vec/function/function_array_element_test.cpp
@@ -130,9 +130,9 @@ TEST(function_array_element_test, element_at) {
         static_cast<void>(check_function<DataTypeDate, true>(func_name, input_types, data_set));
     }
 
-    // element_at(Array<Decimal128>, Int64)
+    // element_at(Array<Decimal128V2>, Int64)
     {
-        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128, TypeIndex::Int64};
+        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128V2, TypeIndex::Int64};
 
         Array vec = {ut_type::DECIMALFIELD(17014116.67), ut_type::DECIMALFIELD(-17014116.67),
                      ut_type::DECIMALFIELD(0.0)};
@@ -146,8 +146,8 @@ TEST(function_array_element_test, element_at) {
                             {{empty_arr, Int64(0)}, Null()},
                             {{empty_arr, Int64(1)}, Null()}};
 
-        static_cast<void>(check_function<DataTypeDecimal<Decimal128>, true>(func_name, input_types,
-                                                                            data_set));
+        static_cast<void>(check_function<DataTypeDecimal<Decimal128V2>, true>(
+                func_name, input_types, data_set));
     }
 
     // element_at(Array<String>, Int32)
diff --git a/be/test/vec/function/function_array_index_test.cpp b/be/test/vec/function/function_array_index_test.cpp
index 0953f0f6f70740..7f496dc1ccee09 100644
--- a/be/test/vec/function/function_array_index_test.cpp
+++ b/be/test/vec/function/function_array_index_test.cpp
@@ -140,9 +140,10 @@ TEST(function_array_index_test, array_contains) {
         static_cast<void>(check_function<DataTypeUInt8, true>(func_name, input_types, data_set));
     }
 
-    // array_contains(Array<Decimal128>, Decimal128)
+    // array_contains(Array<Decimal128V2>, Decimal128V2)
     {
-        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128, TypeIndex::Decimal128};
+        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128V2,
+                                    TypeIndex::Decimal128V2};
 
         Array vec = {ut_type::DECIMALFIELD(17014116.67), ut_type::DECIMALFIELD(-17014116.67),
                      ut_type::DECIMALFIELD(0.0)};
@@ -242,9 +243,10 @@ TEST(function_array_index_test, array_position) {
         static_cast<void>(check_function<DataTypeInt64, true>(func_name, input_types, data_set));
     }
 
-    // array_position(Array<Decimal128>, Decimal128)
+    // array_position(Array<Decimal128V2>, Decimal128V2)
     {
-        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128, TypeIndex::Decimal128};
+        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128V2,
+                                    TypeIndex::Decimal128V2};
 
         Array vec = {ut_type::DECIMALFIELD(17014116.67), ut_type::DECIMALFIELD(-17014116.67),
                      ut_type::DECIMALFIELD(0)};
diff --git a/be/test/vec/function/function_arrays_overlap_test.cpp b/be/test/vec/function/function_arrays_overlap_test.cpp
index 6811f6e6e43dbf..4a13d41b0a3121 100644
--- a/be/test/vec/function/function_arrays_overlap_test.cpp
+++ b/be/test/vec/function/function_arrays_overlap_test.cpp
@@ -105,10 +105,10 @@ TEST(function_arrays_overlap_test, arrays_overlap) {
         static_cast<void>(check_function<DataTypeUInt8, true>(func_name, input_types, data_set));
     }
 
-    // arrays_overlap(Array<Decimal128>, Array<Decimal128>)
+    // arrays_overlap(Array<Decimal128V2>, Array<Decimal128V2>)
     {
-        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128, TypeIndex::Array,
-                                    TypeIndex::Decimal128};
+        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128V2, TypeIndex::Array,
+                                    TypeIndex::Decimal128V2};
 
         Array vec1 = {ut_type::DECIMALFIELD(17014116.67), ut_type::DECIMALFIELD(-17014116.67),
                       ut_type::DECIMALFIELD(0.0)};
diff --git a/be/test/vec/function/function_math_test.cpp b/be/test/vec/function/function_math_test.cpp
index 4fbee8289be57d..c93c6ca324fdc6 100644
--- a/be/test/vec/function/function_math_test.cpp
+++ b/be/test/vec/function/function_math_test.cpp
@@ -516,7 +516,7 @@ TEST(MathFunctionTest, money_format_test) {
         static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
     }
     {
-        InputTypeSet input_types = {TypeIndex::Decimal128};
+        InputTypeSet input_types = {TypeIndex::Decimal128V2};
         DataSet data_set = {{{Null()}, Null()},
                             {{DECIMAL(17014116.67)}, VARCHAR("17,014,116.67")},
                             {{DECIMAL(-17014116.67)}, VARCHAR("-17,014,116.67")}};
diff --git a/be/test/vec/function/function_test_util.cpp b/be/test/vec/function/function_test_util.cpp
index 4ae7907921f253..50d1d3ab090e9d 100644
--- a/be/test/vec/function/function_test_util.cpp
+++ b/be/test/vec/function/function_test_util.cpp
@@ -130,9 +130,9 @@ size_t type_index_to_data_type(const std::vector<AnyType>& input_types, size_t i
         desc.type = doris::PrimitiveType::TYPE_DOUBLE;
         type = std::make_shared<DataTypeFloat64>();
         return 1;
-    case TypeIndex::Decimal128:
+    case TypeIndex::Decimal128V2:
         desc.type = doris::PrimitiveType::TYPE_DECIMALV2;
-        type = std::make_shared<DataTypeDecimal<Decimal128>>();
+        type = std::make_shared<DataTypeDecimal<Decimal128V2>>();
         return 1;
     case TypeIndex::DateTime:
         desc.type = doris::PrimitiveType::TYPE_DATETIME;
@@ -266,7 +266,7 @@ bool insert_cell(MutableColumnPtr& column, DataTypePtr type_ptr, const AnyType&
     } else if (type.is_float64()) {
         auto value = any_cast<ut_type::DOUBLE>(cell);
         column->insert_data(reinterpret_cast<char*>(&value), 0);
-    } else if (type.is_decimal128()) {
+    } else if (type.is_decimal128v2()) {
         auto value = any_cast<Decimal<Int128>>(cell);
         column->insert_data(reinterpret_cast<char*>(&value), 0);
     } else if (type.is_date_time()) {
diff --git a/be/test/vec/function/function_test_util.h b/be/test/vec/function/function_test_util.h
index 62e4f6fd000cd9..f488690f51dbdc 100644
--- a/be/test/vec/function/function_test_util.h
+++ b/be/test/vec/function/function_test_util.h
@@ -102,9 +102,9 @@ using STRING = std::string;
 using DOUBLE = double;
 using FLOAT = float;
 
-inline auto DECIMAL = Decimal128::double_to_decimal;
+inline auto DECIMAL = Decimal128V2::double_to_decimal;
 inline auto DECIMALFIELD = [](double v) {
-    return DecimalField<Decimal128>(Decimal128::double_to_decimal(v), 9);
+    return DecimalField<Decimal128V2>(Decimal128V2::double_to_decimal(v), 9);
 };
 
 using DATETIME = std::string;
@@ -315,8 +315,8 @@ Status check_function(const std::string& func_name, const InputTypeSet& input_ty
                 const auto& expect_data =
                         any_cast<typename ReturnType::FieldType>(data_set[i].second);
 
-                if constexpr (std::is_same_v<ReturnType, DataTypeDecimal<Decimal128>>) {
-                    const auto& column_data = field.get<DecimalField<Decimal128>>().get_value();
+                if constexpr (std::is_same_v<ReturnType, DataTypeDecimal<Decimal128V2>>) {
+                    const auto& column_data = field.get<DecimalField<Decimal128V2>>().get_value();
                     EXPECT_EQ(expect_data.value, column_data.value) << " at row " << i;
                 } else if constexpr (std::is_same_v<ReturnType, DataTypeBitMap>) {
                     const ColumnBitmap* bitmap_col = nullptr;
diff --git a/be/test/vec/function/table_function_test.cpp b/be/test/vec/function/table_function_test.cpp
index 0faaa1515c5e45..95983e4d31d1ff 100644
--- a/be/test/vec/function/table_function_test.cpp
+++ b/be/test/vec/function/table_function_test.cpp
@@ -111,11 +111,11 @@ TEST_F(TableFunctionTest, vexplode_outer) {
 
     // explode_outer(Array<Decimal>)
     {
-        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128};
+        InputTypeSet input_types = {TypeIndex::Array, TypeIndex::Decimal128V2};
         Array vec = {ut_type::DECIMALFIELD(17014116.67), ut_type::DECIMALFIELD(-17014116.67)};
         InputDataSet input_set = {{Null()}, {Array()}, {vec}};
 
-        InputTypeSet output_types = {TypeIndex::Decimal128};
+        InputTypeSet output_types = {TypeIndex::Decimal128V2};
         InputDataSet output_set = {{Null()},
                                    {Null()},
                                    {ut_type::DECIMAL(17014116.67)},
diff --git a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
index 58ae47f75ba89f..152941c5a95c48 100644
--- a/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
+++ b/be/test/vec/utils/arrow_column_to_doris_column_test.cpp
@@ -383,7 +383,7 @@ void test_arrow_to_decimal_column(std::shared_ptr<arrow::Decimal128Type> type,
         data_column = (*std::move(column.column)).mutate();
     }
     auto& decimal_data =
-            static_cast<ColumnDecimal<vectorized::Decimal128>&>(*data_column).get_data();
+            static_cast<ColumnDecimal<vectorized::Decimal128V2>&>(*data_column).get_data();
     for (auto i = 0; i < num_elements; ++i) {
         auto idx = counter - num_elements + i;
         if (is_nullable) {

From 253846a99debe5377aaa66da7089308cbf0b87a2 Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Fri, 29 Dec 2023 10:13:08 +0800
Subject: [PATCH 147/299] [improve](compaction)  enable compaction priority
 scheduling (#29261)

---
 be/src/common/config.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 6ebfafce53ce50..1d297ece456cd5 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -432,7 +432,7 @@ DEFINE_mBool(disable_compaction_trace_log, "true");
 DEFINE_mInt64(pick_rowset_to_compact_interval_sec, "86400");
 
 // Compaction priority schedule
-DEFINE_mBool(enable_compaction_priority_scheduling, "false");
+DEFINE_mBool(enable_compaction_priority_scheduling, "true");
 DEFINE_mInt32(low_priority_compaction_task_num_per_disk, "1");
 DEFINE_mDouble(low_priority_tablet_version_num_ratio, "0.7");
 

From 48d41a8c8b76ce22a8a9235e92862b06a5171aa6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= <jianxie0@gmail.com>
Date: Fri, 29 Dec 2023 10:38:53 +0800
Subject: [PATCH 148/299] [feature](Nereids): support comparing mv with
 inferred predicate (#29132)

---
 .../exploration/mv/ComparisonResult.java      | 11 +++
 .../exploration/mv/HyperGraphComparator.java  | 16 +++--
 .../hypergraph/CompareOuterJoinTest.java      |  2 -
 .../hypergraph/InferPredicateTest.java        | 72 +++++++++++++++++++
 .../hypergraph/PullupExpressionTest.java      |  3 +-
 .../aggregate_with_roll_up.groovy             |  2 +-
 .../aggregate_without_roll_up.groovy          |  2 +-
 .../mv/join/inner/inner_join.groovy           |  2 +-
 .../mv/join/left_outer/outer_join.groovy      |  2 +-
 .../mv/partition_mv_rewrite.groovy            |  2 +-
 10 files changed, 101 insertions(+), 13 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
index 8836745465ec02..1eb49cbfc0e0b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/ComparisonResult.java
@@ -116,4 +116,15 @@ public ComparisonResult build() {
                     viewNoNullableSlotBuilder.build(), valid);
         }
     }
+
+    @Override
+    public String toString() {
+        if (isInvalid()) {
+            return "INVALID";
+        }
+        return String.format("viewExpressions: %s \n "
+                + "queryExpressions :%s \n "
+                + "viewNoNullableSlot :%s \n",
+                viewExpressions, queryExpressions, viewNoNullableSlot);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
index a869fe729a94f1..04efecc9c25fbb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java
@@ -27,6 +27,7 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.util.ExpressionUtils;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -41,6 +42,7 @@
 import java.util.Map.Entry;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * HyperGraphComparator
@@ -108,16 +110,22 @@ private ComparisonResult isLogicCompatible() {
     private ComparisonResult buildComparisonRes() {
         ComparisonResult.Builder builder = new ComparisonResult.Builder();
         for (Entry<Edge, List<? extends Expression>> e : pullUpQueryExprWithEdge.entrySet()) {
-            if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) {
+            List<? extends Expression> rawFilter = e.getValue().stream()
+                    .filter(expr -> !ExpressionUtils.isInferred(expr))
+                    .collect(Collectors.toList());
+            if (!rawFilter.isEmpty() && !canPullUp(e.getKey())) {
                 return ComparisonResult.INVALID;
             }
-            builder.addQueryExpressions(e.getValue());
+            builder.addQueryExpressions(rawFilter);
         }
         for (Entry<Edge, List<? extends Expression>> e : pullUpViewExprWithEdge.entrySet()) {
-            if (!e.getValue().isEmpty() && !canPullUp(e.getKey())) {
+            List<? extends Expression> rawFilter = e.getValue().stream()
+                    .filter(expr -> !ExpressionUtils.isInferred(expr))
+                    .collect(Collectors.toList());
+            if (!rawFilter.isEmpty() && !canPullUp(e.getKey())) {
                 return ComparisonResult.INVALID;
             }
-            builder.addViewExpressions(e.getValue());
+            builder.addViewExpressions(rawFilter);
         }
         for (Pair<JoinType, Set<Slot>> inferredCond : inferredViewEdgeMap.values()) {
             builder.addViewNoNullableSlot(inferredCond.second);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
index 105a5d450c3a81..8914f4ac80b407 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/CompareOuterJoinTest.java
@@ -33,7 +33,6 @@
 import org.apache.doris.nereids.util.PlanChecker;
 
 import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 import java.util.List;
@@ -113,7 +112,6 @@ void testInnerJoinWithFilter() {
         Assertions.assertEquals("(id = 0)", res.getQueryExpressions().get(0).toSql());
     }
 
-    @Disabled
     @Test
     void testInnerJoinWithFilter2() {
         connectContext.getSessionVariable().setDisableNereidsRules("INFER_PREDICATES");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
new file mode 100644
index 00000000000000..bd18f31a7c055f
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/InferPredicateTest.java
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.jobs.joinorder.hypergraph;
+
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.RuleSet;
+import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule;
+import org.apache.doris.nereids.rules.exploration.mv.ComparisonResult;
+import org.apache.doris.nereids.rules.exploration.mv.HyperGraphComparator;
+import org.apache.doris.nereids.rules.exploration.mv.LogicalCompatibilityContext;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping;
+import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
+import org.apache.doris.nereids.sqltest.SqlTestBase;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class InferPredicateTest extends SqlTestBase {
+    @Test
+    void testPullUpQueryFilter() {
+        CascadesContext c1 = createCascadesContext(
+                "select * from T1 left join T2 on T1.id = T2.id where T1.id = 1",
+                connectContext
+        );
+        Plan p1 = PlanChecker.from(c1)
+                .analyze()
+                .rewrite()
+                .getPlan().child(0);
+        CascadesContext c2 = createCascadesContext(
+                "select * from T1 left join T2 on T1.id = T2.id",
+                connectContext
+        );
+        Plan p2 = PlanChecker.from(c2)
+                .analyze()
+                .rewrite()
+                .applyExploration(RuleSet.BUSHY_TREE_JOIN_REORDER)
+                .getAllPlan().get(0).child(0);
+        HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
+        HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
+        ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
+        Assertions.assertFalse(res.isInvalid());
+        Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql());
+    }
+
+    LogicalCompatibilityContext constructContext(Plan p1, Plan p2) {
+        StructInfo st1 = AbstractMaterializedViewRule.extractStructInfo(p1,
+                null).get(0);
+        StructInfo st2 = AbstractMaterializedViewRule.extractStructInfo(p2,
+                null).get(0);
+        RelationMapping rm = RelationMapping.generate(st1.getRelations(), st2.getRelations()).get(0);
+        SlotMapping sm = SlotMapping.generate(rm);
+        return LogicalCompatibilityContext.from(rm, sm, st1, st2);
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
index 4da4905f8aa85a..5489f06379c530 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/PullupExpressionTest.java
@@ -56,9 +56,8 @@ void testPullUpQueryFilter() {
         HyperGraph h1 = HyperGraph.toStructInfo(p1).get(0);
         HyperGraph h2 = HyperGraph.toStructInfo(p2).get(0);
         ComparisonResult res = HyperGraphComparator.isLogicCompatible(h1, h2, constructContext(p1, p2));
-        Assertions.assertEquals(2, res.getQueryExpressions().size());
+        Assertions.assertEquals(1, res.getQueryExpressions().size());
         Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(0).toSql());
-        Assertions.assertEquals("(id = 1)", res.getQueryExpressions().get(1).toSql());
     }
 
     @Test
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
index d6b5c37e59c095..fd3c02408d954d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
@@ -23,7 +23,7 @@ suite("aggregate_with_roll_up") {
     sql "SET enable_materialized_view_rewrite=true"
     sql "SET enable_nereids_timeout = false"
     // tmp disable to rewrite, will be removed in the future
-    sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+    sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
 
     sql """
     drop table if exists orders
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
index 5d40aa03138a6d..65da58cd5bd77c 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_without_roll_up/aggregate_without_roll_up.groovy
@@ -23,7 +23,7 @@ suite("aggregate_without_roll_up") {
     sql "SET enable_materialized_view_rewrite=true"
     sql "SET enable_nereids_timeout = false"
     // tmp disable to rewrite, will be removed in the future
-    sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+    sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
     sql "SET global enable_auto_analyze = false"
 
     sql """
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
index ccf6a83f2c610d..80ca73df03b2d7 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/inner/inner_join.groovy
@@ -23,7 +23,7 @@ suite("inner_join") {
     sql "SET enable_materialized_view_rewrite=true"
     sql "SET enable_nereids_timeout = false"
     // tmp disable to rewrite, will be removed in the future
-    sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+    sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
 
     sql """
     drop table if exists orders
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
index e8b1da2572a22d..37cf716c9428aa 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
@@ -23,7 +23,7 @@ suite("outer_join") {
     sql "SET enable_materialized_view_rewrite=true"
     sql "SET enable_nereids_timeout = false"
     // tmp disable to rewrite, will be removed in the future
-    sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+    sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
 
     sql """
     drop table if exists orders
diff --git a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
index e0144fe636993f..6da7ad89b77b25 100644
--- a/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/partition_mv_rewrite.groovy
@@ -23,7 +23,7 @@ suite("partition_mv_rewrite") {
     sql "SET enable_materialized_view_rewrite=true"
     sql "SET enable_nereids_timeout = false"
     // tmp disable to rewrite, will be removed in the future
-    sql "SET disable_nereids_rules = 'INFER_PREDICATES, ELIMINATE_OUTER_JOIN'"
+    sql "SET disable_nereids_rules = 'ELIMINATE_OUTER_JOIN'"
 
     sql """
     drop table if exists orders

From 3cddb597c60937a90cbc2a063d71df1ad1c4ae91 Mon Sep 17 00:00:00 2001
From: HowardQin <hao.qin@esgyn.cn>
Date: Fri, 29 Dec 2023 11:06:20 +0800
Subject: [PATCH 149/299] [fix](regression) fix pipeline load regression test
 error (#29216)

Co-authored-by: qinhao <qinhao@newland.com.cn>
---
 .../suites/load_p0/stream_load/test_pipeline_load.groovy        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy b/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
index 27940760bfdfdc..d85bc8a2fed5c8 100644
--- a/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_pipeline_load.groovy
@@ -138,7 +138,7 @@ suite("test_pipeline_load", "nonConcurrent") {
             "disable_auto_compaction" = "false"
             );
         """
-        // Trigger error which is different from non-pipeline load
+        // The error should be different from non-pipeline load
         streamLoad {
             table "${tableName}"
             set 'column_separator', ','

From a57ad36c4e8e1ac983210a3f745e56fbe8b35f4a Mon Sep 17 00:00:00 2001
From: KassieZ <139741991+KassieZ@users.noreply.github.com>
Date: Fri, 29 Dec 2023 11:37:50 +0800
Subject: [PATCH 150/299] [fix](docs) Update Link for 404 page of dev docs
 (#29213)

---
 .../advanced/best-practice/import-analysis.md |  2 +-
 docs/en/docs/advanced/broker.md               |  6 ++---
 .../import-scenes/external-storage-load.md    |  8 +++----
 .../import/import-scenes/load-atomicity.md    |  4 ++--
 .../import/import-scenes/load-data-convert.md |  6 ++---
 .../import/import-scenes/load-strict-mode.md  | 10 ++++----
 .../import/import-way/broker-load-manual.md   | 24 +++++++++----------
 docs/zh-CN/docs/advanced/broker.md            |  6 ++---
 .../import-scenes/external-storage-load.md    |  8 +++----
 .../import/import-scenes/load-data-convert.md |  2 +-
 .../import/import-scenes/load-strict-mode.md  |  2 +-
 .../import/import-way/broker-load-manual.md   | 24 +++++++++----------
 .../update-delete/batch-delete-manual.md      |  2 +-
 13 files changed, 52 insertions(+), 52 deletions(-)

diff --git a/docs/en/docs/advanced/best-practice/import-analysis.md b/docs/en/docs/advanced/best-practice/import-analysis.md
index 95fc1c3ace09b1..3c97c3d3a9afac 100644
--- a/docs/en/docs/advanced/best-practice/import-analysis.md
+++ b/docs/en/docs/advanced/best-practice/import-analysis.md
@@ -35,7 +35,7 @@ Doris provides a graphical command to help users analyze a specific import more
 
 If you don't know much about Doris' query plan tree, please read the previous article [DORIS/best practices/query analysis](./query-analysis.md).
 
-The execution process of a [Broker Load](../../data-operate/import/import-way/broker-load-manual.md) request is also based on Doris' query framework. A Broker Load job will be split into multiple subtasks based on the number of DATA INFILE clauses in the import request. Each subtask can be regarded as an independent import execution plan. An import plan consists of only one Fragment, which is composed as follows:
+The execution process of a [Broker Load](../../data-operate/import/import-way/broker-load-manual) request is also based on Doris' query framework. A Broker Load job will be split into multiple subtasks based on the number of DATA INFILE clauses in the import request. Each subtask can be regarded as an independent import execution plan. An import plan consists of only one Fragment, which is composed as follows:
 
 ```sql
 ┌────────────────┐
diff --git a/docs/en/docs/advanced/broker.md b/docs/en/docs/advanced/broker.md
index 47e4c7bd8334e1..549699744b7364 100644
--- a/docs/en/docs/advanced/broker.md
+++ b/docs/en/docs/advanced/broker.md
@@ -78,9 +78,9 @@ Different types of brokers support different storage systems.
 
 ## Function provided by Broker
 
-1. [Broker Load](../data-operate/import/import-way/broker-load-manual.md)
-2. [Export](../data-operate/export/export-manual.md)
-3. [Backup](../admin-manual/data-admin/backup.md)
+1. [Broker Load](../data-operate/import/import-way/broker-load-manual)
+2. [Export](../data-operate/export/export-manual)
+3. [Backup](../admin-manual/data-admin/backup)
 
 ## Broker Information
 
diff --git a/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md b/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
index aaf4bf733963ae..e5e152a8d28d66 100644
--- a/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
+++ b/docs/en/docs/data-operate/import/import-scenes/external-storage-load.md
@@ -35,7 +35,7 @@ Upload the files to be imported to HDFS. For specific commands, please refer to
 
 ### start import
 
-Hdfs load creates an import statement. The import method is basically the same as [Broker Load](../../../data-operate/import/import-way/broker-load-manual.md), only need to `WITH BROKER broker_name () ` statement with the following
+Hdfs load creates an import statement. The import method is basically the same as [Broker Load](../../../data-operate/import/import-way/broker-load-manual), only need to `WITH BROKER broker_name () ` statement with the following
 
 ```
   LOAD LABEL db_name.label_name 
@@ -78,11 +78,11 @@ Hdfs load creates an import statement. The import method is basically the same a
        );
    ```
 
-   For parameter introduction, please refer to [Broker Load](../../../data-operate/import/import-way/broker-load-manual.md), HA cluster creation syntax, view through `HELP BROKER LOAD`
+   For parameter introduction, please refer to [Broker Load](../../../data-operate/import/import-way/broker-load-manual), HA cluster creation syntax, view through `HELP BROKER LOAD`
 
 3. Check import status
 
-   Broker load is an asynchronous import method. The specific import results can be accessed through [SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD.md) command to view
+   Broker load is an asynchronous import method. The specific import results can be accessed through [SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD) command to view
    
    ```
    mysql> show load order by createtime desc limit 1\G;
@@ -128,7 +128,7 @@ This document mainly introduces how to import data stored in AWS S3. It also sup
 Other cloud storage systems can find relevant information compatible with S3 in corresponding documents
 
 ### Start Loading
-Like [Broker Load](../../../data-operate/import/import-way/broker-load-manual.md)  just replace `WITH BROKER broker_name ()` with
+Like [Broker Load](../../../data-operate/import/import-way/broker-load-manual)  just replace `WITH BROKER broker_name ()` with
 ```
     WITH S3
     (
diff --git a/docs/en/docs/data-operate/import/import-scenes/load-atomicity.md b/docs/en/docs/data-operate/import/import-scenes/load-atomicity.md
index 821d6d29b61315..9ffd79e5ed00db 100644
--- a/docs/en/docs/data-operate/import/import-scenes/load-atomicity.md
+++ b/docs/en/docs/data-operate/import/import-scenes/load-atomicity.md
@@ -28,9 +28,9 @@ under the License.
 
 All import operations in Doris have atomicity guarantees, that is, the data in an import job either all succeed or all fail. It will not happen that only part of the data is imported successfully.
 
-In [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) we can also implement atomic import of multiple tables .
+In [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) we can also implement atomic import of multiple tables .
 
-For the [materialized view](../../../query-acceleration/materialized-view.md) attached to the table, atomicity and consistency with the base table are also guaranteed.
+For the [materialized view](../../../query-acceleration/materialized-view) attached to the table, atomicity and consistency with the base table are also guaranteed.
 
 ## Label mechanism
 
diff --git a/docs/en/docs/data-operate/import/import-scenes/load-data-convert.md b/docs/en/docs/data-operate/import/import-scenes/load-data-convert.md
index e686da128c064a..e1a79fda32cb2c 100644
--- a/docs/en/docs/data-operate/import/import-scenes/load-data-convert.md
+++ b/docs/en/docs/data-operate/import/import-scenes/load-data-convert.md
@@ -28,7 +28,7 @@ under the License.
 
 ## Supported import methods
 
-- [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md)
+- [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD)
 
   ```sql
   LOAD LABEL example_db.label1
@@ -48,7 +48,7 @@ under the License.
   );
   ````
 
-- [STREAM LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md)
+- [STREAM LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD)
 
   ```bash
   curl
@@ -60,7 +60,7 @@ under the License.
   http://host:port/api/testDb/testTbl/_stream_load
   ````
 
-- [ROUTINE LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md)
+- [ROUTINE LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD)
 
   ```sql
   CREATE ROUTINE LOAD example_db.label1 ON my_table
diff --git a/docs/en/docs/data-operate/import/import-scenes/load-strict-mode.md b/docs/en/docs/data-operate/import/import-scenes/load-strict-mode.md
index 2887543cc4b4c2..23699ee4ebc8fe 100644
--- a/docs/en/docs/data-operate/import/import-scenes/load-strict-mode.md
+++ b/docs/en/docs/data-operate/import/import-scenes/load-strict-mode.md
@@ -36,7 +36,7 @@ Strict mode is all False by default, i.e. off.
 
 Different import methods set strict mode in different ways.
 
-1. [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md)
+1. [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD)
 
    ```sql
    LOAD LABEL example_db.label1
@@ -57,7 +57,7 @@ Different import methods set strict mode in different ways.
    )
    ````
 
-2. [STREAM LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD.md)
+2. [STREAM LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/STREAM-LOAD)
 
    ```bash
    curl --location-trusted -u user:passwd \
@@ -66,7 +66,7 @@ Different import methods set strict mode in different ways.
    http://host:port/api/example_db/my_table/_stream_load
    ````
 
-3. [ROUTINE LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD.md)
+3. [ROUTINE LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CREATE-ROUTINE-LOAD)
 
    ```sql
    CREATE ROUTINE LOAD example_db.test_job ON my_table
@@ -81,9 +81,9 @@ Different import methods set strict mode in different ways.
    );
    ````
 
-4. [INSERT](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT.md)
+4. [INSERT](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/INSERT)
 
-   Set via [session variables](../../../advanced/variables.md):
+   Set via [session variables](../../../advanced/variables):
 
    ```sql
    SET enable_insert_strict = true;
diff --git a/docs/en/docs/data-operate/import/import-way/broker-load-manual.md b/docs/en/docs/data-operate/import/import-way/broker-load-manual.md
index c059c05c68f7e1..92f4d8ca1422e0 100644
--- a/docs/en/docs/data-operate/import/import-way/broker-load-manual.md
+++ b/docs/en/docs/data-operate/import/import-way/broker-load-manual.md
@@ -26,11 +26,11 @@ under the License.
 
 # Broker Load
 
-Broker load is an asynchronous import method, and the supported data sources depend on the data sources supported by the [Broker](../../../advanced/broker.md) process.
+Broker load is an asynchronous import method, and the supported data sources depend on the data sources supported by the [Broker](../../../advanced/broker) process.
 
-Because the data in the Doris table is ordered, Broker load uses the doris cluster resources to sort the data when importing data. Complete massive historical data migration relative to Spark load, the Doris cluster resource usage is relatively large. , this method is used when the user does not have Spark computing resources. If there are Spark computing resources, it is recommended to use [Spark load](../../../data-operate/import/import-way/spark-load-manual.md).
+Because the data in the Doris table is ordered, Broker load uses the doris cluster resources to sort the data when importing data. Complete massive historical data migration relative to Spark load, the Doris cluster resource usage is relatively large. , this method is used when the user does not have Spark computing resources. If there are Spark computing resources, it is recommended to use [Spark load](../../../data-operate/import/import-way/spark-load-manual).
 
-Users need to create [Broker load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) import through MySQL protocol and import by viewing command to check the import result.
+Users need to create [Broker load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) import through MySQL protocol and import by viewing command to check the import result.
 
 ## Applicable scene
 
@@ -78,7 +78,7 @@ BE pulls data from the broker during execution, and imports the data into the sy
 
 ## start import
 
-Let's look at [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) through several actual scenario examples. use
+Let's look at [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) through several actual scenario examples. use
 
 ### Data import of Hive partition table
 
@@ -111,7 +111,7 @@ Then use Hive's Load command to import your data into the Hive table
 load data local inpath '/opt/custorm' into table ods_demo_detail;
 ````
 
-2. Create a Doris table, refer to the specific table syntax: [CREATE TABLE](../../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md)
+2. Create a Doris table, refer to the specific table syntax: [CREATE TABLE](../../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE)
 
 ````
 CREATE TABLE `doris_ods_test_detail` (
@@ -149,7 +149,7 @@ PROPERTIES (
 
 3. Start importing data
 
-   Specific syntax reference: [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md)
+   Specific syntax reference: [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD)
 
 ```sql
 LOAD LABEL broker_load_2022_03_23
@@ -256,13 +256,13 @@ LOAD LABEL demo.label_20220402
         );
 ````
 
-The specific parameters here can refer to: [Broker](../../../advanced/broker.md) and [Broker Load](../../../sql-manual/sql-reference-v2 /Data-Manipulation-Statements/Load/BROKER-LOAD.md) documentation
+The specific parameters here can refer to: [Broker](../../../advanced/broker.md) and [Broker Load](../../../sql-manual/sql-reference-v2 /Data-Manipulation-Statements/Load/BROKER-LOAD) documentation
 
 ## View import status
 
 We can view the status information of the above import task through the following command,
 
-The specific syntax reference for viewing the import status [SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD.md)
+The specific syntax reference for viewing the import status [SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD)
 
 ```sql
 mysql> show load order by createtime desc limit 1\G;
@@ -287,7 +287,7 @@ LoadFinishTime: 2022-04-01 18:59:11
 
 ## Cancel import
 
-When the broker load job status is not CANCELLED or FINISHED, it can be manually canceled by the user. When canceling, you need to specify the Label of the import task to be canceled. Cancel the import command syntax to execute [CANCEL LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CANCEL-LOAD.md) view.
+When the broker load job status is not CANCELLED or FINISHED, it can be manually canceled by the user. When canceling, you need to specify the Label of the import task to be canceled. Cancel the import command syntax to execute [CANCEL LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CANCEL-LOAD) view.
 
 For example: cancel the import job with the label broker_load_2022_03_23 on the database demo
 
@@ -298,7 +298,7 @@ CANCEL LOAD FROM demo WHERE LABEL = "broker_load_2022_03_23";
 
 ### Broker parameters
 
-Broker Load needs to use the Broker process to access remote storage. Different brokers need to provide different parameters. For details, please refer to [Broker documentation](../../../advanced/broker.md).
+Broker Load needs to use the Broker process to access remote storage. Different brokers need to provide different parameters. For details, please refer to [Broker documentation](../../../advanced/broker).
 
 ### FE configuration
 
@@ -399,7 +399,7 @@ The configuration parameter `async_loading_load_task_pool_size` of FE is used to
 
 Session variables can be enabled by executing `set enable_profile=true` before submitting the LOAD job. Then submit the import job. After the import job is completed, you can view the profile of the import job in the `Queris` tab of the FE web page.
 
-You can check the [SHOW LOAD PROFILE](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD-PROFILE.md) help document for more usage help information.
+You can check the [SHOW LOAD PROFILE](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD-PROFILE) help document for more usage help information.
 
 This Profile can help analyze the running status of import jobs.
 
@@ -442,4 +442,4 @@ Currently the Profile can only be viewed after the job has been successfully exe
 
 ## more help
 
-For more detailed syntax and best practices used by Broker Load, see [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) command manual, you can also enter `HELP BROKER LOAD` in the MySql client command line for more help information.
+For more detailed syntax and best practices used by Broker Load, see [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) command manual, you can also enter `HELP BROKER LOAD` in the MySql client command line for more help information.
diff --git a/docs/zh-CN/docs/advanced/broker.md b/docs/zh-CN/docs/advanced/broker.md
index bb6058d5f8249c..1d0ab3c9de2e72 100644
--- a/docs/zh-CN/docs/advanced/broker.md
+++ b/docs/zh-CN/docs/advanced/broker.md
@@ -73,9 +73,9 @@ Broker 在 Doris 系统架构中的位置如下：
 2. 对象存储
    - 所有支持S3协议的对象存储
 
-1. [Broker Load](../data-operate/import/import-way/broker-load-manual.md)
-2. [数据导出（Export）](../data-operate/export/export-manual.md)
-3. [数据备份](../admin-manual/data-admin/backup.md)
+1. [Broker Load](../data-operate/import/import-way/broker-load-manual)
+2. [数据导出（Export）](../data-operate/export/export-manual)
+3. [数据备份](../admin-manual/data-admin/backup)
 
 ## Broker 信息
 
diff --git a/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md b/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
index e64b27879a65c9..0aae25d0f34a6b 100644
--- a/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
+++ b/docs/zh-CN/docs/data-operate/import/import-scenes/external-storage-load.md
@@ -36,7 +36,7 @@ under the License.
 
 ### 开始导入
 
-Hdfs load 创建导入语句，导入方式和[Broker Load](../../../data-operate/import/import-way/broker-load-manual.md) 基本相同，只需要将 `WITH BROKER broker_name ()` 语句替换成如下部分
+Hdfs load 创建导入语句，导入方式和[Broker Load](../../../data-operate/import/import-way/broker-load-manual) 基本相同，只需要将 `WITH BROKER broker_name ()` 语句替换成如下部分
 
 ```
   LOAD LABEL db_name.label_name 
@@ -82,11 +82,11 @@ Hdfs load 创建导入语句，导入方式和[Broker Load](../../../data-operat
        "max_filter_ratio"="0.1"
        );
    ```
-    关于参数介绍，请参阅[Broker Load](../../../data-operate/import/import-way/broker-load-manual.md)，HA集群的创建语法，通过`HELP BROKER LOAD`查看
+    关于参数介绍，请参阅[Broker Load](../../../data-operate/import/import-way/broker-load-manual)，HA集群的创建语法，通过`HELP BROKER LOAD`查看
   
 3. 查看导入状态
    
-   Broker load 是一个异步的导入方式，具体导入结果可以通过[SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD.md)命令查看
+   Broker load 是一个异步的导入方式，具体导入结果可以通过[SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD)命令查看
    
    ```
    mysql> show load order by createtime desc limit 1\G;
@@ -134,7 +134,7 @@ Hdfs load 创建导入语句，导入方式和[Broker Load](../../../data-operat
 其他云存储系统可以相应的文档找到与S3兼容的相关信息
 
 ### 开始导入
-导入方式和 [Broker Load](../../../data-operate/import/import-way/broker-load-manual.md) 基本相同，只需要将 `WITH BROKER broker_name ()` 语句替换成如下部分
+导入方式和 [Broker Load](../../../data-operate/import/import-way/broker-load-manual) 基本相同，只需要将 `WITH BROKER broker_name ()` 语句替换成如下部分
 ```
     WITH S3
     (
diff --git a/docs/zh-CN/docs/data-operate/import/import-scenes/load-data-convert.md b/docs/zh-CN/docs/data-operate/import/import-scenes/load-data-convert.md
index ffaee2b216259e..85ea1c7b5cbc84 100644
--- a/docs/zh-CN/docs/data-operate/import/import-scenes/load-data-convert.md
+++ b/docs/zh-CN/docs/data-operate/import/import-scenes/load-data-convert.md
@@ -28,7 +28,7 @@ under the License.
 
 ## 支持的导入方式
 
-- [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md)
+- [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD)
 
   ```sql
   LOAD LABEL example_db.label1
diff --git a/docs/zh-CN/docs/data-operate/import/import-scenes/load-strict-mode.md b/docs/zh-CN/docs/data-operate/import/import-scenes/load-strict-mode.md
index 7708a461fc5798..bc2ef5709955b0 100644
--- a/docs/zh-CN/docs/data-operate/import/import-scenes/load-strict-mode.md
+++ b/docs/zh-CN/docs/data-operate/import/import-scenes/load-strict-mode.md
@@ -37,7 +37,7 @@ under the License.
 
 不同的导入方式设置严格模式的方式不尽相同。
 
-1. [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md)
+1. [BROKER LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD)
 
    ```sql
    LOAD LABEL example_db.label1
diff --git a/docs/zh-CN/docs/data-operate/import/import-way/broker-load-manual.md b/docs/zh-CN/docs/data-operate/import/import-way/broker-load-manual.md
index e0a26843360e1f..ef04f67ddd33fd 100644
--- a/docs/zh-CN/docs/data-operate/import/import-way/broker-load-manual.md
+++ b/docs/zh-CN/docs/data-operate/import/import-way/broker-load-manual.md
@@ -26,11 +26,11 @@ under the License.
 
 # Broker Load
 
-Broker load 是一个异步的导入方式，支持的数据源取决于 [Broker](../../../advanced/broker.md) 进程支持的数据源。
+Broker load 是一个异步的导入方式，支持的数据源取决于 [Broker](../../../advanced/broker) 进程支持的数据源。
 
-因为 Doris 表里的数据是有序的，所以 Broker load 在导入数据的时是要利用doris 集群资源对数据进行排序，相对于 Spark load 来完成海量历史数据迁移，对 Doris 的集群资源占用要比较大，这种方式是在用户没有 Spark 这种计算资源的情况下使用，如果有 Spark 计算资源建议使用   [Spark load](../../../data-operate/import/import-way/spark-load-manual.md)。
+因为 Doris 表里的数据是有序的，所以 Broker load 在导入数据的时是要利用doris 集群资源对数据进行排序，相对于 Spark load 来完成海量历史数据迁移，对 Doris 的集群资源占用要比较大，这种方式是在用户没有 Spark 这种计算资源的情况下使用，如果有 Spark 计算资源建议使用   [Spark load](../../../data-operate/import/import-way/spark-load-manual)。
 
-用户需要通过 MySQL 协议 创建 [Broker load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) 导入，并通过查看导入命令检查导入结果。
+用户需要通过 MySQL 协议 创建 [Broker load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) 导入，并通过查看导入命令检查导入结果。
 
 ## 适用场景
 
@@ -78,7 +78,7 @@ BE 在执行的过程中会从 Broker 拉取数据，在对数据 transform 之
 
 ## 开始导入
 
-下面我们通过几个实际的场景示例来看 [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) 的使用
+下面我们通过几个实际的场景示例来看 [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) 的使用
 
 ### Hive 分区表的数据导入
 
@@ -111,7 +111,7 @@ lines terminated by '\n'
 load data local inpath '/opt/custorm' into table ods_demo_detail;
 ```
 
-2. 创建 Doris 表，具体建表语法参照：[CREATE TABLE](../../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md)
+2. 创建 Doris 表，具体建表语法参照：[CREATE TABLE](../../../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE)
 
 ```
 CREATE TABLE `doris_ods_test_detail` (
@@ -149,7 +149,7 @@ PROPERTIES (
 
 3. 开始导入数据
 
-   具体语法参照： [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) 
+   具体语法参照： [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) 
 ```sql
 LOAD LABEL broker_load_2022_03_23
 (
@@ -255,13 +255,13 @@ LOAD LABEL demo.label_20220402
         );
 ```
 
-这里的具体 参数可以参照：  [Broker](../../../advanced/broker.md)  及 [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) 文档
+这里的具体 参数可以参照：  [Broker](../../../advanced/broker)  及 [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) 文档
 
 ## 查看导入状态
 
 我们可以通过下面的命令查看上面导入任务的状态信息，
 
-具体的查看导入状态的语法参考 [SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD.md)
+具体的查看导入状态的语法参考 [SHOW LOAD](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD)
 
 ```sql
 mysql> show load order by createtime desc limit 1\G;
@@ -286,7 +286,7 @@ LoadFinishTime: 2022-04-01 18:59:11
 
 ## 取消导入
 
-当 Broker load 作业状态不为 CANCELLED 或 FINISHED 时，可以被用户手动取消。取消时需要指定待取消导入任务的 Label 。取消导入命令语法可执行 [CANCEL LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CANCEL-LOAD.md) 查看。
+当 Broker load 作业状态不为 CANCELLED 或 FINISHED 时，可以被用户手动取消。取消时需要指定待取消导入任务的 Label 。取消导入命令语法可执行 [CANCEL LOAD](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/CANCEL-LOAD) 查看。
 
 例如：撤销数据库 demo 上， label 为 broker_load_2022_03_23 的导入作业
 
@@ -298,7 +298,7 @@ CANCEL LOAD FROM demo WHERE LABEL = "broker_load_2022_03_23";
 
 ###  Broker 参数
 
-Broker Load 需要借助 Broker 进程访问远端存储，不同的 Broker 需要提供不同的参数，具体请参阅 [Broker文档](../../../advanced/broker.md) 。
+Broker Load 需要借助 Broker 进程访问远端存储，不同的 Broker 需要提供不同的参数，具体请参阅 [Broker文档](../../../advanced/broker) 。
 
 ### FE 配置
 
@@ -399,7 +399,7 @@ FE 的配置参数 `async_loading_load_task_pool_size` 用于限制同时运行
 
 可以在提交 LOAD 作业前，先执行 `set enable_profile=true` 打开会话变量。然后提交导入作业。待导入作业完成后，可以在 FE 的 web 页面的 `Queris` 标签中查看到导入作业的 Profile。
 
-可以查看 [SHOW LOAD PROFILE](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD-PROFILE.md) 帮助文档，获取更多使用帮助信息。
+可以查看 [SHOW LOAD PROFILE](../../../sql-manual/sql-reference/Show-Statements/SHOW-LOAD-PROFILE) 帮助文档，获取更多使用帮助信息。
 
 这个 Profile 可以帮助分析导入作业的运行状态。
 
@@ -442,4 +442,4 @@ FE 的配置参数 `async_loading_load_task_pool_size` 用于限制同时运行
 
 ## 更多帮助
 
-关于 Broker Load 使用的更多详细语法及最佳实践，请参阅 [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD.md) 命令手册，你也可以在 MySql 客户端命令行下输入 `HELP BROKER LOAD` 获取更多帮助信息。
+关于 Broker Load 使用的更多详细语法及最佳实践，请参阅 [Broker Load](../../../sql-manual/sql-reference/Data-Manipulation-Statements/Load/BROKER-LOAD) 命令手册，你也可以在 MySql 客户端命令行下输入 `HELP BROKER LOAD` 获取更多帮助信息。
diff --git a/docs/zh-CN/docs/data-operate/update-delete/batch-delete-manual.md b/docs/zh-CN/docs/data-operate/update-delete/batch-delete-manual.md
index 62a5e869bfab18..237a50b6cdf411 100644
--- a/docs/zh-CN/docs/data-operate/update-delete/batch-delete-manual.md
+++ b/docs/zh-CN/docs/data-operate/update-delete/batch-delete-manual.md
@@ -26,7 +26,7 @@ under the License.
 
 # 批量删除
 
-目前Doris 支持 [Broker Load](../import/import-way/broker-load-manual.md)，[Routine Load](../import/import-way/routine-load-manual.md)， [Stream Load](../import/import-way/stream-load-manual.md) 等多种导入方式，对于数据的删除目前只能通过delete语句进行删除，使用delete 语句的方式删除时，每执行一次delete 都会生成一个新的数据版本，如果频繁删除会严重影响查询性能，并且在使用delete方式删除时，是通过生成一个空的rowset来记录删除条件实现，每次读取都要对删除条件进行过滤，同样在条件较多时会对性能造成影响。对比其他的系统，greenplum 的实现方式更像是传统数据库产品，snowflake 通过merge 语法实现。
+目前Doris 支持 [Broker Load](../import/import-way/broker-load-manual.md)，[Routine Load](../import/import-way/routine-load-manual)， [Stream Load](../import/import-way/stream-load-manual) 等多种导入方式，对于数据的删除目前只能通过delete语句进行删除，使用delete 语句的方式删除时，每执行一次delete 都会生成一个新的数据版本，如果频繁删除会严重影响查询性能，并且在使用delete方式删除时，是通过生成一个空的rowset来记录删除条件实现，每次读取都要对删除条件进行过滤，同样在条件较多时会对性能造成影响。对比其他的系统，greenplum 的实现方式更像是传统数据库产品，snowflake 通过merge 语法实现。
 
 对于类似于cdc数据导入的场景，数据中insert和delete一般是穿插出现的，面对这种场景我们目前的导入方式也无法满足，即使我们能够分离出insert和delete虽然可以解决导入的问题，但是仍然解决不了删除的问题。使用批量删除功能可以解决这些个别场景的需求。数据导入有三种合并方式：
 

From 70f5a26f44ed6491471f4563805e17badc639b3e Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Fri, 29 Dec 2023 12:50:41 +0800
Subject: [PATCH 151/299] [pipelineX](fix) Fix heap-use-after-free for
 AggSource dependency (#29272)

---
 be/src/pipeline/pipeline_x/dependency.h | 4 ++--
 be/src/pipeline/pipeline_x/operator.cpp | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/be/src/pipeline/pipeline_x/dependency.h b/be/src/pipeline/pipeline_x/dependency.h
index 116a9f3e87c364..a0504d17886a16 100644
--- a/be/src/pipeline/pipeline_x/dependency.h
+++ b/be/src/pipeline/pipeline_x/dependency.h
@@ -56,8 +56,8 @@ static constexpr auto TIME_UNIT_DEPENDENCY_LOG = 30 * 1000L * 1000L * 1000L;
 static_assert(TIME_UNIT_DEPENDENCY_LOG < SLOW_DEPENDENCY_THRESHOLD);
 
 struct BasicSharedState {
-    Dependency* source_dep = nullptr;
-    Dependency* sink_dep = nullptr;
+    DependencySPtr source_dep = nullptr;
+    DependencySPtr sink_dep = nullptr;
 
     virtual Status close(RuntimeState* state) { return Status::OK(); }
     virtual ~BasicSharedState() = default;
diff --git a/be/src/pipeline/pipeline_x/operator.cpp b/be/src/pipeline/pipeline_x/operator.cpp
index c5dbd34619f417..7d40e96f38ce78 100644
--- a/be/src/pipeline/pipeline_x/operator.cpp
+++ b/be/src/pipeline/pipeline_x/operator.cpp
@@ -336,15 +336,15 @@ Status PipelineXLocalState<DependencyType>::init(RuntimeState* state, LocalState
             _shared_state =
                     (typename DependencyType::SharedState*)_dependency->shared_state().get();
 
-            _shared_state->source_dep = _dependency;
-            _shared_state->sink_dep = deps.front().get();
+            _shared_state->source_dep = info.dependency;
+            _shared_state->sink_dep = deps.front();
         } else if constexpr (!is_fake_shared) {
             _dependency->set_shared_state(deps.front()->shared_state());
             _shared_state =
                     (typename DependencyType::SharedState*)_dependency->shared_state().get();
 
-            _shared_state->source_dep = _dependency;
-            _shared_state->sink_dep = deps.front().get();
+            _shared_state->source_dep = info.dependency;
+            _shared_state->sink_dep = deps.front();
         }
     }
 

From b9572f9de038c29aa0e763f4ebcdc1395183ec55 Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Fri, 29 Dec 2023 13:24:39 +0800
Subject: [PATCH 152/299] [pipelineX](fix) Fix pip scanner context bug (#29229)

---
 be/src/pipeline/exec/scan_operator.cpp     |   5 +-
 be/src/vec/exec/scan/pip_scanner_context.h | 118 +++++++++++++++------
 be/src/vec/exec/scan/scanner_context.cpp   |   3 +
 be/src/vec/exec/scan/scanner_context.h     |   2 +-
 4 files changed, 92 insertions(+), 36 deletions(-)

diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp
index 0c0ccd42410148..de3214b0465dc7 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1239,10 +1239,9 @@ template <typename Derived>
 Status ScanLocalState<Derived>::_start_scanners(
         const std::list<vectorized::VScannerSPtr>& scanners) {
     auto& p = _parent->cast<typename Derived::Parent>();
-    _scanner_ctx = PipScannerContext::create_shared(
+    _scanner_ctx = PipXScannerContext::create_shared(
             state(), this, p._output_tuple_desc, p.output_row_descriptor(), scanners, p.limit(),
-            state()->scan_queue_mem_limit(), p._col_distribute_ids, 1, _scan_dependency,
-            _finish_dependency);
+            state()->scan_queue_mem_limit(), _scan_dependency, _finish_dependency);
     return Status::OK();
 }
 
diff --git a/be/src/vec/exec/scan/pip_scanner_context.h b/be/src/vec/exec/scan/pip_scanner_context.h
index fbf59fffab20a5..9a717ec08b26d8 100644
--- a/be/src/vec/exec/scan/pip_scanner_context.h
+++ b/be/src/vec/exec/scan/pip_scanner_context.h
@@ -21,11 +21,9 @@
 #include "runtime/descriptors.h"
 #include "scanner_context.h"
 
-namespace doris {
+namespace doris::pipeline {
 
-namespace pipeline {
-
-class PipScannerContext : public vectorized::ScannerContext {
+class PipScannerContext final : public vectorized::ScannerContext {
     ENABLE_FACTORY_CREATOR(PipScannerContext);
 
 public:
@@ -41,19 +39,6 @@ class PipScannerContext : public vectorized::ScannerContext {
               _col_distribute_ids(col_distribute_ids),
               _need_colocate_distribute(!_col_distribute_ids.empty()) {}
 
-    PipScannerContext(RuntimeState* state, ScanLocalStateBase* local_state,
-                      const TupleDescriptor* output_tuple_desc,
-                      const RowDescriptor* output_row_descriptor,
-                      const std::list<vectorized::VScannerSPtr>& scanners, int64_t limit_,
-                      int64_t max_bytes_in_blocks_queue, const std::vector<int>& col_distribute_ids,
-                      const int num_parallel_instances,
-                      std::shared_ptr<pipeline::ScanDependency> dependency,
-                      std::shared_ptr<pipeline::Dependency> finish_dependency)
-            : vectorized::ScannerContext(state, output_tuple_desc, output_row_descriptor, scanners,
-                                         limit_, max_bytes_in_blocks_queue, num_parallel_instances,
-                                         local_state, dependency, finish_dependency),
-              _need_colocate_distribute(false) {}
-
     Status get_block_from_queue(RuntimeState* state, vectorized::BlockUPtr* block, bool* eos,
                                 int id, bool wait = false) override {
         {
@@ -95,9 +80,6 @@ class PipScannerContext : public vectorized::ScannerContext {
 
             if (_blocks_queues[id].empty()) {
                 this->reschedule_scanner_ctx();
-                if (_dependency) {
-                    _dependency->block();
-                }
             }
         }
 
@@ -180,9 +162,6 @@ class PipScannerContext : public vectorized::ScannerContext {
                     for (int j = i; j < block_size; j += queue_size) {
                         _blocks_queues[queue].emplace_back(std::move(blocks[j]));
                     }
-                    if (_dependency) {
-                        _dependency->set_ready();
-                    }
                 }
                 _next_queue_to_feed = queue + 1 < queue_size ? queue + 1 : 0;
             }
@@ -232,9 +211,6 @@ class PipScannerContext : public vectorized::ScannerContext {
                     _blocks_queues[i].emplace_back(std::move(_colocate_blocks[i]));
                     _colocate_mutable_blocks[i]->clear();
                 }
-                if (_dependency) {
-                    _dependency->set_ready();
-                }
             }
         }
     }
@@ -248,7 +224,7 @@ class PipScannerContext : public vectorized::ScannerContext {
         return res;
     }
 
-private:
+protected:
     int _next_queue_to_feed = 0;
     std::vector<std::unique_ptr<std::mutex>> _queue_mutexs;
     std::vector<std::list<vectorized::BlockUPtr>> _blocks_queues;
@@ -286,9 +262,6 @@ class PipScannerContext : public vectorized::ScannerContext {
                     std::lock_guard<std::mutex> queue_l(*_queue_mutexs[loc]);
                     _blocks_queues[loc].emplace_back(std::move(_colocate_blocks[loc]));
                 }
-                if (_dependency) {
-                    _dependency->set_ready();
-                }
                 _colocate_blocks[loc] = get_free_block();
                 _colocate_mutable_blocks[loc]->set_mutable_columns(
                         _colocate_blocks[loc]->mutate_columns());
@@ -297,5 +270,86 @@ class PipScannerContext : public vectorized::ScannerContext {
     }
 };
 
-} // namespace pipeline
-} // namespace doris
+class PipXScannerContext final : public vectorized::ScannerContext {
+    ENABLE_FACTORY_CREATOR(PipXScannerContext);
+
+public:
+    PipXScannerContext(RuntimeState* state, ScanLocalStateBase* local_state,
+                       const TupleDescriptor* output_tuple_desc,
+                       const RowDescriptor* output_row_descriptor,
+                       const std::list<vectorized::VScannerSPtr>& scanners, int64_t limit_,
+                       int64_t max_bytes_in_blocks_queue,
+                       std::shared_ptr<pipeline::ScanDependency> dependency,
+                       std::shared_ptr<pipeline::Dependency> finish_dependency)
+            : vectorized::ScannerContext(state, output_tuple_desc, output_row_descriptor, scanners,
+                                         limit_, max_bytes_in_blocks_queue, 1, local_state,
+                                         dependency, finish_dependency) {}
+    Status get_block_from_queue(RuntimeState* state, vectorized::BlockUPtr* block, bool* eos,
+                                int id, bool wait = false) override {
+        std::unique_lock l(_transfer_lock);
+        if (state->is_cancelled()) {
+            set_status_on_error(Status::Cancelled("cancelled"), false);
+        }
+
+        if (!status().ok()) {
+            return _process_status;
+        }
+
+        std::vector<vectorized::BlockUPtr> merge_blocks;
+        if (_blocks_queue.empty()) {
+            *eos = done();
+            return Status::OK();
+        }
+        if (_process_status.is<ErrorCode::CANCELLED>()) {
+            *eos = true;
+            return Status::OK();
+        }
+        *block = std::move(_blocks_queue.front());
+        _blocks_queue.pop_front();
+
+        auto rows = (*block)->rows();
+        while (!_blocks_queue.empty()) {
+            const auto add_rows = (*_blocks_queue.front()).rows();
+            if (rows + add_rows < state->batch_size()) {
+                rows += add_rows;
+                merge_blocks.emplace_back(std::move(_blocks_queue.front()));
+                _blocks_queue.pop_front();
+            } else {
+                break;
+            }
+        }
+
+        if (_blocks_queue.empty()) {
+            this->reschedule_scanner_ctx();
+            _dependency->block();
+        }
+
+        _cur_bytes_in_queue -= (*block)->allocated_bytes();
+        if (!merge_blocks.empty()) {
+            vectorized::MutableBlock m(block->get());
+            for (auto& merge_block : merge_blocks) {
+                _cur_bytes_in_queue -= merge_block->allocated_bytes();
+                static_cast<void>(m.merge(*merge_block));
+                return_free_block(std::move(merge_block));
+            }
+            (*block)->set_columns(std::move(m.mutable_columns()));
+        }
+
+        return Status::OK();
+    }
+
+    void reschedule_scanner_ctx() override {
+        if (done()) {
+            return;
+        }
+        auto state = _scanner_scheduler->submit(shared_from_this());
+        //todo(wb) rethinking is it better to mark current scan_context failed when submit failed many times?
+        if (state.ok()) {
+            _num_scheduling_ctx++;
+        } else {
+            set_status_on_error(state, false);
+        }
+    }
+};
+
+} // namespace doris::pipeline
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index 954c294574f33d..16bb1ce8487f4a 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -259,6 +259,9 @@ void ScannerContext::append_blocks_to_queue(std::vector<vectorized::BlockUPtr>&
         _blocks_queue.push_back(std::move(b));
     }
     blocks.clear();
+    if (_dependency) {
+        _dependency->set_ready();
+    }
     _blocks_queue_added_cv.notify_one();
     _queued_blocks_memory_usage->add(_cur_bytes_in_queue - old_bytes_in_queue);
 }
diff --git a/be/src/vec/exec/scan/scanner_context.h b/be/src/vec/exec/scan/scanner_context.h
index 6a3e8553f8fc3a..035d396bf65660 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -170,7 +170,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext> {
 
     SimplifiedScanScheduler* get_simple_scan_scheduler() { return _simple_scan_scheduler; }
 
-    void reschedule_scanner_ctx();
+    virtual void reschedule_scanner_ctx();
 
     // the unique id of this context
     std::string ctx_id;

From 2794427e7f3a020e1131a433739c3e638bd61381 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= <jianxie0@gmail.com>
Date: Fri, 29 Dec 2023 13:41:54 +0800
Subject: [PATCH 153/299] [enhancement](Nereids): refactor eliminating inner
 join by foreign key (#28816)

---
 .../rules/rewrite/EliminateJoinByFK.java      | 246 ++++++++----------
 .../nereids/util/ImmutableEquivalenceSet.java |  20 +-
 .../rules/rewrite/EliminateJoinByFkTest.java  | 102 ++++++--
 .../shape/query23.out                         |  41 ++-
 .../noStatsRfPrune/query23.out                |  48 ++--
 .../no_stats_shape/query23.out                |  48 ++--
 .../rf_prune/query23.out                      |  41 ++-
 .../shape/query23.out                         |  41 ++-
 8 files changed, 300 insertions(+), 287 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java
index 17fcf7842ef274..594dee5085356a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFK.java
@@ -29,24 +29,17 @@
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
 import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
-import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
-import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
-import org.apache.doris.nereids.trees.plans.logical.LogicalTopN;
-import org.apache.doris.nereids.trees.plans.logical.LogicalWindow;
 import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
 import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
 import org.apache.doris.nereids.util.ImmutableEquivalenceSet;
 
-import com.google.common.collect.BiMap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableMap.Builder;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 
@@ -54,7 +47,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Map.Entry;
 import java.util.Set;
 import java.util.stream.Collectors;
 import javax.annotation.Nullable;
@@ -80,7 +72,9 @@ public Plan rewriteRoot(Plan plan, JobContext jobContext) {
         @Override
         public Plan visit(Plan plan, ForeignKeyContext context) {
             Plan newPlan = visitChildren(this, plan, context);
-            context.passThroughPlan(plan);
+            // always expire primary key except filter, project and join.
+            // always keep foreign key alive
+            context.expirePrimaryKey(plan);
             return newPlan;
         }
 
@@ -97,35 +91,29 @@ public Plan visitLogicalRelation(LogicalRelation relation, ForeignKeyContext con
             return relation;
         }
 
-        private boolean canEliminate(LogicalJoin<?, ?> join, BiMap<Slot, Slot> equalSlots, ForeignKeyContext context) {
+        private boolean canEliminate(LogicalJoin<?, ?> join, Map<Slot, Slot> primaryToForeign,
+                ForeignKeyContext context) {
             if (!join.getOtherJoinConjuncts().isEmpty()) {
                 return false;
             }
             if (!join.getJoinType().isInnerJoin() && !join.getJoinType().isSemiJoin()) {
                 return false;
             }
-            return context.satisfyConstraint(equalSlots, join);
+            return context.satisfyConstraint(primaryToForeign, join);
         }
 
-        private boolean isForeignKeyAndUnique(Plan plan,
-                Set<Slot> keySet, ForeignKeyContext context) {
-            boolean unique = keySet.stream()
-                    .allMatch(s -> plan.getLogicalProperties().getFunctionalDependencies().isUnique(s));
-            return unique && context.isForeignKey(keySet);
-        }
-
-        private @Nullable Map<Expression, Expression> tryGetOutputToChildMap(Plan child,
-                Set<Slot> output, BiMap<Slot, Slot> equalSlots) {
-            Set<Slot> residual = Sets.difference(output, child.getOutputSet());
-            if (equalSlots.keySet().containsAll(residual)) {
-                return residual.stream()
-                        .collect(ImmutableMap.toImmutableMap(e -> e, equalSlots::get));
-            }
-            if (equalSlots.values().containsAll(residual)) {
-                return residual.stream()
-                        .collect(ImmutableMap.toImmutableMap(e -> e, e -> equalSlots.inverse().get(e)));
+        private @Nullable Map<Expression, Expression> tryMapOutputToForeignPlan(Plan foreignPlan,
+                Set<Slot> output, Map<Slot, Slot> primaryToForeign) {
+            Set<Slot> residualPrimary = Sets.difference(output, foreignPlan.getOutputSet());
+            ImmutableMap.Builder<Expression, Expression> builder = new ImmutableMap.Builder<>();
+            for (Slot slot : residualPrimary) {
+                if (primaryToForeign.containsKey(slot)) {
+                    builder.put(slot, primaryToForeign.get(slot));
+                } else {
+                    return null;
+                }
             }
-            return null;
+            return builder.build();
         }
 
         private Plan applyNullCompensationFilter(Plan child, Set<Slot> childSlots) {
@@ -139,47 +127,56 @@ private Plan applyNullCompensationFilter(Plan child, Set<Slot> childSlots) {
             return new LogicalFilter<>(predicates, child);
         }
 
-        private @Nullable Plan tryConstructPlanWithJoinChild(LogicalProject<LogicalJoin<?, ?>> project, Plan child,
-                BiMap<Slot, Slot> equalSlots, ForeignKeyContext context) {
+        private Plan tryEliminatePrimaryPlan(LogicalProject<LogicalJoin<?, ?>> project,
+                Plan foreignPlan, Set<Slot> foreignKeys,
+                Map<Slot, Slot> primaryToForeign, ForeignKeyContext context) {
             Set<Slot> output = project.getInputSlots();
-            Set<Slot> keySet = child.getOutputSet().containsAll(equalSlots.keySet())
-                    ? equalSlots.keySet()
-                    : equalSlots.values();
-            Map<Expression, Expression> outputToRight = tryGetOutputToChildMap(child, output, equalSlots);
-            if (outputToRight != null && isForeignKeyAndUnique(child, keySet, context)) {
+            Map<Expression, Expression> outputToForeign =
+                    tryMapOutputToForeignPlan(foreignPlan, output, primaryToForeign);
+            if (outputToForeign != null && canEliminate(project.child(), primaryToForeign, context)) {
                 List<NamedExpression> newProjects = project.getProjects().stream()
-                        .map(e -> outputToRight.containsKey(e)
-                                ? new Alias(e.getExprId(), outputToRight.get(e), e.toSql())
-                                : (NamedExpression) e.rewriteUp(s -> outputToRight.getOrDefault(s, s)))
+                        .map(e -> outputToForeign.containsKey(e)
+                                ? new Alias(e.getExprId(), outputToForeign.get(e), e.toSql())
+                                : (NamedExpression) e.rewriteUp(s -> outputToForeign.getOrDefault(s, s)))
                         .collect(ImmutableList.toImmutableList());
                 return project.withProjects(newProjects)
-                        .withChildren(applyNullCompensationFilter(child, keySet));
+                        .withChildren(applyNullCompensationFilter(foreignPlan, foreignKeys));
+            }
+            return project;
+        }
+
+        private @Nullable Map<Slot, Slot> mapPrimaryToForeign(ImmutableEquivalenceSet<Slot> equivalenceSet,
+                Set<Slot> foreignKeys) {
+            ImmutableMap.Builder<Slot, Slot> builder = new ImmutableMap.Builder<>();
+            for (Slot foreignSlot : foreignKeys) {
+                Set<Slot> primarySlots = equivalenceSet.calEqualSet(foreignSlot);
+                if (primarySlots.size() != 1) {
+                    return null;
+                }
+                builder.put(primarySlots.iterator().next(), foreignSlot);
             }
-            return null;
+            return builder.build();
         }
 
         // Right now we only support eliminate inner join, which should meet the following condition:
         // 1. only contain null-reject equal condition, and which all meet fk-pk constraint
         // 2. only output foreign table output or can be converted to foreign table output
-        // 3. foreign key is unique
         // 4. if foreign key is null, add a isNotNull predicate for null-reject join condition
         private Plan eliminateJoin(LogicalProject<LogicalJoin<?, ?>> project, ForeignKeyContext context) {
             LogicalJoin<?, ?> join = project.child();
             ImmutableEquivalenceSet<Slot> equalSet = join.getEqualSlots();
-            BiMap<Slot, Slot> equalSlots = equalSet.tryToMap();
-            if (equalSlots == null) {
-                return project;
-            }
-            if (!canEliminate(join, equalSlots, context)) {
-                return project;
-            }
-            Plan keepLeft = tryConstructPlanWithJoinChild(project, join.left(), equalSlots, context);
-            if (keepLeft != null) {
-                return keepLeft;
-            }
-            Plan keepRight = tryConstructPlanWithJoinChild(project, join.right(), equalSlots, context);
-            if (keepRight != null) {
-                return keepRight;
+            Set<Slot> leftSlots = Sets.intersection(join.left().getOutputSet(), equalSet.getAllItemSet());
+            Set<Slot> rightSlots = Sets.intersection(join.right().getOutputSet(), equalSet.getAllItemSet());
+            if (context.isForeignKey(leftSlots) && context.isPrimaryKey(rightSlots)) {
+                Map<Slot, Slot> primaryToForeignSlot = mapPrimaryToForeign(equalSet, leftSlots);
+                if (primaryToForeignSlot != null) {
+                    return tryEliminatePrimaryPlan(project, join.left(), leftSlots, primaryToForeignSlot, context);
+                }
+            } else if (context.isForeignKey(rightSlots) && context.isPrimaryKey(leftSlots)) {
+                Map<Slot, Slot> primaryToForeignSlot = mapPrimaryToForeign(equalSet, rightSlots);
+                if (primaryToForeignSlot != null) {
+                    return tryEliminatePrimaryPlan(project, join.right(), rightSlots, primaryToForeignSlot, context);
+                }
             }
             return project;
         }
@@ -197,32 +194,29 @@ public Plan visitLogicalProject(LogicalProject<?> project, ForeignKeyContext con
             }
             return project;
         }
+
+        @Override
+        public Plan visitLogicalJoin(LogicalJoin<?, ?> join, ForeignKeyContext context) {
+            Plan plan = visitChildren(this, join, context);
+            context.addJoin(join);
+            return plan;
+        }
+
+        @Override
+        public Plan visitLogicalFilter(LogicalFilter<?> filter, ForeignKeyContext context) {
+            Plan plan = visitChildren(this, filter, context);
+            context.addFilter(filter);
+            return plan;
+        }
     }
 
     private static class ForeignKeyContext {
-        static Set<Class<?>> propagatePrimaryKeyOperator = ImmutableSet
-                .<Class<?>>builder()
-                .add(LogicalProject.class)
-                .add(LogicalSort.class)
-                .add(LogicalJoin.class)
-                .build();
-        static Set<Class<?>> propagateForeignKeyOperator = ImmutableSet
-                .<Class<?>>builder()
-                .add(LogicalProject.class)
-                .add(LogicalSort.class)
-                .add(LogicalJoin.class)
-                .add(LogicalFilter.class)
-                .add(LogicalTopN.class)
-                .add(LogicalLimit.class)
-                .add(LogicalAggregate.class)
-                .add(LogicalWindow.class)
-                .build();
         Set<Map<Column, Column>> constraints = new HashSet<>();
         Set<Column> foreignKeys = new HashSet<>();
         Set<Column> primaryKeys = new HashSet<>();
         Map<Slot, Column> slotToColumn = new HashMap<>();
-        Map<Column, Set<LogicalJoin<?, ?>>> columnWithJoin = new HashMap<>();
-        Map<Column, Set<Expression>> columnWithPredicates = new HashMap<>();
+        Map<Slot, Set<LogicalJoin<?, ?>>> slotWithJoin = new HashMap<>();
+        Map<Slot, Set<Expression>> slotWithPredicates = new HashMap<>();
 
         public void putAllForeignKeys(TableIf table) {
             table.getForeignKeyConstraints().forEach(c -> {
@@ -238,6 +232,11 @@ public boolean isForeignKey(Set<Slot> key) {
                     key.stream().map(s -> slotToColumn.get(s)).collect(Collectors.toSet()));
         }
 
+        public boolean isPrimaryKey(Set<Slot> key) {
+            return primaryKeys.containsAll(
+                    key.stream().map(s -> slotToColumn.get(s)).collect(Collectors.toSet()));
+        }
+
         public void putSlot(SlotReference slot) {
             if (!slot.getColumn().isPresent()) {
                 return;
@@ -252,81 +251,62 @@ public void putAlias(Slot newSlot, Slot originSlot) {
             }
         }
 
-        public void passThroughPlan(Plan plan) {
-            Set<Column> output = plan.getOutput().stream()
+        public void addFilter(LogicalFilter<?> filter) {
+            filter.getOutput().stream()
+                    .filter(slotToColumn::containsKey)
+                    .forEach(slot -> {
+                        slotWithPredicates.computeIfAbsent(slot, v -> new HashSet<>());
+                        slotWithPredicates.get(slot).addAll(filter.getConjuncts());
+                    });
+        }
+
+        public void addJoin(LogicalJoin<?, ?> join) {
+            join.getOutput().stream()
+                    .filter(slotToColumn::containsKey)
+                    .forEach(slot ->
+                            slotWithJoin.computeIfAbsent(slot, v -> Sets.newHashSet((join))));
+        }
+
+        public void expirePrimaryKey(Plan plan) {
+            plan.getOutput().stream()
                     .filter(slotToColumn::containsKey)
                     .map(s -> slotToColumn.get(s))
-                    .collect(ImmutableSet.toImmutableSet());
-            if (plan instanceof LogicalJoin) {
-                output.forEach(c ->
-                        columnWithJoin.computeIfAbsent(c, v -> Sets.newHashSet((LogicalJoin<?, ?>) plan)));
-                return;
-            }
-            if (plan instanceof LogicalFilter) {
-                output.forEach(c -> {
-                    columnWithPredicates.computeIfAbsent(c, v -> new HashSet<>());
-                    columnWithPredicates.get(c).addAll(((LogicalFilter<?>) plan).getConjuncts());
-                });
-                return;
-            }
-            if (!propagatePrimaryKeyOperator.contains(plan.getClass())) {
-                output.forEach(primaryKeys::remove);
-            }
-            if (!propagateForeignKeyOperator.contains(plan.getClass())) {
-                output.forEach(foreignKeys::remove);
-            }
+                    .forEach(primaryKeys::remove);
         }
 
-        public boolean satisfyConstraint(BiMap<Slot, Slot> equalSlots, LogicalJoin<?, ?> join) {
-            ImmutableMap.Builder<Column, Column> foreignToPrimaryBuilder = new Builder<>();
-            for (Entry<Slot, Slot> entry : equalSlots.entrySet()) {
-                Slot left = entry.getKey();
-                Slot right = entry.getValue();
-                if (!slotToColumn.containsKey(left) || !slotToColumn.containsKey(right)) {
-                    return false;
-                }
-                Column leftColumn = slotToColumn.get(left);
-                Column rightColumn = slotToColumn.get(right);
-                if (foreignKeys.contains(leftColumn)) {
-                    foreignToPrimaryBuilder.put(leftColumn, rightColumn);
-                } else if (foreignKeys.contains(rightColumn)) {
-                    foreignToPrimaryBuilder.put(rightColumn, leftColumn);
-                } else {
-                    return false;
-                }
-            }
-            Map<Column, Column> foreignToPrimary = foreignToPrimaryBuilder.build();
+        public boolean satisfyConstraint(Map<Slot, Slot> primaryToForeign, LogicalJoin<?, ?> join) {
+            Map<Column, Column> foreignToPrimary = primaryToForeign.entrySet().stream()
+                    .collect(ImmutableMap.toImmutableMap(
+                            e -> slotToColumn.get(e.getValue()),
+                            e -> slotToColumn.get(e.getKey())));
             // The primary key can only contain join that may be eliminated
-            if (!foreignToPrimary.values().stream().allMatch(p ->
-                    columnWithJoin.get(p).size() == 1 && columnWithJoin.get(p).iterator().next() == join)) {
+            if (!primaryToForeign.keySet().stream().allMatch(p ->
+                    slotWithJoin.get(p).size() == 1 && slotWithJoin.get(p).iterator().next() == join)) {
                 return false;
             }
             // The foreign key's filters must contain primary filters
-            if (!isPredicateCompatible(equalSlots, foreignToPrimary)) {
+            if (!isPredicateCompatible(primaryToForeign)) {
                 return false;
             }
             return constraints.contains(foreignToPrimary);
         }
 
-        private boolean isPredicateCompatible(BiMap<Slot, Slot> equalSlots, Map<Column, Column> foreignToPrimary) {
-            return foreignToPrimary.entrySet().stream().allMatch(fp -> {
-                BiMap<Slot, Slot> primarySlotToForeign = equalSlots.keySet().stream()
-                        .map(slotToColumn::get)
-                        .anyMatch(primaryKeys::contains)
-                        ? equalSlots :
-                        equalSlots.inverse();
-                if (!columnWithPredicates.containsKey(fp.getValue())) {
+        // When predicates of foreign keys is a subset of that of primary keys
+        private boolean isPredicateCompatible(Map<Slot, Slot> primaryToForeign) {
+            return primaryToForeign.entrySet().stream().allMatch(pf -> {
+                // There is no predicate in primary key
+                if (!slotWithPredicates.containsKey(pf.getKey()) || slotWithPredicates.get(pf.getKey()).isEmpty()) {
                     return true;
                 }
-                Set<Expression> primaryPredicates = columnWithPredicates.get(fp.getValue()).stream()
-                        .map(e -> e.rewriteUp(
-                                s -> s instanceof Slot ? primarySlotToForeign.getOrDefault(s, (Slot) s) : s))
-                        .collect(Collectors.toSet());
-                if (columnWithPredicates.get(fp.getKey()) == null && !columnWithPredicates.isEmpty()) {
+                // There are some predicates in primary key but there is no predicate in foreign key
+                if (slotWithPredicates.containsKey(pf.getValue()) && slotWithPredicates.get(pf.getValue()).isEmpty()) {
                     return false;
-                } else {
-                    return columnWithPredicates.get(fp.getKey()).containsAll(primaryPredicates);
                 }
+                Set<Expression> primaryPredicates = slotWithPredicates.get(pf.getKey()).stream()
+                        .map(e -> e.rewriteUp(
+                                s -> s instanceof Slot ? primaryToForeign.getOrDefault(s, (Slot) s) : s))
+                        .collect(Collectors.toSet());
+                return slotWithPredicates.get(pf.getValue()).containsAll(primaryPredicates);
             });
         }
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEquivalenceSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEquivalenceSet.java
index e54ecb7c9e7d06..66d20597fc1804 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEquivalenceSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ImmutableEquivalenceSet.java
@@ -17,15 +17,12 @@
 
 package org.apache.doris.nereids.util;
 
-import com.google.common.collect.BiMap;
-import com.google.common.collect.ImmutableBiMap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
-import javax.annotation.Nullable;
 
 /**
  * EquivalenceSet
@@ -77,25 +74,16 @@ public ImmutableEquivalenceSet<T> build() {
     }
 
     /**
-     * cal equal set for a
+     * cal equal set for a except self
      */
     public Set<T> calEqualSet(T a) {
         T ra = root.get(a);
         return root.keySet().stream()
-                .filter(t -> root.get(t).equals(ra))
+                .filter(t -> root.get(t).equals(ra) && !t.equals(a))
                 .collect(ImmutableSet.toImmutableSet());
     }
 
-    /**
-     * try to convert it to a map, such as a = b c = d.
-     * When meets a = b a = c, return null
-     */
-    public @Nullable BiMap<T, T> tryToMap() {
-        if (root.values().stream().distinct().count() * 2 != root.size()) {
-            return null;
-        }
-        return root.keySet().stream()
-                .filter(t -> !root.get(t).equals(t))
-                .collect(ImmutableBiMap.toImmutableBiMap(t -> t, root::get));
+    public Set<T> getAllItemSet() {
+        return ImmutableSet.copyOf(root.keySet());
     }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
index 721e94b144d05c..921e45412512db 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
@@ -21,6 +21,7 @@
 import org.apache.doris.nereids.util.PlanChecker;
 import org.apache.doris.utframe.TestWithFeService;
 
+import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
 class EliminateJoinByFkTest extends TestWithFeService implements MemoPatternMatchSupported {
@@ -56,59 +57,102 @@ protected void runBeforeAll() throws Exception {
     }
 
     @Test
-    void testNotNull() throws Exception {
-        addConstraint("Alter table foreign_not_null add constraint uk1 unique (id2)\n");
+    void testNotNull() {
         String sql = "select pri.id1 from pri inner join foreign_not_null on pri.id1 = foreign_not_null.id2";
         PlanChecker.from(connectContext)
                 .analyze(sql)
                 .rewrite()
                 .nonMatch(logicalJoin())
                 .printlnTree();
-        dropConstraint("Alter table foreign_not_null drop constraint uk1\n");
+
+        sql = "select foreign_not_null.id2 from pri inner join foreign_not_null on pri.id1 = foreign_not_null.id2";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .nonMatch(logicalJoin())
+                .printlnTree();
     }
 
     @Test
-    void testNotNullWithPredicate() throws Exception {
-        addConstraint("Alter table foreign_not_null add constraint uk2 unique (id2)\n");
+    void testNotNullWithPredicate() {
         String sql = "select pri.id1 from pri inner join foreign_not_null on pri.id1 = foreign_not_null.id2\n"
                 + "where pri.id1 = 1";
         PlanChecker.from(connectContext)
                 .analyze(sql)
                 .rewrite()
                 .nonMatch(logicalJoin())
+                .matches(logicalFilter().when(f -> {
+                    Assertions.assertEquals("(id2 = 1)", f.getPredicate().toSql());
+                    return true;
+                }))
+                .printlnTree();
+        sql = "select foreign_not_null.id2 from pri inner join foreign_not_null on pri.id1 = foreign_not_null.id2\n"
+                + "where pri.id1 = 1";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .nonMatch(logicalJoin())
+                .matches(logicalFilter().when(f -> {
+                    Assertions.assertEquals("(id2 = 1)", f.getPredicate().toSql());
+                    return true;
+                }))
                 .printlnTree();
-        dropConstraint("Alter table foreign_not_null drop constraint uk2\n");
     }
 
     @Test
     void testNull() throws Exception {
-        addConstraint("Alter table foreign_null add constraint uk unique (id3)\n");
         String sql = "select pri.id1 from pri inner join foreign_null on pri.id1 = foreign_null.id3";
         PlanChecker.from(connectContext)
                 .analyze(sql)
                 .rewrite()
                 .nonMatch(logicalJoin())
+                .matches(logicalFilter().when(f -> {
+                    Assertions.assertEquals("( not id3 IS NULL)", f.getPredicate().toSql());
+                    return true;
+                }))
+                .printlnTree();
+        sql = "select foreign_null.id3 from pri inner join foreign_null on pri.id1 = foreign_null.id3";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .nonMatch(logicalJoin())
+                .matches(logicalFilter().when(f -> {
+                    Assertions.assertEquals("( not id3 IS NULL)", f.getPredicate().toSql());
+                    return true;
+                }))
                 .printlnTree();
-        dropConstraint("Alter table foreign_null drop constraint uk\n");
     }
 
     @Test
     void testNullWithPredicate() throws Exception {
-        addConstraint("Alter table foreign_null add constraint uk unique (id3)\n");
         String sql = "select pri.id1 from pri inner join foreign_null on pri.id1 = foreign_null.id3\n"
                 + "where pri.id1 = 1";
         PlanChecker.from(connectContext)
                 .analyze(sql)
                 .rewrite()
                 .nonMatch(logicalJoin())
+                .matches(logicalFilter().when(f -> {
+                    Assertions.assertEquals("( not id3 IS NULL)", f.getExpressions().get(0).toSql());
+                    Assertions.assertEquals("(id3 = 1)", f.getExpressions().get(1).toSql());
+                    return true;
+                }))
+                .printlnTree();
+        sql = "select id3 from pri inner join foreign_null on pri.id1 = foreign_null.id3\n"
+                + "where pri.id1 = 1";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .nonMatch(logicalJoin())
+                .matches(logicalFilter().when(f -> {
+                    Assertions.assertEquals("( not id3 IS NULL)", f.getExpressions().get(0).toSql());
+                    Assertions.assertEquals("(id3 = 1)", f.getExpressions().get(1).toSql());
+                    return true;
+                }))
                 .printlnTree();
-        dropConstraint("Alter table foreign_null drop constraint uk\n");
     }
 
     @Test
-    void testMultiJoin() throws Exception {
-        addConstraint("Alter table foreign_null add constraint uk_id3 unique (id3)\n");
-        addConstraint("Alter table foreign_not_null add constraint uk_id2 unique (id2)\n");
+    void testMultiJoinCanPassForeign() throws Exception {
         String sql = "select id1 from "
                 + "foreign_null inner join foreign_not_null on id2 = id3\n"
                 + "inner join pri on id1 = id3";
@@ -117,7 +161,35 @@ void testMultiJoin() throws Exception {
                 .rewrite()
                 .nonMatch(logicalOlapScan().when(scan -> scan.getTable().getName().equals("pri")))
                 .printlnTree();
-        dropConstraint("Alter table foreign_null drop constraint uk_id3\n");
-        dropConstraint("Alter table foreign_not_null drop constraint uk_id2");
+    }
+
+    @Test
+    void testMultiJoinCannotPassPrimary() throws Exception {
+        String sql = "select id1 from "
+                + "foreign_null "
+                + "inner join pri on id1 = id3\n"
+                + "inner join foreign_not_null on id2 = id3\n";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(logicalOlapScan().when(scan -> scan.getTable().getName().equals("pri")))
+                .printlnTree();
+    }
+
+    @Test
+    void testOtherCond() {
+        String sql = "select pri.id1 from pri inner join foreign_null on pri.id1 > foreign_null.id3 \n"
+                + "where pri.id1 = 1";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(logicalOlapScan().when(scan -> scan.getTable().getName().equals("pri")));
+
+        sql = "select pri.id1 from pri inner join foreign_null on pri.id1 != foreign_null.id3 \n"
+                + "where pri.id1 = 1";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(logicalOlapScan().when(scan -> scan.getTable().getName().equals("pri")));
     }
 }
diff --git a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
index d9254bbe921fc2..c7563fa6bdb1f1 100644
--- a/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf1000_p0/shape/query23.out
@@ -24,35 +24,30 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----PhysicalCteProducer ( cteId=CTEId#2 )
 ------PhysicalProject
 --------NestedLoopJoin[INNER_JOIN](cast(ssales as DOUBLE) > cast((0.9500 * tpcds_cmax) as DOUBLE))
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[customer]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter(( not ss_customer_sk IS NULL))
+--------------------PhysicalOlapScan[store_sales]
 ----------PhysicalDistribute
 ------------PhysicalProject
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
-----------------------------PhysicalDistribute
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+----------------------------------filter(( not ss_customer_sk IS NULL))
+------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 --------------------------------------PhysicalOlapScan[date_dim]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer]
 ----PhysicalResultSink
 ------PhysicalLimit[GLOBAL]
 --------PhysicalLimit[LOCAL]
@@ -67,11 +62,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[cs_bill_customer_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
 ----------------------------PhysicalDistribute
-------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6
+----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
@@ -86,11 +81,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF8 c_customer_sk->[ws_bill_customer_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ws_bill_customer_sk]
 ----------------------------PhysicalDistribute
-------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->[ws_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ws_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF7 RF8
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF5 RF6
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter((date_dim.d_moy = 7) and (date_dim.d_year = 2000))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out
index 4a2a4eee66ff1b..0ee8af28ef871b 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/noStatsRfPrune/query23.out
@@ -23,36 +23,30 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----PhysicalCteProducer ( cteId=CTEId#2 )
 ------PhysicalProject
 --------NestedLoopJoin[INNER_JOIN](cast(ssales as DOUBLE) > cast((0.9500 * tpcds_cmax) as DOUBLE))
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[customer]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter(( not ss_customer_sk IS NULL))
+--------------------PhysicalOlapScan[store_sales]
 ----------PhysicalDistribute
 ------------PhysicalProject
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ss_customer_sk]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
+--------------------------------PhysicalProject
+----------------------------------filter(( not ss_customer_sk IS NULL))
+------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
-----------------------------------PhysicalOlapScan[date_dim]
+------------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
+--------------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
 ------PhysicalLimit[GLOBAL]
 --------PhysicalLimit[LOCAL]
@@ -63,11 +57,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=()
 ----------------------PhysicalDistribute
-------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[cs_sold_date_sk]
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[cs_bill_customer_sk]
+------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[cs_sold_date_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
@@ -81,11 +75,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=()
 ----------------------PhysicalDistribute
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ws_bill_customer_sk]
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_sales] apply RFs: RF7 RF8
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF5 RF6
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out
index 21fdf044f74761..b7577680ebadac 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/no_stats_shape/query23.out
@@ -23,36 +23,30 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----PhysicalCteProducer ( cteId=CTEId#2 )
 ------PhysicalProject
 --------NestedLoopJoin[INNER_JOIN](cast(ssales as DOUBLE) > cast((0.9500 * tpcds_cmax) as DOUBLE))
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[customer]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter(( not ss_customer_sk IS NULL))
+--------------------PhysicalOlapScan[store_sales]
 ----------PhysicalDistribute
 ------------PhysicalProject
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[ss_sold_date_sk]
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
 ----------------------------PhysicalProject
-------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF2 c_customer_sk->[ss_customer_sk]
---------------------------------PhysicalDistribute
-----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
+--------------------------------PhysicalProject
+----------------------------------filter(( not ss_customer_sk IS NULL))
+------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
-------------------------------------PhysicalOlapScan[customer]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
-----------------------------------PhysicalOlapScan[date_dim]
+------------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
+--------------------------------------PhysicalOlapScan[date_dim]
 ----PhysicalResultSink
 ------PhysicalLimit[GLOBAL]
 --------PhysicalLimit[LOCAL]
@@ -63,11 +57,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_item_sk = frequent_ss_items.item_sk)) otherCondition=()
 ----------------------PhysicalDistribute
-------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[cs_sold_date_sk]
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[cs_bill_customer_sk]
+------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF4 d_date_sk->[cs_sold_date_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[cs_bill_customer_sk]
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6
+--------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
@@ -81,11 +75,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ------------------PhysicalProject
 --------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_item_sk = frequent_ss_items.item_sk)) otherCondition=()
 ----------------------PhysicalDistribute
-------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF8 d_date_sk->[ws_sold_date_sk]
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF7 c_customer_sk->[ws_bill_customer_sk]
+------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF6 d_date_sk->[ws_sold_date_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ws_bill_customer_sk]
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[web_sales] apply RFs: RF7 RF8
+--------------------------------PhysicalOlapScan[web_sales] apply RFs: RF5 RF6
 ----------------------------PhysicalDistribute
 ------------------------------PhysicalProject
 --------------------------------PhysicalCteConsumer ( cteId=CTEId#2 )
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out
index eb0339f70109ca..e1aafd4c5a1138 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/rf_prune/query23.out
@@ -24,35 +24,30 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----PhysicalCteProducer ( cteId=CTEId#2 )
 ------PhysicalProject
 --------NestedLoopJoin[INNER_JOIN](cast(ssales as DOUBLE) > cast((0.9500 * tpcds_cmax) as DOUBLE))
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[customer]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter(( not ss_customer_sk IS NULL))
+--------------------PhysicalOlapScan[store_sales]
 ----------PhysicalDistribute
 ------------PhysicalProject
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
-----------------------------PhysicalDistribute
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+----------------------------------filter(( not ss_customer_sk IS NULL))
+------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 --------------------------------------PhysicalOlapScan[date_dim]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer]
 ----PhysicalResultSink
 ------PhysicalLimit[GLOBAL]
 --------PhysicalLimit[LOCAL]
@@ -67,11 +62,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[cs_bill_customer_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
 ----------------------------PhysicalDistribute
-------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6
+----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
@@ -86,11 +81,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF8 c_customer_sk->[ws_bill_customer_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ws_bill_customer_sk]
 ----------------------------PhysicalDistribute
-------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->[ws_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ws_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF7 RF8
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF5 RF6
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
diff --git a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out
index eefbd37e7dfa72..54d92b454d32ea 100644
--- a/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out
+++ b/regression-test/data/nereids_tpcds_shape_sf100_p0/shape/query23.out
@@ -24,35 +24,30 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 ----PhysicalCteProducer ( cteId=CTEId#2 )
 ------PhysicalProject
 --------NestedLoopJoin[INNER_JOIN](cast(ssales as DOUBLE) > cast((0.9500 * tpcds_cmax) as DOUBLE))
-----------hashAgg[LOCAL]
-------------PhysicalProject
---------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk]
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[store_sales] apply RFs: RF4
-----------------PhysicalDistribute
-------------------PhysicalProject
---------------------PhysicalOlapScan[customer]
+----------hashAgg[GLOBAL]
+------------PhysicalDistribute
+--------------hashAgg[LOCAL]
+----------------PhysicalProject
+------------------filter(( not ss_customer_sk IS NULL))
+--------------------PhysicalOlapScan[store_sales]
 ----------PhysicalDistribute
 ------------PhysicalProject
 --------------hashAgg[GLOBAL]
 ----------------PhysicalDistribute
 ------------------hashAgg[LOCAL]
 --------------------PhysicalProject
-----------------------hashAgg[LOCAL]
-------------------------PhysicalProject
---------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF3 c_customer_sk->[ss_customer_sk]
-----------------------------PhysicalDistribute
+----------------------hashAgg[GLOBAL]
+------------------------PhysicalDistribute
+--------------------------hashAgg[LOCAL]
+----------------------------PhysicalProject
 ------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF2 d_date_sk->[ss_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2 RF3
+----------------------------------filter(( not ss_customer_sk IS NULL))
+------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF2
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter(d_year IN (2000, 2001, 2002, 2003))
 --------------------------------------PhysicalOlapScan[date_dim]
-----------------------------PhysicalDistribute
-------------------------------PhysicalProject
---------------------------------PhysicalOlapScan[customer]
 ----PhysicalResultSink
 ------PhysicalLimit[GLOBAL]
 --------PhysicalLimit[LOCAL]
@@ -67,11 +62,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[cs_bill_customer_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((catalog_sales.cs_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[cs_bill_customer_sk]
 ----------------------------PhysicalDistribute
-------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[cs_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((catalog_sales.cs_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[cs_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF5 RF6
+----------------------------------PhysicalOlapScan[catalog_sales] apply RFs: RF3 RF4
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))
@@ -86,11 +81,11 @@ PhysicalCteAnchor ( cteId=CTEId#0 )
 --------------------------PhysicalCteConsumer ( cteId=CTEId#0 )
 ----------------------PhysicalDistribute
 ------------------------PhysicalProject
---------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF8 c_customer_sk->[ws_bill_customer_sk]
+--------------------------hashJoin[LEFT_SEMI_JOIN] hashCondition=((web_sales.ws_bill_customer_sk = best_ss_customer.c_customer_sk)) otherCondition=() build RFs:RF6 c_customer_sk->[ws_bill_customer_sk]
 ----------------------------PhysicalDistribute
-------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF7 d_date_sk->[ws_sold_date_sk]
+------------------------------hashJoin[INNER_JOIN] hashCondition=((web_sales.ws_sold_date_sk = date_dim.d_date_sk)) otherCondition=() build RFs:RF5 d_date_sk->[ws_sold_date_sk]
 --------------------------------PhysicalProject
-----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF7 RF8
+----------------------------------PhysicalOlapScan[web_sales] apply RFs: RF5 RF6
 --------------------------------PhysicalDistribute
 ----------------------------------PhysicalProject
 ------------------------------------filter((date_dim.d_moy = 5) and (date_dim.d_year = 2000))

From 9fc613de9c85543719f57afa3bdc1ea3f8cf5f59 Mon Sep 17 00:00:00 2001
From: seawinde <149132972+seawinde@users.noreply.github.com>
Date: Fri, 29 Dec 2023 13:45:33 +0800
Subject: [PATCH 154/299] [fix](nereids) Fix query rewrite by mv fail when self
 join (#29227)

Fix query rewrite by mv fail when self join, after fix query like following can be rewrited

def materialized view = """
    select
    a.o_orderkey,
    count(distinct a.o_orderstatus) num1,
    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate = '2023-12-08' AND b.o_orderdate = '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num2,
    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate >= '2023-12-01' AND a.o_orderdate <= '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num3,
    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority in (1,2) AND a.o_orderdate >= '2023-12-08' AND b.o_orderdate <= '2023-12-09' THEN a.o_shippriority-b.o_custkey ELSE 0 END) num4,
    AVG(a.o_totalprice) num5,
    MAX(b.o_totalprice) num6,
    MIN(a.o_totalprice) num7
    from
    orders a
    left outer join orders b
    on a.o_orderkey = b.o_orderkey
    and a.o_custkey = b.o_custkey
    group by a.o_orderkey;
"""

def query = """
    select
    a.o_orderkey,
    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate = '2023-12-08' AND b.o_orderdate = '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num2,
    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate >= '2023-12-01' AND a.o_orderdate <= '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num3,
    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority in (1,2) AND a.o_orderdate >= '2023-12-08' AND b.o_orderdate <= '2023-12-09' THEN a.o_shippriority-b.o_custkey ELSE 0 END) num4,
    AVG(a.o_totalprice) num5,
    MAX(b.o_totalprice) num6,
    MIN(a.o_totalprice) num7
    from
    orders a
    left outer join orders b
    on a.o_orderkey = b.o_orderkey
    and a.o_custkey = b.o_custkey
    group by a.o_orderkey;
"""
---
 .../mv/mapping/RelationMapping.java           | 81 +++++++++++++------
 .../rules/exploration/mv/MappingTest.java     | 70 +++++++++++++++-
 .../mv/join/left_outer/outer_join.out         | 14 ++++
 .../mv/join/left_outer/outer_join.groovy      | 39 +++++++++
 4 files changed, 179 insertions(+), 25 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
index fdf47d13433f00..b5494c01b1b924 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java
@@ -22,11 +22,13 @@
 import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
 
 import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
+import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableBiMap;
 import com.google.common.collect.ImmutableBiMap.Builder;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.LinkedListMultimap;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -59,54 +61,85 @@ public static RelationMapping of(ImmutableBiMap<MappedRelation, MappedRelation>
      */
     public static List<RelationMapping> generate(List<CatalogRelation> sources, List<CatalogRelation> targets) {
         // Construct tmp map, key is the table qualifier, value is the corresponding catalog relations
-        LinkedListMultimap<Long, MappedRelation> sourceTableRelationIdMap = LinkedListMultimap.create();
+        HashMultimap<Long, MappedRelation> sourceTableRelationIdMap = HashMultimap.create();
         for (CatalogRelation relation : sources) {
             sourceTableRelationIdMap.put(getTableQualifier(relation.getTable()),
                     MappedRelation.of(relation.getRelationId(), relation));
         }
-        LinkedListMultimap<Long, MappedRelation> targetTableRelationIdMap = LinkedListMultimap.create();
+        HashMultimap<Long, MappedRelation> targetTableRelationIdMap = HashMultimap.create();
         for (CatalogRelation relation : targets) {
             targetTableRelationIdMap.put(getTableQualifier(relation.getTable()),
                     MappedRelation.of(relation.getRelationId(), relation));
         }
         Set<Long> sourceTableKeySet = sourceTableRelationIdMap.keySet();
-        List<List<Pair<MappedRelation, MappedRelation>>> mappedRelations = new ArrayList<>();
+        List<List<BiMap<MappedRelation, MappedRelation>>> mappedRelations = new ArrayList<>();
 
-        for (Long sourceTableQualifier : sourceTableKeySet) {
-            List<MappedRelation> sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableQualifier);
-            List<MappedRelation> targetMappedRelations = targetTableRelationIdMap.get(sourceTableQualifier);
+        for (Long sourceTableId : sourceTableKeySet) {
+            Set<MappedRelation> sourceMappedRelations = sourceTableRelationIdMap.get(sourceTableId);
+            Set<MappedRelation> targetMappedRelations = targetTableRelationIdMap.get(sourceTableId);
             if (targetMappedRelations.isEmpty()) {
                 continue;
             }
             // if source and target relation appear once, just map them
             if (targetMappedRelations.size() == 1 && sourceMappedRelations.size() == 1) {
-                mappedRelations.add(ImmutableList.of(Pair.of(sourceMappedRelations.get(0),
-                        targetMappedRelations.get(0))));
+                ImmutableBiMap.Builder<MappedRelation, MappedRelation> biMapBuilder = ImmutableBiMap.builder();
+                mappedRelations.add(ImmutableList.of(
+                        biMapBuilder.put(sourceMappedRelations.iterator().next(),
+                                targetMappedRelations.iterator().next()).build()));
                 continue;
             }
-            // relation appear more than once, should cartesian them
-            ImmutableList<Pair<MappedRelation, MappedRelation>> relationMapping = Lists.cartesianProduct(
-                            sourceTableRelationIdMap.get(sourceTableQualifier), targetMappedRelations)
+            // relation appear more than once, should cartesian them and power set to correct combination
+            // if query is select * from tableA0, tableA1, materialized view is select * from tableA2, tableA3,
+            // tableA is the same table used by both query and materialized view
+            // relationMapping will be
+            // tableA0 tableA2
+            // tableA0 tableA3
+            // tableA1 tableA2
+            // tableA1 tableA3
+            ImmutableList<Pair<MappedRelation, MappedRelation>> relationMapping = Sets.cartesianProduct(
+                            sourceMappedRelations, targetMappedRelations)
                     .stream()
                     .map(listPair -> Pair.of(listPair.get(0), listPair.get(1)))
                     .collect(ImmutableList.toImmutableList());
-            mappedRelations.add(relationMapping);
-        }
-
-        int mappedRelationCount = mappedRelations.size();
 
-        return Lists.cartesianProduct(mappedRelations).stream()
-                .map(mappedRelationList -> {
-                    Builder<MappedRelation, MappedRelation> mapBuilder = ImmutableBiMap.builder();
-                    for (int relationIndex = 0; relationIndex < mappedRelationCount; relationIndex++) {
-                        mapBuilder.put(mappedRelationList.get(relationIndex).key(),
-                                mappedRelationList.get(relationIndex).value());
+            // the mapping in relationMappingPowerList should be bi-direction
+            // [
+            //    {tableA0 tableA2, tableA1 tableA3}
+            //    {tableA0 tableA3, tableA1 tableA2}
+            // ]
+            List<BiMap<MappedRelation, MappedRelation>> relationMappingPowerList = new ArrayList<>();
+            int relationMappingSize = relationMapping.size();
+            int relationMappingMinSize = Math.min(sourceMappedRelations.size(), targetMappedRelations.size());
+            for (int i = 0; i < relationMappingSize; i++) {
+                HashBiMap<MappedRelation, MappedRelation> relationBiMap = HashBiMap.create();
+                relationBiMap.put(relationMapping.get(i).key(), relationMapping.get(i).value());
+                for (int j = i + 1; j < relationMappingSize; j++) {
+                    if (!relationBiMap.containsKey(relationMapping.get(j).key())
+                            && !relationBiMap.containsValue(relationMapping.get(j).value())) {
+                        relationBiMap.put(relationMapping.get(j).key(), relationMapping.get(j).value());
                     }
-                    return RelationMapping.of(mapBuilder.build());
-                })
+                }
+                // mapping should contain min num of relation in source or target at least
+                if (relationBiMap.size() >= relationMappingMinSize) {
+                    relationMappingPowerList.add(relationBiMap);
+                }
+            }
+            mappedRelations.add(relationMappingPowerList);
+        }
+        // mappedRelations product and merge into each relationMapping
+        return Lists.cartesianProduct(mappedRelations).stream()
+                .map(RelationMapping::merge)
                 .collect(ImmutableList.toImmutableList());
     }
 
+    public static RelationMapping merge(List<BiMap<MappedRelation, MappedRelation>> relationMappings) {
+        Builder<MappedRelation, MappedRelation> mappingBuilder = ImmutableBiMap.builder();
+        for (BiMap<MappedRelation, MappedRelation> relationMapping : relationMappings) {
+            relationMapping.forEach(mappingBuilder::put);
+        }
+        return RelationMapping.of(mappingBuilder.build());
+    }
+
     private static Long getTableQualifier(TableIf tableIf) {
         return tableIf.getId();
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
index d818c15b4b9df7..4371b9f578594b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MappingTest.java
@@ -35,7 +35,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-/**MappingTest*/
+/**
+ * MappingTest
+ */
 public class MappingTest extends TestWithFeService {
 
     @Override
@@ -275,6 +277,72 @@ public void testGenerateMapping4() {
         assertRelationMapping(generateRelationMapping.get(1), expectedRelationMapping, expectedSlotMapping);
     }
 
+    @Test
+    public void testGenerateMapping5() {
+        Plan sourcePlan = PlanChecker.from(connectContext)
+                .analyze("SELECT orders.*, l1.* "
+                        + "FROM\n"
+                        + "  orders,\n"
+                        + "  lineitem l1,\n"
+                        + "  lineitem l2\n"
+                        + "WHERE\n"
+                        + "  l1.l_orderkey = l2.l_orderkey\n"
+                        + "  AND l1.l_orderkey = o_orderkey")
+                .getPlan();
+
+        Plan targetPlan = PlanChecker.from(connectContext)
+                .analyze("SELECT orders.*, l1.* "
+                        + "FROM\n"
+                        + "  lineitem l1,\n"
+                        + "  orders,\n"
+                        + "  lineitem l2\n"
+                        + "WHERE\n"
+                        + "  l1.l_orderkey = l2.l_orderkey\n"
+                        + " AND l2.l_orderkey = o_orderkey")
+                .getPlan();
+        List<CatalogRelation> sourceRelations = new ArrayList<>();
+        sourcePlan.accept(RelationCollector.INSTANCE, sourceRelations);
+
+        List<CatalogRelation> targetRelations = new ArrayList<>();
+        targetPlan.accept(RelationCollector.INSTANCE, targetRelations);
+
+        List<RelationMapping> generateRelationMapping = RelationMapping.generate(sourceRelations, targetRelations);
+        Assertions.assertNotNull(generateRelationMapping);
+        Assertions.assertEquals(2, generateRelationMapping.size());
+
+        // expected slot mapping
+        BiMap<ExprId, ExprId> expectedSlotMapping = HashBiMap.create();
+        expectedSlotMapping.put(new ExprId(0), new ExprId(2));
+        expectedSlotMapping.put(new ExprId(1), new ExprId(3));
+        expectedSlotMapping.put(new ExprId(2), new ExprId(4));
+        expectedSlotMapping.put(new ExprId(3), new ExprId(0));
+        expectedSlotMapping.put(new ExprId(4), new ExprId(1));
+        expectedSlotMapping.put(new ExprId(5), new ExprId(5));
+        expectedSlotMapping.put(new ExprId(6), new ExprId(6));
+        // expected relation mapping
+        BiMap<RelationId, RelationId> expectedRelationMapping = HashBiMap.create();
+        expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+        expectedRelationMapping.put(new RelationId(1), new RelationId(0));
+        expectedRelationMapping.put(new RelationId(2), new RelationId(2));
+        assertRelationMapping(generateRelationMapping.get(1), expectedRelationMapping, expectedSlotMapping);
+
+        // expected slot mapping
+        expectedSlotMapping = HashBiMap.create();
+        expectedSlotMapping.put(new ExprId(0), new ExprId(2));
+        expectedSlotMapping.put(new ExprId(1), new ExprId(3));
+        expectedSlotMapping.put(new ExprId(2), new ExprId(4));
+        expectedSlotMapping.put(new ExprId(3), new ExprId(5));
+        expectedSlotMapping.put(new ExprId(4), new ExprId(6));
+        expectedSlotMapping.put(new ExprId(5), new ExprId(0));
+        expectedSlotMapping.put(new ExprId(6), new ExprId(1));
+        // expected relation mapping
+        expectedRelationMapping = HashBiMap.create();
+        expectedRelationMapping.put(new RelationId(0), new RelationId(1));
+        expectedRelationMapping.put(new RelationId(1), new RelationId(2));
+        expectedRelationMapping.put(new RelationId(2), new RelationId(0));
+        assertRelationMapping(generateRelationMapping.get(0), expectedRelationMapping, expectedSlotMapping);
+    }
+
     private void assertRelationMapping(RelationMapping relationMapping,
             BiMap<RelationId, RelationId> expectRelationMapping,
             BiMap<ExprId, ExprId> expectSlotMapping) {
diff --git a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
index e8665ccb9484fc..4d0e131c5ff319 100644
--- a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
+++ b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
@@ -237,3 +237,17 @@
 -- !query7_0_after --
 3	3	2023-12-11
 
+-- !query8_0_before --
+1	0	8	0	10.0000	10.50	9.50
+2	0	2	0	11.5000	11.50	11.50
+3	0	0	0	23.0000	33.50	12.50
+4	0	0	0	43.2000	43.20	43.20
+5	0	0	0	28.7000	56.20	1.20
+
+-- !query8_0_after --
+1	0	8	0	10.0000	10.50	9.50
+2	0	2	0	11.5000	11.50	11.50
+3	0	0	0	23.0000	33.50	12.50
+4	0	0	0	43.2000	43.20	43.20
+5	0	0	0	28.7000	56.20	1.20
+
diff --git a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
index 37cf716c9428aa..a0717dfc0a352d 100644
--- a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
@@ -361,4 +361,43 @@ suite("outer_join") {
     order_qt_query7_0_after "${query7_0}"
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv7_0"""
 
+
+    // self join test
+    def mv8_0 = """
+    select 
+    a.o_orderkey,
+    count(distinct a.o_orderstatus) num1,
+    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate = '2023-12-08' AND b.o_orderdate = '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num2,
+    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate >= '2023-12-01' AND a.o_orderdate <= '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num3,
+    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority in (1,2) AND a.o_orderdate >= '2023-12-08' AND b.o_orderdate <= '2023-12-09' THEN a.o_shippriority-b.o_custkey ELSE 0 END) num4,
+    AVG(a.o_totalprice) num5,
+    MAX(b.o_totalprice) num6,
+    MIN(a.o_totalprice) num7
+    from
+    orders a
+    left outer join orders b
+    on a.o_orderkey = b.o_orderkey
+    and a.o_custkey = b.o_custkey
+    group by a.o_orderkey;
+    """
+    def query8_0 = """
+    select 
+    a.o_orderkey,
+    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate = '2023-12-08' AND b.o_orderdate = '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num2,
+    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority = 1 AND a.o_orderdate >= '2023-12-01' AND a.o_orderdate <= '2023-12-09' THEN a.o_shippriority+b.o_custkey ELSE 0 END) num3,
+    SUM(CASE WHEN a.o_orderstatus = 'o' AND a.o_shippriority in (1,2) AND a.o_orderdate >= '2023-12-08' AND b.o_orderdate <= '2023-12-09' THEN a.o_shippriority-b.o_custkey ELSE 0 END) num4,
+    AVG(a.o_totalprice) num5,
+    MAX(b.o_totalprice) num6,
+    MIN(a.o_totalprice) num7
+    from
+    orders a
+    left outer join orders b
+    on a.o_orderkey = b.o_orderkey
+    and a.o_custkey = b.o_custkey
+    group by a.o_orderkey;
+    """
+    order_qt_query8_0_before "${query8_0}"
+    check_rewrite(mv8_0, query8_0, "mv8_0")
+    order_qt_query8_0_after "${query8_0}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv8_0"""
 }

From f34b46a366ef2ce698df224a50943d91c175fd05 Mon Sep 17 00:00:00 2001
From: Mingyu Chen <morningman@163.com>
Date: Fri, 29 Dec 2023 13:50:30 +0800
Subject: [PATCH 155/299] [fix](glue) support amazonaws.com.cn endpoint
 (#29128)

---
 .../property/PropertyConverter.java           |  6 +++++-
 .../property/PropertyConverterTest.java       | 20 ++++++++++---------
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index a6cad30883939a..66396e2c337ca2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -550,7 +550,11 @@ private static Map<String, String> convertToGlueProperties(Map<String, String> p
         String region = S3Properties.getRegionOfEndpoint(endpoint);
         if (!Strings.isNullOrEmpty(region)) {
             props.put(S3Properties.REGION, region);
-            String s3Endpoint = "s3." + region + ".amazonaws.com";
+            String suffix = ".amazonaws.com";
+            if (endpoint.endsWith(".amazonaws.com.cn")) {
+                suffix = ".amazonaws.com.cn";
+            }
+            String s3Endpoint = "s3." + region + suffix;
             if (isGlueIceberg) {
                 s3Endpoint = "https://" + s3Endpoint;
             }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index cbd4b9b113df17..573ff48b99f6bb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -432,25 +432,27 @@ public void testGlueCatalogPropertiesConverter() throws Exception {
         String catalogName = "hms_glue_old";
         CreateCatalogStmt analyzedStmt = createStmt(queryOld);
         HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, catalogName);
-        Map<String, String> properties = catalog.getCatalogProperty().getProperties();
+        Map<String, String> properties = catalog.getProperties();
         Assertions.assertEquals(properties.size(), 20);
+        Assertions.assertEquals("s3.us-east-1.amazonaws.com", properties.get(S3Properties.ENDPOINT));
 
         Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
         Assertions.assertEquals(hdProps.size(), 29);
 
         String query = "create catalog hms_glue properties (\n"
-                    + "    'type'='hms',\n"
-                    + "    'hive.metastore.type'='glue',\n"
-                    + "    'hive.metastore.uris' = 'thrift://172.21.0.1:7004',\n"
-                    + "    'glue.endpoint' = 'glue.us-east-1.amazonaws.com',\n"
-                    + "    'glue.access_key' = 'akk',\n"
-                    + "    'glue.secret_key' = 'skk'\n"
-                    + ");";
+                + "    'type'='hms',\n"
+                + "    'hive.metastore.type'='glue',\n"
+                + "    'hive.metastore.uris' = 'thrift://172.21.0.1:7004',\n"
+                + "    'glue.endpoint' = 'glue.us-east-1.amazonaws.com.cn',\n"
+                + "    'glue.access_key' = 'akk',\n"
+                + "    'glue.secret_key' = 'skk'\n"
+                + ");";
         catalogName = "hms_glue";
         CreateCatalogStmt analyzedStmtNew = createStmt(query);
         HMSExternalCatalog catalogNew = createAndGetCatalog(analyzedStmtNew, catalogName);
-        Map<String, String> propertiesNew = catalogNew.getCatalogProperty().getProperties();
+        Map<String, String> propertiesNew = catalogNew.getProperties();
         Assertions.assertEquals(propertiesNew.size(), 20);
+        Assertions.assertEquals("s3.us-east-1.amazonaws.com.cn", propertiesNew.get(S3Properties.ENDPOINT));
 
         Map<String, String> hdPropsNew = catalogNew.getCatalogProperty().getHadoopProperties();
         Assertions.assertEquals(hdPropsNew.size(), 29);

From 738a1644ffd5157069be121f4541894d624098c8 Mon Sep 17 00:00:00 2001
From: zhiqiang <seuhezhiqiang@163.com>
Date: Fri, 29 Dec 2023 13:54:54 +0800
Subject: [PATCH 156/299] [refactor](operator) We dont need OperatorBuilder to
 instantiate Operator (#29133)

---
 .../pipeline/exec/aggregation_sink_operator.h |  2 +-
 .../exec/aggregation_source_operator.h        |  2 +-
 be/src/pipeline/exec/analytic_sink_operator.h |  2 +-
 .../pipeline/exec/analytic_source_operator.h  |  2 +-
 .../pipeline/exec/assert_num_rows_operator.h  |  2 +-
 be/src/pipeline/exec/const_value_operator.h   |  2 +-
 be/src/pipeline/exec/datagen_operator.h       |  2 +-
 ...inct_streaming_aggregation_sink_operator.h |  2 +-
 ...ct_streaming_aggregation_source_operator.h |  2 +-
 be/src/pipeline/exec/empty_set_operator.h     |  2 +-
 be/src/pipeline/exec/exchange_sink_operator.h |  2 +-
 .../pipeline/exec/exchange_source_operator.h  |  2 +-
 .../exec/group_commit_block_sink_operator.h   |  2 +-
 be/src/pipeline/exec/hashjoin_build_sink.h    |  2 +-
 .../pipeline/exec/hashjoin_probe_operator.h   |  2 +-
 .../exec/multi_cast_data_stream_sink.h        |  2 +-
 be/src/pipeline/exec/mysql_scan_operator.h    |  2 +-
 .../exec/nested_loop_join_build_operator.h    |  2 +-
 .../exec/nested_loop_join_probe_operator.h    |  2 +-
 .../pipeline/exec/olap_table_sink_operator.h  |  2 +-
 .../exec/olap_table_sink_v2_operator.h        |  2 +-
 be/src/pipeline/exec/operator.h               | 48 ++++++++-----------
 .../exec/partition_sort_sink_operator.h       |  2 +-
 .../exec/partition_sort_source_operator.h     |  3 +-
 be/src/pipeline/exec/repeat_operator.h        |  2 +-
 .../pipeline/exec/result_file_sink_operator.h |  2 +-
 be/src/pipeline/exec/result_sink_operator.h   |  2 +-
 be/src/pipeline/exec/scan_operator.h          |  2 +-
 be/src/pipeline/exec/schema_scan_operator.h   |  2 +-
 be/src/pipeline/exec/select_operator.h        |  2 +-
 .../pipeline/exec/set_probe_sink_operator.cpp |  3 +-
 .../pipeline/exec/set_probe_sink_operator.h   |  2 +-
 be/src/pipeline/exec/set_sink_operator.cpp    |  2 +-
 be/src/pipeline/exec/set_sink_operator.h      |  2 +-
 be/src/pipeline/exec/set_source_operator.cpp  |  2 +-
 be/src/pipeline/exec/set_source_operator.h    |  2 +-
 be/src/pipeline/exec/sort_sink_operator.h     |  2 +-
 be/src/pipeline/exec/sort_source_operator.h   |  2 +-
 .../streaming_aggregation_sink_operator.h     |  2 +-
 .../streaming_aggregation_source_operator.h   |  2 +-
 .../pipeline/exec/table_function_operator.h   |  2 +-
 be/src/pipeline/exec/table_sink_operator.h    |  2 +-
 be/src/pipeline/exec/union_sink_operator.h    |  2 +-
 be/src/pipeline/exec/union_source_operator.h  |  2 +-
 44 files changed, 63 insertions(+), 73 deletions(-)

diff --git a/be/src/pipeline/exec/aggregation_sink_operator.h b/be/src/pipeline/exec/aggregation_sink_operator.h
index 97be9dcd6a3e5a..d02e9a0f52ace5 100644
--- a/be/src/pipeline/exec/aggregation_sink_operator.h
+++ b/be/src/pipeline/exec/aggregation_sink_operator.h
@@ -39,7 +39,7 @@ class AggSinkOperatorBuilder final : public OperatorBuilder<vectorized::Aggregat
     bool is_sink() const override { return true; }
 };
 
-class AggSinkOperator final : public StreamingOperator<AggSinkOperatorBuilder> {
+class AggSinkOperator final : public StreamingOperator<vectorized::AggregationNode> {
 public:
     AggSinkOperator(OperatorBuilderBase* operator_builder, ExecNode* node);
     bool can_write() override { return true; }
diff --git a/be/src/pipeline/exec/aggregation_source_operator.h b/be/src/pipeline/exec/aggregation_source_operator.h
index c30277cbec50f2..a7125c42ff66d9 100644
--- a/be/src/pipeline/exec/aggregation_source_operator.h
+++ b/be/src/pipeline/exec/aggregation_source_operator.h
@@ -39,7 +39,7 @@ class AggSourceOperatorBuilder final : public OperatorBuilder<vectorized::Aggreg
     OperatorPtr build_operator() override;
 };
 
-class AggSourceOperator final : public SourceOperator<AggSourceOperatorBuilder> {
+class AggSourceOperator final : public SourceOperator<vectorized::AggregationNode> {
 public:
     AggSourceOperator(OperatorBuilderBase*, ExecNode*);
     // if exec node split to: sink, source operator. the source operator
diff --git a/be/src/pipeline/exec/analytic_sink_operator.h b/be/src/pipeline/exec/analytic_sink_operator.h
index 3e0eb85f76d17d..e99a0731b343a1 100644
--- a/be/src/pipeline/exec/analytic_sink_operator.h
+++ b/be/src/pipeline/exec/analytic_sink_operator.h
@@ -38,7 +38,7 @@ class AnalyticSinkOperatorBuilder final : public OperatorBuilder<vectorized::VAn
     bool is_sink() const override { return true; }
 };
 
-class AnalyticSinkOperator final : public StreamingOperator<AnalyticSinkOperatorBuilder> {
+class AnalyticSinkOperator final : public StreamingOperator<vectorized::VAnalyticEvalNode> {
 public:
     AnalyticSinkOperator(OperatorBuilderBase* operator_builder, ExecNode* node);
 
diff --git a/be/src/pipeline/exec/analytic_source_operator.h b/be/src/pipeline/exec/analytic_source_operator.h
index b9674b908c6da1..ba7b00be90611b 100644
--- a/be/src/pipeline/exec/analytic_source_operator.h
+++ b/be/src/pipeline/exec/analytic_source_operator.h
@@ -39,7 +39,7 @@ class AnalyticSourceOperatorBuilder final : public OperatorBuilder<vectorized::V
     OperatorPtr build_operator() override;
 };
 
-class AnalyticSourceOperator final : public SourceOperator<AnalyticSourceOperatorBuilder> {
+class AnalyticSourceOperator final : public SourceOperator<vectorized::VAnalyticEvalNode> {
 public:
     AnalyticSourceOperator(OperatorBuilderBase*, ExecNode*);
 
diff --git a/be/src/pipeline/exec/assert_num_rows_operator.h b/be/src/pipeline/exec/assert_num_rows_operator.h
index bb5e65168b631a..43036b597cf586 100644
--- a/be/src/pipeline/exec/assert_num_rows_operator.h
+++ b/be/src/pipeline/exec/assert_num_rows_operator.h
@@ -33,7 +33,7 @@ class AssertNumRowsOperatorBuilder final : public OperatorBuilder<vectorized::VA
     OperatorPtr build_operator() override;
 };
 
-class AssertNumRowsOperator final : public StreamingOperator<AssertNumRowsOperatorBuilder> {
+class AssertNumRowsOperator final : public StreamingOperator<vectorized::VAssertNumRowsNode> {
 public:
     AssertNumRowsOperator(OperatorBuilderBase* operator_builder, ExecNode* node)
             : StreamingOperator(operator_builder, node) {}
diff --git a/be/src/pipeline/exec/const_value_operator.h b/be/src/pipeline/exec/const_value_operator.h
index 120c6753ee8372..e87edae494b078 100644
--- a/be/src/pipeline/exec/const_value_operator.h
+++ b/be/src/pipeline/exec/const_value_operator.h
@@ -36,7 +36,7 @@ class ConstValueOperatorBuilder final : public OperatorBuilder<vectorized::VUnio
     bool is_source() const override { return true; }
 };
 
-class ConstValueOperator final : public SourceOperator<ConstValueOperatorBuilder> {
+class ConstValueOperator final : public SourceOperator<vectorized::VUnionNode> {
 public:
     ConstValueOperator(OperatorBuilderBase* operator_builder, ExecNode* node)
             : SourceOperator(operator_builder, node) {}
diff --git a/be/src/pipeline/exec/datagen_operator.h b/be/src/pipeline/exec/datagen_operator.h
index 1b8ef2977f9616..f34dfc61df01c3 100644
--- a/be/src/pipeline/exec/datagen_operator.h
+++ b/be/src/pipeline/exec/datagen_operator.h
@@ -38,7 +38,7 @@ class DataGenOperatorBuilder : public OperatorBuilder<vectorized::VDataGenFuncti
     OperatorPtr build_operator() override;
 };
 
-class DataGenOperator : public SourceOperator<DataGenOperatorBuilder> {
+class DataGenOperator : public SourceOperator<vectorized::VDataGenFunctionScanNode> {
 public:
     DataGenOperator(OperatorBuilderBase* operator_builder, ExecNode* datagen_node);
 
diff --git a/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h b/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h
index b30a8298727a95..d62178460ea96a 100644
--- a/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h
+++ b/be/src/pipeline/exec/distinct_streaming_aggregation_sink_operator.h
@@ -54,7 +54,7 @@ class DistinctStreamingAggSinkOperatorBuilder final
 };
 
 class DistinctStreamingAggSinkOperator final
-        : public StreamingOperator<DistinctStreamingAggSinkOperatorBuilder> {
+        : public StreamingOperator<vectorized::DistinctAggregationNode> {
 public:
     DistinctStreamingAggSinkOperator(OperatorBuilderBase* operator_builder, ExecNode*,
                                      std::shared_ptr<DataQueue>);
diff --git a/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h b/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h
index 0b4851ad97d7d6..81edf18567f19e 100644
--- a/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h
+++ b/be/src/pipeline/exec/distinct_streaming_aggregation_source_operator.h
@@ -51,7 +51,7 @@ class DistinctStreamingAggSourceOperatorBuilder final
 };
 
 class DistinctStreamingAggSourceOperator final
-        : public SourceOperator<DistinctStreamingAggSourceOperatorBuilder> {
+        : public SourceOperator<vectorized::DistinctAggregationNode> {
 public:
     DistinctStreamingAggSourceOperator(OperatorBuilderBase*, ExecNode*, std::shared_ptr<DataQueue>);
     bool can_read() override;
diff --git a/be/src/pipeline/exec/empty_set_operator.h b/be/src/pipeline/exec/empty_set_operator.h
index 126f132dd98a00..4e2607cd4ce882 100644
--- a/be/src/pipeline/exec/empty_set_operator.h
+++ b/be/src/pipeline/exec/empty_set_operator.h
@@ -37,7 +37,7 @@ class EmptySetSourceOperatorBuilder final : public OperatorBuilder<vectorized::V
     OperatorPtr build_operator() override;
 };
 
-class EmptySetSourceOperator final : public SourceOperator<EmptySetSourceOperatorBuilder> {
+class EmptySetSourceOperator final : public SourceOperator<vectorized::VEmptySetNode> {
 public:
     EmptySetSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* empty_set_node);
     bool can_read() override { return true; }
diff --git a/be/src/pipeline/exec/exchange_sink_operator.h b/be/src/pipeline/exec/exchange_sink_operator.h
index a34c4f4b435c04..1ddfc5f6174d07 100644
--- a/be/src/pipeline/exec/exchange_sink_operator.h
+++ b/be/src/pipeline/exec/exchange_sink_operator.h
@@ -46,7 +46,7 @@ class ExchangeSinkOperatorBuilder final
 };
 
 // Now local exchange is not supported since VDataStreamRecvr is considered as a pipeline broker.
-class ExchangeSinkOperator final : public DataSinkOperator<ExchangeSinkOperatorBuilder> {
+class ExchangeSinkOperator final : public DataSinkOperator<vectorized::VDataStreamSender> {
 public:
     ExchangeSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink, int mult_cast_id);
     Status init(const TDataSink& tsink) override;
diff --git a/be/src/pipeline/exec/exchange_source_operator.h b/be/src/pipeline/exec/exchange_source_operator.h
index b621da38072f78..bbccfe52987ad5 100644
--- a/be/src/pipeline/exec/exchange_source_operator.h
+++ b/be/src/pipeline/exec/exchange_source_operator.h
@@ -43,7 +43,7 @@ class ExchangeSourceOperatorBuilder final : public OperatorBuilder<vectorized::V
     OperatorPtr build_operator() override;
 };
 
-class ExchangeSourceOperator final : public SourceOperator<ExchangeSourceOperatorBuilder> {
+class ExchangeSourceOperator final : public SourceOperator<vectorized::VExchangeNode> {
 public:
     ExchangeSourceOperator(OperatorBuilderBase*, ExecNode*);
     bool can_read() override;
diff --git a/be/src/pipeline/exec/group_commit_block_sink_operator.h b/be/src/pipeline/exec/group_commit_block_sink_operator.h
index 0cf36818db1fa4..ad9198f4399f45 100644
--- a/be/src/pipeline/exec/group_commit_block_sink_operator.h
+++ b/be/src/pipeline/exec/group_commit_block_sink_operator.h
@@ -34,7 +34,7 @@ class GroupCommitBlockSinkOperatorBuilder final
 };
 
 class GroupCommitBlockSinkOperator final
-        : public DataSinkOperator<GroupCommitBlockSinkOperatorBuilder> {
+        : public DataSinkOperator<vectorized::GroupCommitBlockSink> {
 public:
     GroupCommitBlockSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink)
             : DataSinkOperator(operator_builder, sink) {}
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.h b/be/src/pipeline/exec/hashjoin_build_sink.h
index b5ae146c1828b2..fa4635afad1eca 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.h
+++ b/be/src/pipeline/exec/hashjoin_build_sink.h
@@ -37,7 +37,7 @@ class HashJoinBuildSinkBuilder final : public OperatorBuilder<vectorized::HashJo
     bool is_sink() const override { return true; }
 };
 
-class HashJoinBuildSink final : public StreamingOperator<HashJoinBuildSinkBuilder> {
+class HashJoinBuildSink final : public StreamingOperator<vectorized::HashJoinNode> {
 public:
     HashJoinBuildSink(OperatorBuilderBase* operator_builder, ExecNode* node);
     bool can_write() override { return _node->can_sink_write(); }
diff --git a/be/src/pipeline/exec/hashjoin_probe_operator.h b/be/src/pipeline/exec/hashjoin_probe_operator.h
index 5dde597ec762ed..cdb1bc05c93d61 100644
--- a/be/src/pipeline/exec/hashjoin_probe_operator.h
+++ b/be/src/pipeline/exec/hashjoin_probe_operator.h
@@ -36,7 +36,7 @@ class HashJoinProbeOperatorBuilder final : public OperatorBuilder<vectorized::Ha
     OperatorPtr build_operator() override;
 };
 
-class HashJoinProbeOperator final : public StatefulOperator<HashJoinProbeOperatorBuilder> {
+class HashJoinProbeOperator final : public StatefulOperator<vectorized::HashJoinNode> {
 public:
     HashJoinProbeOperator(OperatorBuilderBase*, ExecNode*);
     // if exec node split to: sink, source operator. the source operator
diff --git a/be/src/pipeline/exec/multi_cast_data_stream_sink.h b/be/src/pipeline/exec/multi_cast_data_stream_sink.h
index a30361ebe14383..965605fd3c726c 100644
--- a/be/src/pipeline/exec/multi_cast_data_stream_sink.h
+++ b/be/src/pipeline/exec/multi_cast_data_stream_sink.h
@@ -33,7 +33,7 @@ class MultiCastDataStreamSinkOperatorBuilder final
 };
 
 class MultiCastDataStreamSinkOperator final
-        : public DataSinkOperator<MultiCastDataStreamSinkOperatorBuilder> {
+        : public DataSinkOperator<vectorized::MultiCastDataStreamSink> {
 public:
     MultiCastDataStreamSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink)
             : DataSinkOperator(operator_builder, sink) {}
diff --git a/be/src/pipeline/exec/mysql_scan_operator.h b/be/src/pipeline/exec/mysql_scan_operator.h
index 3c5714a7f4ee86..6e21d8d2ebef36 100644
--- a/be/src/pipeline/exec/mysql_scan_operator.h
+++ b/be/src/pipeline/exec/mysql_scan_operator.h
@@ -29,7 +29,7 @@ class MysqlScanOperatorBuilder : public OperatorBuilder<vectorized::VMysqlScanNo
     OperatorPtr build_operator() override;
 };
 
-class MysqlScanOperator : public SourceOperator<MysqlScanOperatorBuilder> {
+class MysqlScanOperator : public SourceOperator<vectorized::VMysqlScanNode> {
 public:
     MysqlScanOperator(OperatorBuilderBase* operator_builder, ExecNode* mysql_scan_node);
 
diff --git a/be/src/pipeline/exec/nested_loop_join_build_operator.h b/be/src/pipeline/exec/nested_loop_join_build_operator.h
index ea0820253cc099..27f847cc00f4eb 100644
--- a/be/src/pipeline/exec/nested_loop_join_build_operator.h
+++ b/be/src/pipeline/exec/nested_loop_join_build_operator.h
@@ -38,7 +38,7 @@ class NestLoopJoinBuildOperatorBuilder final
     bool is_sink() const override { return true; }
 };
 
-class NestLoopJoinBuildOperator final : public StreamingOperator<NestLoopJoinBuildOperatorBuilder> {
+class NestLoopJoinBuildOperator final : public StreamingOperator<vectorized::VNestedLoopJoinNode> {
 public:
     NestLoopJoinBuildOperator(OperatorBuilderBase* operator_builder, ExecNode* node);
     bool can_write() override { return true; }
diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.h b/be/src/pipeline/exec/nested_loop_join_probe_operator.h
index 525640beb48519..1a3a0e9f9a16ed 100644
--- a/be/src/pipeline/exec/nested_loop_join_probe_operator.h
+++ b/be/src/pipeline/exec/nested_loop_join_probe_operator.h
@@ -40,7 +40,7 @@ class NestLoopJoinProbeOperatorBuilder final
     OperatorPtr build_operator() override;
 };
 
-class NestLoopJoinProbeOperator final : public StatefulOperator<NestLoopJoinProbeOperatorBuilder> {
+class NestLoopJoinProbeOperator final : public StatefulOperator<vectorized::VNestedLoopJoinNode> {
 public:
     NestLoopJoinProbeOperator(OperatorBuilderBase* operator_builder, ExecNode* node);
 
diff --git a/be/src/pipeline/exec/olap_table_sink_operator.h b/be/src/pipeline/exec/olap_table_sink_operator.h
index 55fea64c6346e4..6aa80776a3aab0 100644
--- a/be/src/pipeline/exec/olap_table_sink_operator.h
+++ b/be/src/pipeline/exec/olap_table_sink_operator.h
@@ -34,7 +34,7 @@ class OlapTableSinkOperatorBuilder final
     OperatorPtr build_operator() override;
 };
 
-class OlapTableSinkOperator final : public DataSinkOperator<OlapTableSinkOperatorBuilder> {
+class OlapTableSinkOperator final : public DataSinkOperator<vectorized::VOlapTableSink> {
 public:
     OlapTableSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink)
             : DataSinkOperator(operator_builder, sink) {}
diff --git a/be/src/pipeline/exec/olap_table_sink_v2_operator.h b/be/src/pipeline/exec/olap_table_sink_v2_operator.h
index 5c8cce62b8b5a8..72b6a15e733a93 100644
--- a/be/src/pipeline/exec/olap_table_sink_v2_operator.h
+++ b/be/src/pipeline/exec/olap_table_sink_v2_operator.h
@@ -34,7 +34,7 @@ class OlapTableSinkV2OperatorBuilder final
     OperatorPtr build_operator() override;
 };
 
-class OlapTableSinkV2Operator final : public DataSinkOperator<OlapTableSinkV2OperatorBuilder> {
+class OlapTableSinkV2Operator final : public DataSinkOperator<vectorized::VOlapTableSinkV2> {
 public:
     OlapTableSinkV2Operator(OperatorBuilderBase* operator_builder, DataSink* sink)
             : DataSinkOperator(operator_builder, sink) {}
diff --git a/be/src/pipeline/exec/operator.h b/be/src/pipeline/exec/operator.h
index 7ff91fba695aa2..bb0b8f30911d64 100644
--- a/be/src/pipeline/exec/operator.h
+++ b/be/src/pipeline/exec/operator.h
@@ -275,18 +275,17 @@ class OperatorBase {
 };
 
 /**
- * All operators inherited from DataSinkOperator will hold a SinkNode inside. Namely, it is a one-to-one relation between DataSinkOperator and DataSink.
+ * All operators inherited from DataSinkOperator will hold a SinkNode inside.
+ * Namely, it is a one-to-one relation between DataSinkOperator and DataSink.
  *
- * It should be mentioned that, not all SinkOperators are inherited from this (e.g. SortSinkOperator which holds a sort node inside instead of a DataSink).
+ * It should be mentioned that, not all SinkOperators are inherited from this
+ * (e.g. SortSinkOperator which holds a sort node inside instead of a DataSink).
  */
-template <typename OperatorBuilderType>
+template <typename DataSinkType>
 class DataSinkOperator : public OperatorBase {
 public:
-    using NodeType =
-            std::remove_pointer_t<decltype(std::declval<OperatorBuilderType>().exec_node())>;
-
     DataSinkOperator(OperatorBuilderBase* builder, DataSink* sink)
-            : OperatorBase(builder), _sink(reinterpret_cast<NodeType*>(sink)) {}
+            : OperatorBase(builder), _sink(reinterpret_cast<DataSinkType*>(sink)) {}
 
     ~DataSinkOperator() override = default;
 
@@ -323,20 +322,17 @@ class DataSinkOperator : public OperatorBase {
     }
 
 protected:
-    NodeType* _sink = nullptr;
+    DataSinkType* _sink = nullptr;
 };
 
 /**
  * All operators inherited from Operator will hold a ExecNode inside.
  */
-template <typename OperatorBuilderType>
+template <typename StreamingNodeType>
 class StreamingOperator : public OperatorBase {
 public:
-    using NodeType =
-            std::remove_pointer_t<decltype(std::declval<OperatorBuilderType>().exec_node())>;
-
     StreamingOperator(OperatorBuilderBase* builder, ExecNode* node)
-            : OperatorBase(builder), _node(reinterpret_cast<NodeType*>(node)) {}
+            : OperatorBase(builder), _node(reinterpret_cast<StreamingNodeType*>(node)) {}
 
     ~StreamingOperator() override = default;
 
@@ -400,24 +396,21 @@ class StreamingOperator : public OperatorBase {
     }
 
 protected:
-    NodeType* _node = nullptr;
+    StreamingNodeType* _node = nullptr;
     bool _use_projection;
 };
 
-template <typename OperatorBuilderType>
-class SourceOperator : public StreamingOperator<OperatorBuilderType> {
+template <typename SourceNodeType>
+class SourceOperator : public StreamingOperator<SourceNodeType> {
 public:
-    using NodeType =
-            std::remove_pointer_t<decltype(std::declval<OperatorBuilderType>().exec_node())>;
-
     SourceOperator(OperatorBuilderBase* builder, ExecNode* node)
-            : StreamingOperator<OperatorBuilderType>(builder, node) {}
+            : StreamingOperator<SourceNodeType>(builder, node) {}
 
     ~SourceOperator() override = default;
 
     Status get_block(RuntimeState* state, vectorized::Block* block,
                      SourceState& source_state) override {
-        auto& node = StreamingOperator<OperatorBuilderType>::_node;
+        auto& node = StreamingOperator<SourceNodeType>::_node;
         bool eos = false;
         RETURN_IF_ERROR(node->get_next_after_projects(
                 state, block, &eos,
@@ -436,14 +429,11 @@ class SourceOperator : public StreamingOperator<OperatorBuilderType> {
  * If there are still remain rows in probe block, we can get output block by calling `get_block` without any data from its child.
  * In a nutshell, it is a one-to-many relation between input blocks and output blocks for StatefulOperator.
  */
-template <typename OperatorBuilderType>
-class StatefulOperator : public StreamingOperator<OperatorBuilderType> {
+template <typename StatefulNodeType>
+class StatefulOperator : public StreamingOperator<StatefulNodeType> {
 public:
-    using NodeType =
-            std::remove_pointer_t<decltype(std::declval<OperatorBuilderType>().exec_node())>;
-
     StatefulOperator(OperatorBuilderBase* builder, ExecNode* node)
-            : StreamingOperator<OperatorBuilderType>(builder, node),
+            : StreamingOperator<StatefulNodeType>(builder, node),
               _child_block(vectorized::Block::create_shared()),
               _child_source_state(SourceState::DEPEND_ON_SOURCE) {}
 
@@ -451,8 +441,8 @@ class StatefulOperator : public StreamingOperator<OperatorBuilderType> {
 
     Status get_block(RuntimeState* state, vectorized::Block* block,
                      SourceState& source_state) override {
-        auto& node = StreamingOperator<OperatorBuilderType>::_node;
-        auto& child = StreamingOperator<OperatorBuilderType>::_child;
+        auto& node = StreamingOperator<StatefulNodeType>::_node;
+        auto& child = StreamingOperator<StatefulNodeType>::_child;
 
         if (node->need_more_input_data()) {
             _child_block->clear_column_data();
diff --git a/be/src/pipeline/exec/partition_sort_sink_operator.h b/be/src/pipeline/exec/partition_sort_sink_operator.h
index 486e7056213838..a46dc46433c37e 100644
--- a/be/src/pipeline/exec/partition_sort_sink_operator.h
+++ b/be/src/pipeline/exec/partition_sort_sink_operator.h
@@ -42,7 +42,7 @@ class PartitionSortSinkOperatorBuilder final
     OperatorPtr build_operator() override;
 };
 
-class PartitionSortSinkOperator final : public StreamingOperator<PartitionSortSinkOperatorBuilder> {
+class PartitionSortSinkOperator final : public StreamingOperator<vectorized::VPartitionSortNode> {
 public:
     PartitionSortSinkOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node)
             : StreamingOperator(operator_builder, sort_node) {};
diff --git a/be/src/pipeline/exec/partition_sort_source_operator.h b/be/src/pipeline/exec/partition_sort_source_operator.h
index 0b9556455325f9..d0ae26c7c4aadd 100644
--- a/be/src/pipeline/exec/partition_sort_source_operator.h
+++ b/be/src/pipeline/exec/partition_sort_source_operator.h
@@ -41,8 +41,7 @@ class PartitionSortSourceOperatorBuilder final
     OperatorPtr build_operator() override;
 };
 
-class PartitionSortSourceOperator final
-        : public SourceOperator<PartitionSortSourceOperatorBuilder> {
+class PartitionSortSourceOperator final : public SourceOperator<vectorized::VPartitionSortNode> {
 public:
     PartitionSortSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node)
             : SourceOperator(operator_builder, sort_node) {}
diff --git a/be/src/pipeline/exec/repeat_operator.h b/be/src/pipeline/exec/repeat_operator.h
index 03196a22bdcf3d..00555ce9dcffe7 100644
--- a/be/src/pipeline/exec/repeat_operator.h
+++ b/be/src/pipeline/exec/repeat_operator.h
@@ -36,7 +36,7 @@ class RepeatOperatorBuilder final : public OperatorBuilder<vectorized::VRepeatNo
     OperatorPtr build_operator() override;
 };
 
-class RepeatOperator final : public StatefulOperator<RepeatOperatorBuilder> {
+class RepeatOperator final : public StatefulOperator<vectorized::VRepeatNode> {
 public:
     RepeatOperator(OperatorBuilderBase* operator_builder, ExecNode* repeat_node);
 
diff --git a/be/src/pipeline/exec/result_file_sink_operator.h b/be/src/pipeline/exec/result_file_sink_operator.h
index 69235ee67070c9..f064b8f2b7f922 100644
--- a/be/src/pipeline/exec/result_file_sink_operator.h
+++ b/be/src/pipeline/exec/result_file_sink_operator.h
@@ -36,7 +36,7 @@ class ResultFileSinkOperatorBuilder final
     OperatorPtr build_operator() override;
 };
 
-class ResultFileSinkOperator final : public DataSinkOperator<ResultFileSinkOperatorBuilder> {
+class ResultFileSinkOperator final : public DataSinkOperator<vectorized::VResultFileSink> {
 public:
     ResultFileSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink);
 
diff --git a/be/src/pipeline/exec/result_sink_operator.h b/be/src/pipeline/exec/result_sink_operator.h
index bed6aed8969679..7a66738e4b4771 100644
--- a/be/src/pipeline/exec/result_sink_operator.h
+++ b/be/src/pipeline/exec/result_sink_operator.h
@@ -37,7 +37,7 @@ class ResultSinkOperatorBuilder final : public DataSinkOperatorBuilder<vectorize
     OperatorPtr build_operator() override;
 };
 
-class ResultSinkOperator final : public DataSinkOperator<ResultSinkOperatorBuilder> {
+class ResultSinkOperator final : public DataSinkOperator<vectorized::VResultSink> {
 public:
     ResultSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink);
 
diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h
index 833ca55f0957fc..3ad1ac1536a89d 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -43,7 +43,7 @@ class ScanOperatorBuilder : public OperatorBuilder<vectorized::VScanNode> {
     OperatorPtr build_operator() override;
 };
 
-class ScanOperator : public SourceOperator<ScanOperatorBuilder> {
+class ScanOperator : public SourceOperator<vectorized::VScanNode> {
 public:
     ScanOperator(OperatorBuilderBase* operator_builder, ExecNode* scan_node);
 
diff --git a/be/src/pipeline/exec/schema_scan_operator.h b/be/src/pipeline/exec/schema_scan_operator.h
index fdd3cb59312f90..2e00a2cbd4f582 100644
--- a/be/src/pipeline/exec/schema_scan_operator.h
+++ b/be/src/pipeline/exec/schema_scan_operator.h
@@ -38,7 +38,7 @@ class SchemaScanOperatorBuilder : public OperatorBuilder<vectorized::VSchemaScan
     OperatorPtr build_operator() override;
 };
 
-class SchemaScanOperator : public SourceOperator<SchemaScanOperatorBuilder> {
+class SchemaScanOperator : public SourceOperator<vectorized::VSchemaScanNode> {
 public:
     SchemaScanOperator(OperatorBuilderBase* operator_builder, ExecNode* scan_node);
 
diff --git a/be/src/pipeline/exec/select_operator.h b/be/src/pipeline/exec/select_operator.h
index d75c0a78481645..84a66528875b19 100644
--- a/be/src/pipeline/exec/select_operator.h
+++ b/be/src/pipeline/exec/select_operator.h
@@ -35,7 +35,7 @@ class SelectOperatorBuilder final : public OperatorBuilder<vectorized::VSelectNo
     OperatorPtr build_operator() override;
 };
 
-class SelectOperator final : public StreamingOperator<SelectOperatorBuilder> {
+class SelectOperator final : public StreamingOperator<vectorized::VSelectNode> {
 public:
     SelectOperator(OperatorBuilderBase* operator_builder, ExecNode* select_node);
 };
diff --git a/be/src/pipeline/exec/set_probe_sink_operator.cpp b/be/src/pipeline/exec/set_probe_sink_operator.cpp
index f1615df60da772..440936d59518ab 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.cpp
+++ b/be/src/pipeline/exec/set_probe_sink_operator.cpp
@@ -50,7 +50,8 @@ OperatorPtr SetProbeSinkOperatorBuilder<is_intersect>::build_operator() {
 template <bool is_intersect>
 SetProbeSinkOperator<is_intersect>::SetProbeSinkOperator(OperatorBuilderBase* operator_builder,
                                                          int child_id, ExecNode* set_node)
-        : StreamingOperator<SetProbeSinkOperatorBuilder<is_intersect>>(operator_builder, set_node),
+        : StreamingOperator<vectorized::VSetOperationNode<is_intersect>>(operator_builder,
+                                                                         set_node),
           _child_id(child_id) {}
 
 template <bool is_intersect>
diff --git a/be/src/pipeline/exec/set_probe_sink_operator.h b/be/src/pipeline/exec/set_probe_sink_operator.h
index 6f453ff31fcef0..6ac09ccfd9db6f 100644
--- a/be/src/pipeline/exec/set_probe_sink_operator.h
+++ b/be/src/pipeline/exec/set_probe_sink_operator.h
@@ -54,7 +54,7 @@ class SetProbeSinkOperatorBuilder final
 };
 
 template <bool is_intersect>
-class SetProbeSinkOperator : public StreamingOperator<SetProbeSinkOperatorBuilder<is_intersect>> {
+class SetProbeSinkOperator : public StreamingOperator<vectorized::VSetOperationNode<is_intersect>> {
 public:
     SetProbeSinkOperator(OperatorBuilderBase* operator_builder, int child_id, ExecNode* set_node);
 
diff --git a/be/src/pipeline/exec/set_sink_operator.cpp b/be/src/pipeline/exec/set_sink_operator.cpp
index d46c69223221a1..367ff41acce104 100644
--- a/be/src/pipeline/exec/set_sink_operator.cpp
+++ b/be/src/pipeline/exec/set_sink_operator.cpp
@@ -42,7 +42,7 @@ OperatorPtr SetSinkOperatorBuilder<is_intersect>::build_operator() {
 template <bool is_intersect>
 SetSinkOperator<is_intersect>::SetSinkOperator(
         OperatorBuilderBase* builder, vectorized::VSetOperationNode<is_intersect>* set_node)
-        : StreamingOperator<SetSinkOperatorBuilder<is_intersect>>(builder, set_node) {}
+        : StreamingOperator<vectorized::VSetOperationNode<is_intersect>>(builder, set_node) {}
 
 template class SetSinkOperatorBuilder<true>;
 template class SetSinkOperatorBuilder<false>;
diff --git a/be/src/pipeline/exec/set_sink_operator.h b/be/src/pipeline/exec/set_sink_operator.h
index 635d1ee86753d4..b851e18ce5b952 100644
--- a/be/src/pipeline/exec/set_sink_operator.h
+++ b/be/src/pipeline/exec/set_sink_operator.h
@@ -49,7 +49,7 @@ class SetSinkOperatorBuilder final
 };
 
 template <bool is_intersect>
-class SetSinkOperator : public StreamingOperator<SetSinkOperatorBuilder<is_intersect>> {
+class SetSinkOperator : public StreamingOperator<vectorized::VSetOperationNode<is_intersect>> {
 public:
     SetSinkOperator(OperatorBuilderBase* operator_builder,
                     vectorized::VSetOperationNode<is_intersect>* set_node);
diff --git a/be/src/pipeline/exec/set_source_operator.cpp b/be/src/pipeline/exec/set_source_operator.cpp
index e8a73c00ad6232..e74e8b22ec0cf9 100644
--- a/be/src/pipeline/exec/set_source_operator.cpp
+++ b/be/src/pipeline/exec/set_source_operator.cpp
@@ -42,7 +42,7 @@ OperatorPtr SetSourceOperatorBuilder<is_intersect>::build_operator() {
 template <bool is_intersect>
 SetSourceOperator<is_intersect>::SetSourceOperator(
         OperatorBuilderBase* builder, vectorized::VSetOperationNode<is_intersect>* set_node)
-        : SourceOperator<SetSourceOperatorBuilder<is_intersect>>(builder, set_node) {}
+        : SourceOperator<vectorized::VSetOperationNode<is_intersect>>(builder, set_node) {}
 
 template class SetSourceOperatorBuilder<true>;
 template class SetSourceOperatorBuilder<false>;
diff --git a/be/src/pipeline/exec/set_source_operator.h b/be/src/pipeline/exec/set_source_operator.h
index 44800f23f417d1..caf3e447f56afd 100644
--- a/be/src/pipeline/exec/set_source_operator.h
+++ b/be/src/pipeline/exec/set_source_operator.h
@@ -45,7 +45,7 @@ class SetSourceOperatorBuilder
 };
 
 template <bool is_intersect>
-class SetSourceOperator : public SourceOperator<SetSourceOperatorBuilder<is_intersect>> {
+class SetSourceOperator : public SourceOperator<vectorized::VSetOperationNode<is_intersect>> {
 public:
     SetSourceOperator(OperatorBuilderBase* builder,
                       vectorized::VSetOperationNode<is_intersect>* set_node);
diff --git a/be/src/pipeline/exec/sort_sink_operator.h b/be/src/pipeline/exec/sort_sink_operator.h
index 2f5512e108be85..64beb53ba9e4dd 100644
--- a/be/src/pipeline/exec/sort_sink_operator.h
+++ b/be/src/pipeline/exec/sort_sink_operator.h
@@ -38,7 +38,7 @@ class SortSinkOperatorBuilder final : public OperatorBuilder<vectorized::VSortNo
     OperatorPtr build_operator() override;
 };
 
-class SortSinkOperator final : public StreamingOperator<SortSinkOperatorBuilder> {
+class SortSinkOperator final : public StreamingOperator<vectorized::VSortNode> {
 public:
     SortSinkOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node);
 
diff --git a/be/src/pipeline/exec/sort_source_operator.h b/be/src/pipeline/exec/sort_source_operator.h
index efbddfd1050c6d..216c6290700202 100644
--- a/be/src/pipeline/exec/sort_source_operator.h
+++ b/be/src/pipeline/exec/sort_source_operator.h
@@ -39,7 +39,7 @@ class SortSourceOperatorBuilder final : public OperatorBuilder<vectorized::VSort
     OperatorPtr build_operator() override;
 };
 
-class SortSourceOperator final : public SourceOperator<SortSourceOperatorBuilder> {
+class SortSourceOperator final : public SourceOperator<vectorized::VSortNode> {
 public:
     SortSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* sort_node);
     Status open(RuntimeState*) override { return Status::OK(); }
diff --git a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h b/be/src/pipeline/exec/streaming_aggregation_sink_operator.h
index a7fcdcf847b761..16f28bbeb76821 100644
--- a/be/src/pipeline/exec/streaming_aggregation_sink_operator.h
+++ b/be/src/pipeline/exec/streaming_aggregation_sink_operator.h
@@ -50,7 +50,7 @@ class StreamingAggSinkOperatorBuilder final : public OperatorBuilder<vectorized:
     std::shared_ptr<DataQueue> _data_queue;
 };
 
-class StreamingAggSinkOperator final : public StreamingOperator<StreamingAggSinkOperatorBuilder> {
+class StreamingAggSinkOperator final : public StreamingOperator<vectorized::AggregationNode> {
 public:
     StreamingAggSinkOperator(OperatorBuilderBase* operator_builder, ExecNode*,
                              std::shared_ptr<DataQueue>);
diff --git a/be/src/pipeline/exec/streaming_aggregation_source_operator.h b/be/src/pipeline/exec/streaming_aggregation_source_operator.h
index 3e05955470db22..51e3c9ed972942 100644
--- a/be/src/pipeline/exec/streaming_aggregation_source_operator.h
+++ b/be/src/pipeline/exec/streaming_aggregation_source_operator.h
@@ -49,7 +49,7 @@ class StreamingAggSourceOperatorBuilder final
     std::shared_ptr<DataQueue> _data_queue;
 };
 
-class StreamingAggSourceOperator final : public SourceOperator<StreamingAggSourceOperatorBuilder> {
+class StreamingAggSourceOperator final : public SourceOperator<vectorized::AggregationNode> {
 public:
     StreamingAggSourceOperator(OperatorBuilderBase*, ExecNode*, std::shared_ptr<DataQueue>);
     bool can_read() override;
diff --git a/be/src/pipeline/exec/table_function_operator.h b/be/src/pipeline/exec/table_function_operator.h
index 2198f336c8ea4a..a6c90e3dec4759 100644
--- a/be/src/pipeline/exec/table_function_operator.h
+++ b/be/src/pipeline/exec/table_function_operator.h
@@ -38,7 +38,7 @@ class TableFunctionOperatorBuilder final : public OperatorBuilder<vectorized::VT
     OperatorPtr build_operator() override;
 };
 
-class TableFunctionOperator final : public StatefulOperator<TableFunctionOperatorBuilder> {
+class TableFunctionOperator final : public StatefulOperator<vectorized::VTableFunctionNode> {
 public:
     TableFunctionOperator(OperatorBuilderBase* operator_builder, ExecNode* node);
 
diff --git a/be/src/pipeline/exec/table_sink_operator.h b/be/src/pipeline/exec/table_sink_operator.h
index 429c350eda51b7..46843c23f870b3 100644
--- a/be/src/pipeline/exec/table_sink_operator.h
+++ b/be/src/pipeline/exec/table_sink_operator.h
@@ -33,7 +33,7 @@ class TableSinkOperatorBuilder final : public DataSinkOperatorBuilder<DataSink>
     OperatorPtr build_operator() override;
 };
 
-class TableSinkOperator final : public DataSinkOperator<TableSinkOperatorBuilder> {
+class TableSinkOperator final : public DataSinkOperator<DataSink> {
 public:
     TableSinkOperator(OperatorBuilderBase* operator_builder, DataSink* sink)
             : DataSinkOperator(operator_builder, sink) {}
diff --git a/be/src/pipeline/exec/union_sink_operator.h b/be/src/pipeline/exec/union_sink_operator.h
index f3031dd101535f..ed51e8b7398674 100644
--- a/be/src/pipeline/exec/union_sink_operator.h
+++ b/be/src/pipeline/exec/union_sink_operator.h
@@ -48,7 +48,7 @@ class UnionSinkOperatorBuilder final : public OperatorBuilder<vectorized::VUnion
     std::shared_ptr<DataQueue> _data_queue;
 };
 
-class UnionSinkOperator final : public StreamingOperator<UnionSinkOperatorBuilder> {
+class UnionSinkOperator final : public StreamingOperator<vectorized::VUnionNode> {
 public:
     UnionSinkOperator(OperatorBuilderBase* operator_builder, int child_id, ExecNode* node,
                       std::shared_ptr<DataQueue> queue);
diff --git a/be/src/pipeline/exec/union_source_operator.h b/be/src/pipeline/exec/union_source_operator.h
index 8d1b25bc4c412f..0c150a072b65b4 100644
--- a/be/src/pipeline/exec/union_source_operator.h
+++ b/be/src/pipeline/exec/union_source_operator.h
@@ -48,7 +48,7 @@ class UnionSourceOperatorBuilder final : public OperatorBuilder<vectorized::VUni
     std::shared_ptr<DataQueue> _data_queue;
 };
 
-class UnionSourceOperator final : public SourceOperator<UnionSourceOperatorBuilder> {
+class UnionSourceOperator final : public SourceOperator<vectorized::VUnionNode> {
 public:
     UnionSourceOperator(OperatorBuilderBase* operator_builder, ExecNode* node,
                         std::shared_ptr<DataQueue>);

From 36b45b570b0c16ac0a4cbe8f05e6a7ea0b946851 Mon Sep 17 00:00:00 2001
From: fornaix <foxnaix@gmail.com>
Date: Fri, 29 Dec 2023 13:57:22 +0800
Subject: [PATCH 157/299] [fix](mysql) fix mysql channel infinite blocking
 (#28808)

Call the Channels blocking method with timeout instead.

Using session variables net_write_timeout and net_read_timeout as the timeout parameter.
---
 .../java/org/apache/doris/mysql/MysqlChannel.java  | 14 ++++++++++----
 .../java/org/apache/doris/qe/ConnectContext.java   | 10 +++++++++-
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java
index 5eaee47fa4b377..8e7c5f79ffd0ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlChannel.java
@@ -31,6 +31,7 @@
 import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.ByteBuffer;
+import java.util.concurrent.TimeUnit;
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLEngineResult;
 import javax.net.ssl.SSLException;
@@ -79,6 +80,8 @@ public class MysqlChannel {
     // mysql flag CLIENT_DEPRECATE_EOF
     private boolean clientDeprecatedEOF;
 
+    private ConnectContext context;
+
     protected MysqlChannel() {
         // For DummyMysqlChannel
     }
@@ -91,7 +94,7 @@ public boolean clientDeprecatedEOF() {
         return clientDeprecatedEOF;
     }
 
-    public MysqlChannel(StreamConnection connection) {
+    public MysqlChannel(StreamConnection connection, ConnectContext context) {
         Preconditions.checkNotNull(connection);
         this.sequenceId = 0;
         this.isSend = false;
@@ -113,6 +116,7 @@ public MysqlChannel(StreamConnection connection) {
         this.defaultBuffer = ByteBuffer.allocate(16 * 1024);
         this.headerByteBuffer = ByteBuffer.allocate(PACKET_HEADER_LEN);
         this.sendBuffer = ByteBuffer.allocate(2 * 1024 * 1024);
+        this.context = context;
     }
 
     public void initSslBuffer() {
@@ -195,7 +199,8 @@ protected int readAll(ByteBuffer dstBuf, boolean isHeader) throws IOException {
         }
         try {
             while (dstBuf.remaining() != 0) {
-                int ret = Channels.readBlocking(conn.getSourceChannel(), dstBuf);
+                int ret = Channels.readBlocking(conn.getSourceChannel(), dstBuf, context.getNetReadTimeout(),
+                        TimeUnit.SECONDS);
                 // return -1 when remote peer close the channel
                 if (ret == -1) {
                     decryptData(dstBuf, isHeader);
@@ -365,12 +370,13 @@ private ByteBuffer expandPacket(ByteBuffer result, int packetLen) {
     protected void realNetSend(ByteBuffer buffer) throws IOException {
         buffer = encryptData(buffer);
         long bufLen = buffer.remaining();
-        long writeLen = Channels.writeBlocking(conn.getSinkChannel(), buffer);
+        long writeLen = Channels.writeBlocking(conn.getSinkChannel(), buffer, context.getNetWriteTimeout(),
+                TimeUnit.SECONDS);
         if (bufLen != writeLen) {
             throw new IOException("Write mysql packet failed.[write=" + writeLen
                     + ", needToWrite=" + bufLen + "]");
         }
-        Channels.flushBlocking(conn.getSinkChannel());
+        Channels.flushBlocking(conn.getSinkChannel(), context.getNetWriteTimeout(), TimeUnit.SECONDS);
         isSend = true;
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index a3590a23c4bee2..8a1a16999b6a14 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -327,7 +327,7 @@ public ConnectContext(StreamConnection connection) {
         connectType = ConnectType.MYSQL;
         serverCapability = MysqlCapability.DEFAULT_CAPABILITY;
         if (connection != null) {
-            mysqlChannel = new MysqlChannel(connection);
+            mysqlChannel = new MysqlChannel(connection, this);
         } else {
             mysqlChannel = new DummyMysqlChannel();
         }
@@ -1033,5 +1033,13 @@ public boolean isSkipAuth() {
     public void setSkipAuth(boolean skipAuth) {
         this.skipAuth = skipAuth;
     }
+
+    public int getNetReadTimeout() {
+        return this.sessionVariable.getNetReadTimeout();
+    }
+
+    public int getNetWriteTimeout() {
+        return this.sessionVariable.getNetWriteTimeout();
+    }
 }
 

From 4fc5860a7c2dca18a643326deab73abfdb0caf1f Mon Sep 17 00:00:00 2001
From: xy <whuxingying@163.com>
Date: Fri, 29 Dec 2023 14:15:37 +0800
Subject: [PATCH 158/299] [Fix](doc)Fix word spelling errors (#29268)

Co-authored-by: xingying01 <xingying01@corp.netease.com>
---
 .../docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md  | 2 +-
 .../docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md
index 3b3088c126fd2c..949c3f0ae3dd7e 100644
--- a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md
+++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md
@@ -97,7 +97,7 @@ illustrate:
 11. current_query_stmts: Returns the currently executing query.
 12. dbs: Mainly used to view the metadata information of each database and the tables in the Doris cluster. This information includes table structure, partitions, materialized views, data shards and replicas, and more. Through this directory and its subdirectories, you can clearly display the table metadata in the cluster, and locate some problems such as data skew, replica failure, etc.
 13. diagnose: Report and diagnose common management and control issues in the cluster, including replica balance and migration, transaction exceptions, etc.
-14. frontends: Display all FE node information in the cluster, including IP address, role, status, whether it is a mater, etc., equivalent to [SHOW FRONTENDS](./SHOW-FRONTENDS.md)
+14. frontends: Display all FE node information in the cluster, including IP address, role, status, whether it is a master, etc., equivalent to [SHOW FRONTENDS](./SHOW-FRONTENDS.md)
 15. jobs: show statistics of all kind of jobs. If a specific `dbId` is given, will return statistics data of the database. If `dbId` is -1, will return total statistics data of all databases
 16. load_error_hub: Doris supports centralized storage of error information generated by load jobs in an error hub. Then view the error message directly through the <code>SHOW LOAD WARNINGS;</code> statement. Shown here is the configuration information of the error hub.
 17. monitor : shows the resource usage of FE JVM
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md
index dd174009b15404..2744d83ee68806 100644
--- a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-PROC.md
@@ -97,7 +97,7 @@ mysql> show proc "/";
 11. current_query_stmts : 返回当前正在执行的 query。
 12. dbs ： 主要用于查看 Doris 集群中各个数据库以及其中的表的元数据信息。这些信息包括表结构、分区、物化视图、数据分片和副本等等。通过这个目录和其子目录，可以清楚的展示集群中的表元数据情况，以及定位一些如数据倾斜、副本故障等问题
 13. diagnose : 报告和诊断集群中的常见管控问题，主要包括副本均衡和迁移、事务异常等
-14. frontends ：显示集群中所有的 FE 节点信息，包括IP地址、角色、状态、是否是mater等，等同于 [SHOW FRONTENDS](./SHOW-FRONTENDS.md)
+14. frontends ：显示集群中所有的 FE 节点信息，包括IP地址、角色、状态、是否是master等，等同于 [SHOW FRONTENDS](./SHOW-FRONTENDS.md)
 15. jobs ：各类任务的统计信息，可查看指定数据库的 Job 的统计信息，如果 `dbId` = -1, 则返回所有库的汇总信息
 16. load_error_hub ：Doris 支持将 load 作业产生的错误信息集中存储到一个 error hub 中。然后直接通过 <code>SHOW LOAD WARNINGS;</code> 语句查看错误信息。这里展示的就是 error hub 的配置信息。
 17. monitor : 显示的是 FE JVM 的资源使用情况

From 02b27c16731702df4b1b6776471cca965bddecac Mon Sep 17 00:00:00 2001
From: Dongyang Li <hello_stephen@qq.com>
Date: Fri, 29 Dec 2023 14:21:23 +0800
Subject: [PATCH 159/299] [ci](trigger) support "run performance" (#29259)

* [ci](trigger) support "run performance"
Co-authored-by: stephen <hello-stephen@qq.com>
---
 .../workflows/comment-to-trigger-teamcity.yml | 23 +++++++++++++++++--
 .../pipeline/common/github-utils.sh           |  2 +-
 .../pipeline/common/teamcity-utils.sh         |  2 ++
 .../pipeline/performance/prepare.sh           |  4 ++--
 4 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml
index eb81eff09e5fc9..7634a8ead5b0c6 100644
--- a/.github/workflows/comment-to-trigger-teamcity.yml
+++ b/.github/workflows/comment-to-trigger-teamcity.yml
@@ -47,7 +47,8 @@ jobs:
             "${COMMENT_BODY}" == *'run pipelinex_p0'* ||
             "${COMMENT_BODY}" == *'run clickbench'* ||
             "${COMMENT_BODY}" == *'run arm'* ||
-            "${COMMENT_BODY}" == *'run tpch'* ]]; then
+            "${COMMENT_BODY}" == *'run tpch'* ||
+            "${COMMENT_BODY}" == *'run performance'* ]]; then
             echo "comment_trigger=true" | tee -a "$GITHUB_OUTPUT"
         else
             echo "comment_trigger=false" | tee -a "$GITHUB_OUTPUT"
@@ -63,7 +64,7 @@ jobs:
         echo "TARGET_BRANCH='${TARGET_BRANCH}'" | tee -a "$GITHUB_OUTPUT"
         echo "COMMENT_BODY='${COMMENT_BODY}'" | tee -a "$GITHUB_OUTPUT"
 
-        reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch)( [1-9]*[0-9]+)*"
+        reg="run (buildall|compile|p0|p1|feut|beut|external|clickbench|pipelinex_p0|arm|tpch|performance)( [1-9]*[0-9]+)*"
         COMMENT_TRIGGER_TYPE="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $2}' | sed -n 1p | sed 's/\r//g')"
         COMMENT_REPEAT_TIMES="$(echo -e "${COMMENT_BODY}" | xargs | grep -E "${reg}" | awk -F' ' '{print $3}' | sed -n 1p | sed 's/\r//g')"
         echo "COMMENT_TRIGGER_TYPE=${COMMENT_TRIGGER_TYPE}" | tee -a "$GITHUB_OUTPUT"
@@ -113,6 +114,11 @@ jobs:
           else
             echo "changed_p1=false" | tee -a "$GITHUB_OUTPUT"
           fi
+          if file_changed_performance; then
+            echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
+          else
+            echo "changed_performance=false" | tee -a "$GITHUB_OUTPUT"
+          fi
         else
           echo "INFO: failed to _get_pr_changed_files, default trigger all"
           echo "changed_fe_ut=true" | tee -a "$GITHUB_OUTPUT"
@@ -124,6 +130,7 @@ jobs:
           echo "changed_pipelinex_p0=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_arm=true" | tee -a "$GITHUB_OUTPUT"
           echo "changed_p1=true" | tee -a "$GITHUB_OUTPUT"
+          echo "changed_performance=true" | tee -a "$GITHUB_OUTPUT"
         fi
 
     # - name: "Setup tmate session"
@@ -266,3 +273,15 @@ jobs:
           "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
           "tpch" \
           "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+
+    - name: "Trigger or Skip performance"
+      if: ${{ fromJSON(steps.parse.outputs.comment_trigger) && contains(fromJSON('["performance", "buildall"]'), steps.parse.outputs.COMMENT_TRIGGER_TYPE) }}
+      run: |
+        source ./regression-test/pipeline/common/teamcity-utils.sh
+        set -x
+        trigger_or_skip_build \
+          "${{ steps.changes.outputs.changed_performance }}" \
+          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+          "performance" \
+          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
diff --git a/regression-test/pipeline/common/github-utils.sh b/regression-test/pipeline/common/github-utils.sh
index ecf7a055df0d25..687d9c589b93ad 100644
--- a/regression-test/pipeline/common/github-utils.sh
+++ b/regression-test/pipeline/common/github-utils.sh
@@ -298,7 +298,7 @@ file_changed_ckb() {
     echo "return no need" && return 1
 }
 
-file_changed_perf() {
+file_changed_performance() {
     local all_files
     all_files=$(cat all_files)
     if _only_modified_regression_conf; then echo "return no need" && return 1; fi
diff --git a/regression-test/pipeline/common/teamcity-utils.sh b/regression-test/pipeline/common/teamcity-utils.sh
index fff7048ce17bf2..8e780d6f83f67e 100644
--- a/regression-test/pipeline/common/teamcity-utils.sh
+++ b/regression-test/pipeline/common/teamcity-utils.sh
@@ -35,6 +35,7 @@ comment_to_pipeline=(
     ['pipelinex_p0']='Doris_DorisRegression_P0RegressionPipelineX'
     ['arm']='Doris_ArmPipeline_P0Regression'
     ['tpch']='Tpch_TpchSf100'
+    ['performance']='Doris_PerformanceNew_Performance'
 )
 
 # github中评论的要触发的流水线名字
@@ -54,6 +55,7 @@ conment_to_context=(
     ['clickbench']='clickbench-new (clickbench)'
     ['arm']='P0 Regression (ARM pipeline)'
     ['tpch']='tpch-sf100 (tpch)'
+    ['performance']='performance (Performance New)'
 )
 
 get_commit_id_of_build() {
diff --git a/regression-test/pipeline/performance/prepare.sh b/regression-test/pipeline/performance/prepare.sh
index c96a0c8629474e..b3b54b85b28439 100644
--- a/regression-test/pipeline/performance/prepare.sh
+++ b/regression-test/pipeline/performance/prepare.sh
@@ -68,10 +68,10 @@ fi
 source "$(bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'get')"
 if ${skip_pipeline:=false}; then echo "INFO: skip build pipline" && exit 0; else echo "INFO: no skip"; fi
 # shellcheck source=/dev/null
-# _get_pr_changed_files file_changed_perf
+# _get_pr_changed_files file_changed_performance
 source "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/github-utils.sh
 if _get_pr_changed_files "${pull_request_num}"; then
-    if ! file_changed_perf; then
+    if ! file_changed_performance; then
         bash "${teamcity_build_checkoutDir}"/regression-test/pipeline/common/get-or-set-tmp-env.sh 'set' "export skip_pipeline=true"
         exit 0
     fi

From 2308881e9f593d2fc87d397c7e11917eb5bbedd6 Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Fri, 29 Dec 2023 14:36:48 +0800
Subject: [PATCH 160/299] [improvement](statistics) Analyze partition columns
 when new partition loaded data for the first time. (#29154)

The first time load data to a partition, we need to analyze the partition columns even when the health rate is high. Because if not, the min max value of the column may not include the new partition values, which may cause bad plan.
---
 .../doris/analysis/ShowTableStatsStmt.java    |  2 ++
 .../doris/statistics/AnalysisManager.java     |  9 +++++
 .../statistics/StatisticsAutoCollector.java   | 34 ++++++++++++------
 .../doris/statistics/TableStatsMeta.java      | 14 ++++++++
 .../transaction/DatabaseTransactionMgr.java   |  6 +++-
 .../StatisticsAutoCollectorTest.java          |  5 +++
 .../suites/statistics/analyze_stats.groovy    | 36 +++++++++++++++++++
 7 files changed, 94 insertions(+), 12 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
index 3a80daebc913d1..284b6248b85b5f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowTableStatsStmt.java
@@ -54,6 +54,7 @@ public class ShowTableStatsStmt extends ShowStmt {
                     .add("updated_time")
                     .add("columns")
                     .add("trigger")
+                    .add("new_partition")
                     .build();
 
     private final TableName tableName;
@@ -149,6 +150,7 @@ public ShowResultSet constructResultSet(TableStatsMeta tableStatistic) {
         row.add(formattedDateTime);
         row.add(tableStatistic.analyzeColumns().toString());
         row.add(tableStatistic.jobType.toString());
+        row.add(String.valueOf(tableStatistic.newPartitionLoaded.get()));
         result.add(row);
         return new ShowResultSet(getMetaData(), result);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 0a52cc8faeaa9a..63ae94e37d9e39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -933,6 +933,15 @@ public void updateUpdatedRows(long tblId, long rows) {
         }
     }
 
+    // Set to true means new partition loaded data
+    public void setNewPartitionLoaded(long tblId) {
+        TableStatsMeta statsStatus = idToTblStats.get(tblId);
+        if (statsStatus != null) {
+            statsStatus.newPartitionLoaded.set(true);
+            logCreateTableStats(statsStatus);
+        }
+    }
+
     public void updateTableStatsStatus(TableStatsMeta tableStats) {
         replayUpdateTableStatsStatus(tableStats);
         logCreateTableStats(tableStats);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 17d666b7ea4a2a..fe90a2a59d44cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -21,6 +21,7 @@
 import org.apache.doris.catalog.DatabaseIf;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Partition;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.external.HMSExternalTable;
 import org.apache.doris.common.Config;
@@ -38,6 +39,7 @@
 
 import java.time.LocalTime;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -152,8 +154,8 @@ protected boolean skip(TableIf table) {
             return false;
         }
         TableStatsMeta tableStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
-        // means it's never got analyzed
-        if (tableStats == null) {
+        // means it's never got analyzed or new partition loaded data.
+        if (tableStats == null || tableStats.newPartitionLoaded.get()) {
             return false;
         }
         return System.currentTimeMillis()
@@ -191,8 +193,7 @@ protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
 
     @VisibleForTesting
     protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
-        TableIf table = StatisticsUtil
-                .findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId);
+        TableIf table = StatisticsUtil.findTable(jobInfo.catalogId, jobInfo.dbId, jobInfo.tblId);
         // Skip tables that are too width.
         if (table.getBaseSchema().size() > StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) {
             return null;
@@ -200,16 +201,27 @@ protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
 
         AnalysisManager analysisManager = Env.getServingEnv().getAnalysisManager();
         TableStatsMeta tblStats = analysisManager.findTableStatsStatus(table.getId());
-        if (!table.needReAnalyzeTable(tblStats)) {
-            return null;
-        }
 
-        Map<String, Set<String>> needRunPartitions = table.findReAnalyzeNeededPartitions();
+        Map<String, Set<String>> needRunPartitions = null;
+        String colNames = jobInfo.colName;
+        if (table.needReAnalyzeTable(tblStats)) {
+            needRunPartitions = table.findReAnalyzeNeededPartitions();
+        } else if (table instanceof OlapTable && tblStats.newPartitionLoaded.get()) {
+            OlapTable olapTable = (OlapTable) table;
+            needRunPartitions = new HashMap<>();
+            Set<String> partitionColumnNames = olapTable.getPartitionInfo().getPartitionColumns().stream()
+                    .map(Column::getName).collect(Collectors.toSet());
+            colNames = partitionColumnNames.stream().collect(Collectors.joining(","));
+            Set<String> partitionNames = olapTable.getAllPartitions().stream()
+                    .map(Partition::getName).collect(Collectors.toSet());
+            for (String column : partitionColumnNames) {
+                needRunPartitions.put(column, partitionNames);
+            }
+        }
 
-        if (needRunPartitions.isEmpty()) {
+        if (needRunPartitions == null || needRunPartitions.isEmpty()) {
             return null;
         }
-
-        return new AnalysisInfoBuilder(jobInfo).setColToPartitions(needRunPartitions).build();
+        return new AnalysisInfoBuilder(jobInfo).setColName(colNames).setColToPartitions(needRunPartitions).build();
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index eb6672ffe1856f..00878adcc44427 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -19,6 +19,7 @@
 
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PartitionInfo;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
@@ -37,6 +38,7 @@
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Collectors;
 
@@ -67,6 +69,9 @@ public class TableStatsMeta implements Writable {
     @SerializedName("trigger")
     public JobType jobType;
 
+    @SerializedName("newPartitionLoaded")
+    public AtomicBoolean newPartitionLoaded = new AtomicBoolean(false);
+
     @VisibleForTesting
     public TableStatsMeta() {
         tblId = 0;
@@ -154,6 +159,15 @@ public void update(AnalysisInfo analyzedJob, TableIf tableIf) {
                             .filter(c -> !StatisticsUtil.isUnsupportedType(c.getType()))
                             .map(Column::getName).collect(Collectors.toSet()))) {
                 updatedRows.set(0);
+                newPartitionLoaded.set(false);
+            }
+            if (tableIf instanceof OlapTable) {
+                PartitionInfo partitionInfo = ((OlapTable) tableIf).getPartitionInfo();
+                if (partitionInfo != null && analyzedJob.colToPartitions.keySet()
+                        .containsAll(partitionInfo.getPartitionColumns().stream()
+                            .map(Column::getName).collect(Collectors.toSet()))) {
+                    newPartitionLoaded.set(false);
+                }
             }
         }
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 5bb954289cc514..f0217e71d22fac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -1830,6 +1830,7 @@ private void updateCatalogAfterCommitted(TransactionState transactionState, Data
 
     private boolean updateCatalogAfterVisible(TransactionState transactionState, Database db) {
         Set<Long> errorReplicaIds = transactionState.getErrorReplicas();
+        AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
         for (TableCommitInfo tableCommitInfo : transactionState.getIdToTableCommitInfos().values()) {
             long tableId = tableCommitInfo.getTableId();
             OlapTable table = (OlapTable) db.getTableNullable(tableId);
@@ -1889,6 +1890,10 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat
                 } // end for indices
                 long version = partitionCommitInfo.getVersion();
                 long versionTime = partitionCommitInfo.getVersionTime();
+                if (partition.getVisibleVersion() == Partition.PARTITION_INIT_VERSION
+                        && version > Partition.PARTITION_INIT_VERSION) {
+                    analysisManager.setNewPartitionLoaded(tableId);
+                }
                 partition.updateVisibleVersionAndTime(version, versionTime);
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("transaction state {} set partition {}'s version to [{}]",
@@ -1896,7 +1901,6 @@ private boolean updateCatalogAfterVisible(TransactionState transactionState, Dat
                 }
             }
         }
-        AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
         Map<Long, Long> tableIdToTotalNumDeltaRows = transactionState.getTableIdToTotalNumDeltaRows();
         Map<Long, Long> tableIdToNumDeltaRows = Maps.newHashMap();
         tableIdToTotalNumDeltaRows
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 87342202fb2a27..f6def60908d4e6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -298,8 +298,13 @@ public TableStatsMeta findTableStatsStatus(long tblId) {
         };
         // A very huge table has been updated recently, so we should skip it this time
         stats.updatedTime = System.currentTimeMillis() - 1000;
+        stats.newPartitionLoaded = new AtomicBoolean();
+        stats.newPartitionLoaded.set(true);
         StatisticsAutoCollector autoCollector = new StatisticsAutoCollector();
+        // Test new partition loaded data for the first time. Not skip.
         Assertions.assertFalse(autoCollector.skip(olapTable));
+        stats.newPartitionLoaded.set(false);
+        // Assertions.assertTrue(autoCollector.skip(olapTable));
         // The update of this huge table is long time ago, so we shouldn't skip it this time
         stats.updatedTime = System.currentTimeMillis()
                 - StatisticsUtil.getHugeTableAutoAnalyzeIntervalInMillis() - 10000;
diff --git a/regression-test/suites/statistics/analyze_stats.groovy b/regression-test/suites/statistics/analyze_stats.groovy
index 64967280ce9274..7bacd4c833f161 100644
--- a/regression-test/suites/statistics/analyze_stats.groovy
+++ b/regression-test/suites/statistics/analyze_stats.groovy
@@ -2576,6 +2576,42 @@ PARTITION `p599` VALUES IN (599)
    assertEquals("\'name1\'", result[0][6])
    assertEquals("\'name3\'", result[0][7])
 
+   // Test partititon load data for the first time.
+   sql """
+     CREATE TABLE `partition_test` (
+      `id` INT NOT NULL,
+      `name` VARCHAR(25) NOT NULL,
+      `comment` VARCHAR(152) NULL
+      ) ENGINE=OLAP
+      DUPLICATE KEY(`id`)
+      COMMENT 'OLAP'
+      PARTITION BY RANGE(`id`)
+      (PARTITION p1 VALUES [("0"), ("100")),
+       PARTITION p2 VALUES [("100"), ("200")),
+       PARTITION p3 VALUES [("200"), ("300")))
+      DISTRIBUTED BY HASH(`id`) BUCKETS 1
+      PROPERTIES (
+       "replication_num" = "1");
+     """
+
+   sql """analyze table partition_test with sync"""
+   sql """insert into partition_test values (1, '1', '1')"""
+   def partition_result = sql """show table stats partition_test"""
+   assertEquals(partition_result[0][6], "true")
+   assertEquals(partition_result[0][0], "1")
+   sql """analyze table partition_test with sync"""
+   partition_result = sql """show table stats partition_test"""
+   assertEquals(partition_result[0][6], "false")
+   sql """insert into partition_test values (101, '1', '1')"""
+   partition_result = sql """show table stats partition_test"""
+   assertEquals(partition_result[0][6], "true")
+   sql """analyze table partition_test(id) with sync"""
+   partition_result = sql """show table stats partition_test"""
+   assertEquals(partition_result[0][6], "false")
+   sql """insert into partition_test values (102, '1', '1')"""
+   partition_result = sql """show table stats partition_test"""
+   assertEquals(partition_result[0][6], "false")
+
    // Test trigger type.
    sql """DROP DATABASE IF EXISTS trigger"""
    sql """CREATE DATABASE IF NOT EXISTS trigger"""

From 411f819306a15151fd9ebef8d5edaf6177f72184 Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@hotmail.com>
Date: Fri, 29 Dec 2023 14:39:33 +0800
Subject: [PATCH 161/299] [Opt](join) opt the join copy column from build side
 (#29007)

---
 be/src/vec/columns/column_decimal.h  | 13 +++++++++----
 be/src/vec/columns/column_vector.cpp | 14 +++++++++-----
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h
index 9b1ad69bbad96b..3681f9190ae3ff 100644
--- a/be/src/vec/columns/column_decimal.h
+++ b/be/src/vec/columns/column_decimal.h
@@ -124,11 +124,16 @@ class ColumnDecimal final : public COWHelper<ColumnVectorHelper, ColumnDecimal<T
         auto origin_size = size();
         auto new_size = indices_end - indices_begin;
         data.resize(origin_size + new_size);
-        const T* __restrict src_data = reinterpret_cast<const T*>(src.get_raw_data().data);
 
-        for (uint32_t i = 0; i < new_size; ++i) {
-            data[origin_size + i] = src_data[indices_begin[i]];
-        }
+        auto copy = [](const T* __restrict src, T* __restrict dest,
+                       const uint32_t* __restrict begin, const uint32_t* __restrict end) {
+            for (auto it = begin; it != end; ++it) {
+                *dest = src[*it];
+                ++dest;
+            }
+        };
+        copy(reinterpret_cast<const T*>(src.get_raw_data().data), data.data() + origin_size,
+             indices_begin, indices_end);
     }
 
     void insert_many_fix_len_data(const char* data_ptr, size_t num) override;
diff --git a/be/src/vec/columns/column_vector.cpp b/be/src/vec/columns/column_vector.cpp
index ca8db58fc98f2d..7329f898044007 100644
--- a/be/src/vec/columns/column_vector.cpp
+++ b/be/src/vec/columns/column_vector.cpp
@@ -372,11 +372,15 @@ void ColumnVector<T>::insert_indices_from(const IColumn& src, const uint32_t* in
     auto new_size = indices_end - indices_begin;
     data.resize(origin_size + new_size);
 
-    const T* __restrict src_data = reinterpret_cast<const T*>(src.get_raw_data().data);
-
-    for (uint32_t i = 0; i < new_size; ++i) {
-        data[origin_size + i] = src_data[indices_begin[i]];
-    }
+    auto copy = [](const T* __restrict src, T* __restrict dest, const uint32_t* __restrict begin,
+                   const uint32_t* __restrict end) {
+        for (auto it = begin; it != end; ++it) {
+            *dest = src[*it];
+            ++dest;
+        }
+    };
+    copy(reinterpret_cast<const T*>(src.get_raw_data().data), data.data() + origin_size,
+         indices_begin, indices_end);
 }
 
 template <typename T>

From 660f12c7175cbe6140b4f9173f40472acc008a13 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Fri, 29 Dec 2023 14:50:46 +0800
Subject: [PATCH 162/299] [dependency](fe)Upgrade dependency library (#29049)

---
 .../metastore/AWSGlueDecoratorBase.java       | 77 +++++++++++++++++++
 fe/pom.xml                                    | 59 +++++++++++---
 2 files changed, 127 insertions(+), 9 deletions(-)

diff --git a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueDecoratorBase.java b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueDecoratorBase.java
index 027dec347a9d6f..21be9a75bf125b 100644
--- a/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueDecoratorBase.java
+++ b/fe/fe-core/src/main/java/com/amazonaws/glue/catalog/metastore/AWSGlueDecoratorBase.java
@@ -48,6 +48,8 @@
 import com.amazonaws.services.glue.model.BatchGetJobsResult;
 import com.amazonaws.services.glue.model.BatchGetPartitionRequest;
 import com.amazonaws.services.glue.model.BatchGetPartitionResult;
+import com.amazonaws.services.glue.model.BatchGetTableOptimizerRequest;
+import com.amazonaws.services.glue.model.BatchGetTableOptimizerResult;
 import com.amazonaws.services.glue.model.BatchGetTriggersRequest;
 import com.amazonaws.services.glue.model.BatchGetTriggersResult;
 import com.amazonaws.services.glue.model.BatchGetWorkflowsRequest;
@@ -100,6 +102,8 @@
 import com.amazonaws.services.glue.model.CreateSecurityConfigurationResult;
 import com.amazonaws.services.glue.model.CreateSessionRequest;
 import com.amazonaws.services.glue.model.CreateSessionResult;
+import com.amazonaws.services.glue.model.CreateTableOptimizerRequest;
+import com.amazonaws.services.glue.model.CreateTableOptimizerResult;
 import com.amazonaws.services.glue.model.CreateTableRequest;
 import com.amazonaws.services.glue.model.CreateTableResult;
 import com.amazonaws.services.glue.model.CreateTriggerRequest;
@@ -148,6 +152,8 @@
 import com.amazonaws.services.glue.model.DeleteSecurityConfigurationResult;
 import com.amazonaws.services.glue.model.DeleteSessionRequest;
 import com.amazonaws.services.glue.model.DeleteSessionResult;
+import com.amazonaws.services.glue.model.DeleteTableOptimizerRequest;
+import com.amazonaws.services.glue.model.DeleteTableOptimizerResult;
 import com.amazonaws.services.glue.model.DeleteTableRequest;
 import com.amazonaws.services.glue.model.DeleteTableResult;
 import com.amazonaws.services.glue.model.DeleteTableVersionRequest;
@@ -170,6 +176,10 @@
 import com.amazonaws.services.glue.model.GetClassifierResult;
 import com.amazonaws.services.glue.model.GetClassifiersRequest;
 import com.amazonaws.services.glue.model.GetClassifiersResult;
+import com.amazonaws.services.glue.model.GetColumnStatisticsTaskRunRequest;
+import com.amazonaws.services.glue.model.GetColumnStatisticsTaskRunResult;
+import com.amazonaws.services.glue.model.GetColumnStatisticsTaskRunsRequest;
+import com.amazonaws.services.glue.model.GetColumnStatisticsTaskRunsResult;
 import com.amazonaws.services.glue.model.GetConnectionRequest;
 import com.amazonaws.services.glue.model.GetColumnStatisticsForPartitionResult;
 import com.amazonaws.services.glue.model.GetColumnStatisticsForPartitionRequest;
@@ -256,6 +266,8 @@
 import com.amazonaws.services.glue.model.GetSessionResult;
 import com.amazonaws.services.glue.model.GetStatementRequest;
 import com.amazonaws.services.glue.model.GetStatementResult;
+import com.amazonaws.services.glue.model.GetTableOptimizerRequest;
+import com.amazonaws.services.glue.model.GetTableOptimizerResult;
 import com.amazonaws.services.glue.model.GetTableRequest;
 import com.amazonaws.services.glue.model.GetTableResult;
 import com.amazonaws.services.glue.model.GetTableVersionRequest;
@@ -292,6 +304,8 @@
 import com.amazonaws.services.glue.model.ImportCatalogToGlueResult;
 import com.amazonaws.services.glue.model.ListBlueprintsRequest;
 import com.amazonaws.services.glue.model.ListBlueprintsResult;
+import com.amazonaws.services.glue.model.ListColumnStatisticsTaskRunsRequest;
+import com.amazonaws.services.glue.model.ListColumnStatisticsTaskRunsResult;
 import com.amazonaws.services.glue.model.ListCrawlersRequest;
 import com.amazonaws.services.glue.model.ListCrawlersResult;
 import com.amazonaws.services.glue.model.ListCrawlsRequest;
@@ -322,6 +336,8 @@
 import com.amazonaws.services.glue.model.ListSessionsResult;
 import com.amazonaws.services.glue.model.ListStatementsRequest;
 import com.amazonaws.services.glue.model.ListStatementsResult;
+import com.amazonaws.services.glue.model.ListTableOptimizerRunsRequest;
+import com.amazonaws.services.glue.model.ListTableOptimizerRunsResult;
 import com.amazonaws.services.glue.model.ListTriggersRequest;
 import com.amazonaws.services.glue.model.ListTriggersResult;
 import com.amazonaws.services.glue.model.ListWorkflowsRequest;
@@ -350,6 +366,8 @@
 import com.amazonaws.services.glue.model.SearchTablesResult;
 import com.amazonaws.services.glue.model.StartBlueprintRunRequest;
 import com.amazonaws.services.glue.model.StartBlueprintRunResult;
+import com.amazonaws.services.glue.model.StartColumnStatisticsTaskRunRequest;
+import com.amazonaws.services.glue.model.StartColumnStatisticsTaskRunResult;
 import com.amazonaws.services.glue.model.StartCrawlerRequest;
 import com.amazonaws.services.glue.model.StartCrawlerResult;
 import com.amazonaws.services.glue.model.StartCrawlerScheduleRequest;
@@ -372,6 +390,8 @@
 import com.amazonaws.services.glue.model.StartTriggerResult;
 import com.amazonaws.services.glue.model.StartWorkflowRunRequest;
 import com.amazonaws.services.glue.model.StartWorkflowRunResult;
+import com.amazonaws.services.glue.model.StopColumnStatisticsTaskRunRequest;
+import com.amazonaws.services.glue.model.StopColumnStatisticsTaskRunResult;
 import com.amazonaws.services.glue.model.StopCrawlerRequest;
 import com.amazonaws.services.glue.model.StopCrawlerResult;
 import com.amazonaws.services.glue.model.StopCrawlerScheduleRequest;
@@ -420,6 +440,8 @@
 import com.amazonaws.services.glue.model.UpdateSchemaResult;
 import com.amazonaws.services.glue.model.UpdateSourceControlFromJobRequest;
 import com.amazonaws.services.glue.model.UpdateSourceControlFromJobResult;
+import com.amazonaws.services.glue.model.UpdateTableOptimizerRequest;
+import com.amazonaws.services.glue.model.UpdateTableOptimizerResult;
 import com.amazonaws.services.glue.model.UpdateTableRequest;
 import com.amazonaws.services.glue.model.UpdateTableResult;
 import com.amazonaws.services.glue.model.UpdateTriggerRequest;
@@ -495,6 +517,11 @@ public BatchGetPartitionResult batchGetPartition(BatchGetPartitionRequest batchG
         return decoratedAwsGlue.batchGetPartition(batchGetPartitionRequest);
     }
 
+    @Override
+    public BatchGetTableOptimizerResult batchGetTableOptimizer(BatchGetTableOptimizerRequest batchGetTableOptimizerRequest) {
+        return null;
+    }
+
     @Override
     public BatchGetTriggersResult batchGetTriggers(BatchGetTriggersRequest batchGetTriggersRequest) {
         return decoratedAwsGlue.batchGetTriggers(batchGetTriggersRequest);
@@ -615,6 +642,11 @@ public CreateTableResult createTable(CreateTableRequest createTableRequest) {
         return decoratedAwsGlue.createTable(createTableRequest);
     }
 
+    @Override
+    public CreateTableOptimizerResult createTableOptimizer(CreateTableOptimizerRequest createTableOptimizerRequest) {
+        return null;
+    }
+
     @Override
     public CreateTriggerResult createTrigger(CreateTriggerRequest createTriggerRequest) {
         return decoratedAwsGlue.createTrigger(createTriggerRequest);
@@ -720,6 +752,11 @@ public DeleteTableResult deleteTable(DeleteTableRequest deleteTableRequest) {
         return decoratedAwsGlue.deleteTable(deleteTableRequest);
     }
 
+    @Override
+    public DeleteTableOptimizerResult deleteTableOptimizer(DeleteTableOptimizerRequest deleteTableOptimizerRequest) {
+        return null;
+    }
+
     @Override
     public DeleteTableVersionResult deleteTableVersion(DeleteTableVersionRequest deleteTableVersionRequest) {
         return decoratedAwsGlue.deleteTableVersion(deleteTableVersionRequest);
@@ -955,6 +992,11 @@ public GetTableResult getTable(GetTableRequest getTableRequest) {
         return decoratedAwsGlue.getTable(getTableRequest);
     }
 
+    @Override
+    public GetTableOptimizerResult getTableOptimizer(GetTableOptimizerRequest getTableOptimizerRequest) {
+        return null;
+    }
+
     @Override
     public GetTableVersionResult getTableVersion(GetTableVersionRequest getTableVersionRequest) {
         return decoratedAwsGlue.getTableVersion(getTableVersionRequest);
@@ -1040,6 +1082,11 @@ public ListBlueprintsResult listBlueprints(ListBlueprintsRequest listBlueprintsR
         return decoratedAwsGlue.listBlueprints(listBlueprintsRequest);
     }
 
+    @Override
+    public ListColumnStatisticsTaskRunsResult listColumnStatisticsTaskRuns(ListColumnStatisticsTaskRunsRequest listColumnStatisticsTaskRunsRequest) {
+        return null;
+    }
+
     @Override
     public ListCrawlersResult listCrawlers(ListCrawlersRequest listCrawlersRequest) {
         return decoratedAwsGlue.listCrawlers(listCrawlersRequest);
@@ -1095,6 +1142,11 @@ public ListStatementsResult listStatements(ListStatementsRequest listStatementsR
         return decoratedAwsGlue.listStatements(listStatementsRequest);
     }
 
+    @Override
+    public ListTableOptimizerRunsResult listTableOptimizerRuns(ListTableOptimizerRunsRequest listTableOptimizerRunsRequest) {
+        return null;
+    }
+
     @Override
     public ListTriggersResult listTriggers(ListTriggersRequest listTriggersRequest) {
         return decoratedAwsGlue.listTriggers(listTriggersRequest);
@@ -1155,6 +1207,11 @@ public StartBlueprintRunResult startBlueprintRun(StartBlueprintRunRequest startB
         return decoratedAwsGlue.startBlueprintRun(startBlueprintRunRequest);
     }
 
+    @Override
+    public StartColumnStatisticsTaskRunResult startColumnStatisticsTaskRun(StartColumnStatisticsTaskRunRequest startColumnStatisticsTaskRunRequest) {
+        return null;
+    }
+
     @Override
     public StartCrawlerResult startCrawler(StartCrawlerRequest startCrawlerRequest) {
         return decoratedAwsGlue.startCrawler(startCrawlerRequest);
@@ -1200,6 +1257,11 @@ public StartWorkflowRunResult startWorkflowRun(StartWorkflowRunRequest startWork
         return decoratedAwsGlue.startWorkflowRun(startWorkflowRunRequest);
     }
 
+    @Override
+    public StopColumnStatisticsTaskRunResult stopColumnStatisticsTaskRun(StopColumnStatisticsTaskRunRequest stopColumnStatisticsTaskRunRequest) {
+        return null;
+    }
+
     @Override
     public StopCrawlerResult stopCrawler(StopCrawlerRequest stopCrawlerRequest) {
         return decoratedAwsGlue.stopCrawler(stopCrawlerRequest);
@@ -1300,6 +1362,11 @@ public UpdateTableResult updateTable(UpdateTableRequest updateTableRequest) {
         return decoratedAwsGlue.updateTable(updateTableRequest);
     }
 
+    @Override
+    public UpdateTableOptimizerResult updateTableOptimizer(UpdateTableOptimizerRequest updateTableOptimizerRequest) {
+        return null;
+    }
+
     @Override
     public UpdateTriggerResult updateTrigger(UpdateTriggerRequest updateTriggerRequest) {
         return decoratedAwsGlue.updateTrigger(updateTriggerRequest);
@@ -1356,6 +1423,16 @@ public GetColumnStatisticsForTableResult getColumnStatisticsForTable(GetColumnSt
         return decoratedAwsGlue.getColumnStatisticsForTable(getColumnStatisticsForTableRequest);
     }
 
+    @Override
+    public GetColumnStatisticsTaskRunResult getColumnStatisticsTaskRun(GetColumnStatisticsTaskRunRequest getColumnStatisticsTaskRunRequest) {
+        return null;
+    }
+
+    @Override
+    public GetColumnStatisticsTaskRunsResult getColumnStatisticsTaskRuns(GetColumnStatisticsTaskRunsRequest getColumnStatisticsTaskRunsRequest) {
+        return null;
+    }
+
     @Override
     public GetColumnStatisticsForPartitionResult getColumnStatisticsForPartition(GetColumnStatisticsForPartitionRequest getColumnStatisticsForPartitionRequest) {
         return decoratedAwsGlue.getColumnStatisticsForPartition(getColumnStatisticsForPartitionRequest);
diff --git a/fe/pom.xml b/fe/pom.xml
index c8f862dd540811..ef0747931cfb24 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -260,14 +260,14 @@ under the License.
         <slf4j.version>2.0.6</slf4j.version>
         <metrics-core.version>4.0.2</metrics-core.version>
         <!--Netty 4.1.94 is not compatible with arrow flight.-->
-        <netty-all.version>4.1.96.Final</netty-all.version>
+        <netty-all.version>4.1.104.Final</netty-all.version>
         <!--The dependence of transitive dependence cannot be ruled out, only Saving the nation through twisted ways.-->
         <netty-3-test.version>3.10.6.Final</netty-3-test.version>
         <objenesis.version>2.1</objenesis.version>
         <!-- NOTE: Using grpc-java whose version is newer than 1.34.0 will break the build on CentOS 6 due to the obsolete GLIBC -->
         <grpc-java.version>1.34.0</grpc-java.version>
-        <grpc.version>1.58.0</grpc.version>
-        <check.freamework.version>3.38.0</check.freamework.version>
+        <grpc.version>1.60.1</grpc.version>
+        <check.freamework.version>3.42.0</check.freamework.version>
         <protobuf.version>3.24.3</protobuf.version>
         <!-- we use protoc-jar-maven-plugin to generate protobuf generated code -->
         <!-- see https://repo.maven.apache.org/maven2/com/google/protobuf/protoc/ to get correct version -->
@@ -275,7 +275,7 @@ under the License.
         <protoc.artifact>com.google.protobuf:protoc:${protoc.artifact.version}</protoc.artifact>
         <grpc.java.artifact>io.grpc:protoc-gen-grpc-java:${grpc-java.version}</grpc.java.artifact>
         <protoparser.version>3.1.5</protoparser.version>
-        <snappy-java.version>1.1.10.1</snappy-java.version>
+        <snappy-java.version>1.1.10.5</snappy-java.version>
         <automaton.version>1.11-8</automaton.version>
         <generex.version>1.0.1</generex.version>
         <fabric8.kubernetes.version>6.7.2</fabric8.kubernetes.version>
@@ -304,8 +304,8 @@ under the License.
          you can find avro version info in iceberg mvn repository -->
         <iceberg.version>1.1.0</iceberg.version>
         <maxcompute.version>0.45.2-public</maxcompute.version>
-        <avro.version>1.11.2</avro.version>
-        <arrow.version>14.0.1</arrow.version>
+        <avro.version>1.11.3</avro.version>
+        <arrow.version>14.0.2</arrow.version>
         <!-- hudi -->
         <hudi.version>0.13.1</hudi.version>
         <presto.hadoop.version>2.7.4-11</presto.hadoop.version>
@@ -321,7 +321,7 @@ under the License.
         <hamcrest.version>2.1</hamcrest.version>
         <httpclient.version>4.5.13</httpclient.version>
         <httpcore.version>4.4.15</httpcore.version>
-        <aws-java-sdk.version>1.12.519</aws-java-sdk.version>
+        <aws-java-sdk.version>1.12.625</aws-java-sdk.version>
         <mariadb-java-client.version>3.0.4</mariadb-java-client.version>
         <dlf-metastore-client-hive.version>0.2.14</dlf-metastore-client-hive.version>
         <hadoop.version>3.3.6</hadoop.version>
@@ -330,13 +330,14 @@ under the License.
         <spring.version>2.7.13</spring.version>
         <orc.version>1.8.4</orc.version>
         <ojdbc8.version>12.2.0.1</ojdbc8.version>
-        <zookeeper.version>3.4.14</zookeeper.version>
+        <zookeeper.version>3.9.1</zookeeper.version>
         <velocity-engine-core.version>2.3</velocity-engine-core.version>
         <ranger-plugins-common.version>2.4.0</ranger-plugins-common.version>
         <bcprov-jdk15on.version>1.70</bcprov-jdk15on.version>
         <woodstox.version>6.5.1</woodstox.version>
         <kerby.version>2.0.3</kerby.version>
         <jettison.version>1.5.4</jettison.version>
+        <jetty.version>9.4.53.v20231009</jetty.version>
         <immutables.version>2.9.3</immutables.version>
         <vesoft.client.version>3.0.0</vesoft.client.version>
         <!--todo waiting release-->
@@ -553,8 +554,38 @@ under the License.
                 <groupId>org.apache.zookeeper</groupId>
                 <artifactId>zookeeper</artifactId>
                 <version>${zookeeper.version}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>ch.qos.logback</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty.websocket</groupId>
+                <artifactId>websocket-server</artifactId>
+                <version>${jetty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty.websocket</groupId>
+                <artifactId>websocket-client</artifactId>
+                <version>${jetty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty.websocket</groupId>
+                <artifactId>websocket-servlet</artifactId>
+                <version>${jetty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty.websocket</groupId>
+                <artifactId>websocket-common</artifactId>
+                <version>${jetty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty</groupId>
+                <artifactId>jetty-server</artifactId>
+                <version>${jetty.version}</version>
             </dependency>
-
             <dependency>
                 <groupId>${project.groupId}</groupId>
                 <artifactId>fe-common</artifactId>
@@ -806,6 +837,16 @@ under the License.
                 <artifactId>netty-codec</artifactId>
                 <version>${netty-all.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-transport-udt</artifactId>
+                <version>${netty-all.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.netty</groupId>
+                <artifactId>netty-transport-rxtx</artifactId>
+                <version>${netty-all.version}</version>
+            </dependency>
             <dependency>
                 <groupId>io.netty</groupId>
                 <artifactId>netty-codec-memcache</artifactId>

From 24cdc00b60a999a6f8fbeddd347bccf8ae24d41f Mon Sep 17 00:00:00 2001
From: yujun <yu.jun.reach@gmail.com>
Date: Fri, 29 Dec 2023 15:05:28 +0800
Subject: [PATCH 163/299] [improvement](doris compose) avoid docker network
 conflict (#28957)

---
 docker/runtime/doris-compose/cluster.py       | 52 ++++++++++++-------
 docker/runtime/doris-compose/command.py       |  8 ++-
 docker/runtime/doris-compose/requirements.txt |  1 +
 .../runtime/doris-compose/resource/common.sh  |  3 +-
 docker/runtime/doris-compose/utils.py         |  7 +++
 .../regression/suite/SuiteCluster.groovy      | 22 +++++---
 6 files changed, 63 insertions(+), 30 deletions(-)

diff --git a/docker/runtime/doris-compose/cluster.py b/docker/runtime/doris-compose/cluster.py
index 0f6afb906ab311..d78c035f077ae6 100644
--- a/docker/runtime/doris-compose/cluster.py
+++ b/docker/runtime/doris-compose/cluster.py
@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import filelock
 import json
 import jsonpickle
 import os
@@ -22,10 +23,8 @@
 import utils
 
 DOCKER_DORIS_PATH = "/opt/apache-doris"
-LOCAL_DORIS_PATH = os.getenv("LOCAL_DORIS_PATH")
-if not LOCAL_DORIS_PATH:
-    LOCAL_DORIS_PATH = "/tmp/doris"
-
+LOCAL_DORIS_PATH = os.getenv("LOCAL_DORIS_PATH", "/tmp/doris")
+DORIS_SUBNET_START = int(os.getenv("DORIS_SUBNET_START", 128))
 LOCAL_RESOURCE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                    "resource")
 DOCKER_RESOURCE_PATH = os.path.join(DOCKER_DORIS_PATH, "resource")
@@ -59,17 +58,30 @@ def get_status_path(cluster_name):
     return os.path.join(get_cluster_path(cluster_name), "status")
 
 
+def get_all_cluster_names():
+    if not os.path.exists(LOCAL_DORIS_PATH):
+        return []
+    else:
+        return [
+            subdir for subdir in os.listdir(LOCAL_DORIS_PATH)
+            if os.path.isdir(os.path.join(LOCAL_DORIS_PATH, subdir))
+        ]
+
+
+def lock_network():
+    return filelock.FileLock(os.path.join(LOCAL_DORIS_PATH, "lock"))
+
+
 def gen_subnet_prefix16():
     used_subnet = utils.get_docker_subnets_prefix16()
-    if os.path.exists(LOCAL_DORIS_PATH):
-        for cluster_name in os.listdir(LOCAL_DORIS_PATH):
-            try:
-                cluster = Cluster.load(cluster_name)
-                used_subnet[cluster.subnet] = True
-            except:
-                pass
-
-    for i in range(128, 192):
+    for cluster_name in get_all_cluster_names():
+        try:
+            cluster = Cluster.load(cluster_name)
+            used_subnet[cluster.subnet] = True
+        except:
+            pass
+
+    for i in range(DORIS_SUBNET_START, 192):
         for j in range(256):
             subnet = "{}.{}".format(i, j)
             if not used_subnet.get(subnet, False):
@@ -392,12 +404,14 @@ def __init__(self, name, subnet, image, fe_config, be_config, be_disks,
 
     @staticmethod
     def new(name, image, fe_config, be_config, be_disks, coverage_dir):
-        subnet = gen_subnet_prefix16()
-        cluster = Cluster(name, subnet, image, fe_config, be_config, be_disks,
-                          coverage_dir)
-        os.makedirs(cluster.get_path(), exist_ok=True)
-        os.makedirs(get_status_path(name), exist_ok=True)
-        return cluster
+        with filelock.FileLock(os.path.join(LOCAL_DORIS_PATH, "lock")):
+            subnet = gen_subnet_prefix16()
+            cluster = Cluster(name, subnet, image, fe_config, be_config,
+                              be_disks, coverage_dir)
+            os.makedirs(cluster.get_path(), exist_ok=True)
+            os.makedirs(get_status_path(name), exist_ok=True)
+            cluster._save_meta()
+            return cluster
 
     @staticmethod
     def load(name):
diff --git a/docker/runtime/doris-compose/command.py b/docker/runtime/doris-compose/command.py
index cfc3f137e166c3..4d1d6e7edb3117 100644
--- a/docker/runtime/doris-compose/command.py
+++ b/docker/runtime/doris-compose/command.py
@@ -359,6 +359,10 @@ def run(self, args):
             utils.exec_docker_compose_command(cluster.get_compose_file(),
                                               "down",
                                               ["-v", "--remove-orphans"])
+            try:
+                utils.remove_docker_network(cluster.name)
+            except Exception as e:
+                LOG.warn("prune network has exception: " + str(e))
             if args.clean:
                 utils.enable_dir_with_rw_perm(cluster.get_path())
                 shutil.rmtree(cluster.get_path())
@@ -544,8 +548,8 @@ def parse_cluster_compose_file(cluster_name):
         search_names = []
         if args.NAME:
             search_names = args.NAME
-        elif os.path.exists(CLUSTER.LOCAL_DORIS_PATH):
-            search_names = os.listdir(CLUSTER.LOCAL_DORIS_PATH)
+        else:
+            search_names = CLUSTER.get_all_cluster_names()
 
         for cluster_name in search_names:
             status, services = parse_cluster_compose_file(cluster_name)
diff --git a/docker/runtime/doris-compose/requirements.txt b/docker/runtime/doris-compose/requirements.txt
index 7d03121e899331..ac177eddf82ad0 100644
--- a/docker/runtime/doris-compose/requirements.txt
+++ b/docker/runtime/doris-compose/requirements.txt
@@ -17,6 +17,7 @@
 
 docker
 docker-compose
+filelock
 jsonpickle
 prettytable
 pymysql
diff --git a/docker/runtime/doris-compose/resource/common.sh b/docker/runtime/doris-compose/resource/common.sh
index 8971ac251530a8..5342eb0e54b27c 100644
--- a/docker/runtime/doris-compose/resource/common.sh
+++ b/docker/runtime/doris-compose/resource/common.sh
@@ -45,7 +45,8 @@ wait_pid() {
 
     health_log "wait process $pid"
     while true; do
-        if [ ! ps -p $pid ] >/dev/null; then
+        ps -p $pid >/dev/null
+        if [ $? -ne 0 ]; then
             break
         fi
         sleep 1s
diff --git a/docker/runtime/doris-compose/utils.py b/docker/runtime/doris-compose/utils.py
index c9c0c6593b5595..8b4b39619bce66 100644
--- a/docker/runtime/doris-compose/utils.py
+++ b/docker/runtime/doris-compose/utils.py
@@ -168,6 +168,13 @@ def get_doris_running_containers(cluster_name):
     }
 
 
+def remove_docker_network(cluster_name):
+    client = docker.client.from_env()
+    for network in client.networks.list(
+            names=[cluster_name + "_" + with_doris_prefix(cluster_name)]):
+        network.remove()
+
+
 def is_dir_empty(dir):
     return False if os.listdir(dir) else True
 
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
index 039913b7248257..47b642af4853ce 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/SuiteCluster.groovy
@@ -34,7 +34,7 @@ class ClusterOptions {
 
     int feNum = 1
     int beNum = 3
-    List<String> feConfigs = []
+    List<String> feConfigs = ['heartbeat_interval_second=5']
     List<String> beConfigs = []
 
     // each be disks, a disks format is: disk_type=disk_num[,disk_capacity]
@@ -85,6 +85,10 @@ class ServerNode {
         node.alive = fields.get(header.indexOf('alive')) == 'true'
     }
 
+    static long toLongOrDefault(Object val, long defValue) {
+        return val == '' ? defValue : (long) val
+    }
+
     def getHttpAddress() {
         return [host, httpPort]
     }
@@ -137,8 +141,8 @@ class Backend extends ServerNode {
     static Backend fromCompose(ListHeader header, int index, List<Object> fields) {
         Backend be = new Backend()
         ServerNode.fromCompose(be, header, index, fields)
-        be.backendId = (long) fields.get(header.indexOf('backend_id'))
-        be.tabletNum = (int) fields.get(header.indexOf('tablet_num'))
+        be.backendId = toLongOrDefault(fields.get(header.indexOf('backend_id')), -1L)
+        be.tabletNum = (int) toLongOrDefault(fields.get(header.indexOf('tablet_num')), 0L)
         return be
     }
 
@@ -438,6 +442,7 @@ class SuiteCluster {
     }
 
     private void waitHbChanged() {
+        // heart beat interval is 5s
         Thread.sleep(7000)
     }
 
@@ -467,14 +472,15 @@ class SuiteCluster {
         } else {
             proc.waitFor()
         }
-        def out = outBuf.toString()
-        def err = errBuf.toString()
-        if (proc.exitValue()) {
+        if (proc.exitValue() != 0) {
             throw new Exception(String.format('Exit value: %s != 0, stdout: %s, stderr: %s',
-                                              proc.exitValue(), out, err))
+                                              proc.exitValue(), outBuf.toString(), errBuf.toString()))
         }
         def parser = new JsonSlurper()
-        def object = (Map<String, Object>) parser.parseText(out)
+        if (outBuf.toString().size() == 0) {
+            throw new Exception(String.format('doris compose output is empty, err: %s', errBuf.toString()))
+        }
+        def object = (Map<String, Object>) parser.parseText(outBuf.toString())
         if (object.get('code') != 0) {
             throw new Exception(String.format('Code: %s != 0, err: %s', object.get('code'), object.get('err')))
         }

From a7c4abde9fad0617585f2a43a6f7f0756a48b109 Mon Sep 17 00:00:00 2001
From: yujun <yu.jun.reach@gmail.com>
Date: Fri, 29 Dec 2023 15:07:11 +0800
Subject: [PATCH 164/299] [fix](regression test) fix
 test_schema_change_concurrent_with_txn (#28994)

---
 ...t_schema_change_concurrent_with_txn.groovy | 46 +++++++++----------
 1 file changed, 22 insertions(+), 24 deletions(-)

diff --git a/regression-test/suites/schema_change/test_schema_change_concurrent_with_txn.groovy b/regression-test/suites/schema_change/test_schema_change_concurrent_with_txn.groovy
index 4e2f717ed4658d..d250a000c0123f 100644
--- a/regression-test/suites/schema_change/test_schema_change_concurrent_with_txn.groovy
+++ b/regression-test/suites/schema_change/test_schema_change_concurrent_with_txn.groovy
@@ -28,7 +28,7 @@ suite('test_schema_change_concurrent_with_txn') {
         def result = sql 'SELECT DATABASE()'
         def dbName = result[0][0]
 
-        sql 'CREATE TABLE tbl_1 (k1 INT, k2 INT) PROPERTIES ( "light_schema_change" = "false")'
+        sql 'CREATE TABLE tbl_1 (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 10 PROPERTIES ( "light_schema_change" = "false")'
         sql 'INSERT INTO tbl_1 VALUES (1, 10)'
         sql 'INSERT INTO tbl_1 VALUES (2, 20)'
         order_qt_select_1_1 'SELECT * FROM tbl_1'
@@ -36,12 +36,12 @@ suite('test_schema_change_concurrent_with_txn') {
         sql 'CREATE TABLE tbl_2 AS SELECT * FROM tbl_1'
         order_qt_select_2_1 'SELECT * FROM tbl_2'
 
-        sql 'CREATE TABLE tbl_3 (k1 INT, k2 INT, v INT SUM) AGGREGATE KEY (k1, k2)'
+        sql 'CREATE TABLE tbl_3 (k1 INT, k2 INT, v INT SUM) AGGREGATE KEY (k1, k2) DISTRIBUTED BY HASH(k1) BUCKETS 10'
         sql 'INSERT INTO tbl_3 VALUES (1, 11, 111)'
         sql 'INSERT INTO tbl_3 VALUES (2, 22, 222)'
         order_qt_select_3_1 'SELECT * FROM tbl_3'
 
-        sql 'CREATE TABLE tbl_4 (k1 INT, k2 INT)'
+        sql 'CREATE TABLE tbl_4 (k1 INT, k2 INT) DISTRIBUTED BY HASH(k1) BUCKETS 10'
         sql 'INSERT INTO tbl_4 VALUES (1, 10)'
         sql 'INSERT INTO tbl_4 VALUES (2, 20)'
         order_qt_select_4_1 'SELECT * FROM tbl_4'
@@ -65,9 +65,9 @@ suite('test_schema_change_concurrent_with_txn') {
         sql 'INSERT INTO tbl_4 VALUES (4, 40)'
         order_qt_select_4_2 'SELECT * FROM tbl_4'
 
-        result = sql 'SHOW PROC "/transactions"'
-        def runningTxn = result.find({ it[1].indexOf(dbName) > 0 })[2]
-        assertEquals(8, runningTxn as int)
+        result = sql_return_maparray 'SHOW PROC "/transactions"'
+        def runningTxn = result.find { it.DbName.indexOf(dbName) >= 0 }.RunningTransactionNum as int
+        assertEquals(8, runningTxn)
 
         sql "ALTER TABLE tbl_1 ADD COLUMN k3 INT DEFAULT '-1'"
         sql 'CREATE MATERIALIZED VIEW tbl_2_mv AS SELECT k1, k1 + k2 FROM tbl_2'
@@ -77,24 +77,22 @@ suite('test_schema_change_concurrent_with_txn') {
         sleep(5000)
 
         def jobs = null
-        def scJobState = job -> job[9]
-        def rollupJobState = job -> job[8]
 
-        jobs = sql "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_1'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_1'"
         assertEquals(1, jobs.size())
-        assertEquals('WAITING_TXN', scJobState(jobs[0]))
+        assertEquals('WAITING_TXN', jobs[0].State)
 
-        jobs = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName = 'tbl_2'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName = 'tbl_2'"
         assertEquals(1, jobs.size())
-        assertEquals('WAITING_TXN', rollupJobState(jobs[0]))
+        assertEquals('WAITING_TXN', jobs[0].State)
 
-        jobs = sql "SHOW ALTER TABLE ROLLUP WHERE TableName = 'tbl_3'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE ROLLUP WHERE TableName = 'tbl_3'"
         assertEquals(1, jobs.size())
-        assertEquals('WAITING_TXN', rollupJobState(jobs[0]))
+        assertEquals('WAITING_TXN', jobs[0].State)
 
-        jobs = sql "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_4'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_4'"
         assertEquals(1, jobs.size())
-        assertEquals('WAITING_TXN', scJobState(jobs[0]))
+        assertEquals('WAITING_TXN', jobs[0].State)
 
         sql 'INSERT INTO tbl_1(k1, k2) VALUES (5, 50)'
         sql 'INSERT INTO tbl_2 VALUES (5, 50)'
@@ -116,20 +114,20 @@ suite('test_schema_change_concurrent_with_txn') {
         order_qt_select_3_3 'SELECT * FROM tbl_3'
         order_qt_select_4_3 'SELECT * FROM tbl_4'
 
-        jobs = sql "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_1'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_1'"
         assertEquals(1, jobs.size())
-        assertEquals('FINISHED', scJobState(jobs[0]))
+        assertEquals('FINISHED', jobs[0].State)
 
-        jobs = sql "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName = 'tbl_2'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE MATERIALIZED VIEW WHERE TableName = 'tbl_2'"
         assertEquals(1, jobs.size())
-        assertEquals('FINISHED', rollupJobState(jobs[0]))
+        assertEquals('FINISHED', jobs[0].State)
 
-        jobs = sql "SHOW ALTER TABLE ROLLUP WHERE TableName = 'tbl_3'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE ROLLUP WHERE TableName = 'tbl_3'"
         assertEquals(1, jobs.size())
-        assertEquals('FINISHED', rollupJobState(jobs[0]))
+        assertEquals('FINISHED', jobs[0].State)
 
-        jobs = sql "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_4'"
+        jobs = sql_return_maparray "SHOW ALTER TABLE COLUMN WHERE TableName = 'tbl_4'"
         assertEquals(1, jobs.size())
-        assertEquals('FINISHED', scJobState(jobs[0]))
+        assertEquals('FINISHED', jobs[0].State)
     }
 }

From d6dcf962a9385b27659989795bd40a03be445683 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B5=B5=E7=A1=95?= <1443539042@qq.com>
Date: Fri, 29 Dec 2023 15:11:41 +0800
Subject: [PATCH 165/299] [Enhancement](page cache) insert into setting to
 disable page cache (#28913)

---
 .../org/apache/doris/analysis/QueryStmt.java  |  4 ++
 .../processor/pre/PlanPreprocessors.java      |  3 +-
 .../TurnOffPageCacheForInsertIntoSelect.java  | 66 +++++++++++++++++++
 .../org/apache/doris/qe/StmtExecutor.java     |  6 ++
 4 files changed, 78 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
index e92fcc2c8f2e38..c1d879fd0604b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
@@ -824,6 +824,10 @@ public void setFromInsert(boolean value) {
         this.fromInsert = value;
     }
 
+    public boolean isFromInsert() {
+        return fromInsert;
+    }
+
     @Override
     public abstract QueryStmt clone();
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PlanPreprocessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PlanPreprocessors.java
index 9dfc12b2c10610..dad0a4d07f0de3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PlanPreprocessors.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PlanPreprocessors.java
@@ -46,7 +46,8 @@ public LogicalPlan process(LogicalPlan logicalPlan) {
     public List<PlanPreprocessor> getProcessors() {
         // add processor if we need
         return ImmutableList.of(
-                new TurnOffPipelineForDml()
+                new TurnOffPipelineForDml(),
+                new TurnOffPageCacheForInsertIntoSelect()
         );
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java
new file mode 100644
index 00000000000000..77955a94114ccb
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.processor.pre;
+
+import org.apache.doris.analysis.SetVar;
+import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.analyzer.UnboundTableSink;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.VariableMgr;
+
+/**
+ * TODO turnoff pagecache for insert into select
+ */
+public class TurnOffPageCacheForInsertIntoSelect extends PlanPreprocessor {
+
+    @Override
+    public Plan visitUnboundTableSink(UnboundTableSink<? extends Plan> unboundTableSink,
+            StatementContext context) {
+        turnOffPageCache(context);
+        return unboundTableSink;
+    }
+
+    @Override
+    public Plan visitLogicalFileSink(LogicalFileSink<? extends Plan> fileSink, StatementContext context) {
+        turnOffPageCache(context);
+        return fileSink;
+    }
+
+    @Override
+    public Plan visitLogicalOlapTableSink(LogicalOlapTableSink<? extends Plan> tableSink, StatementContext context) {
+        turnOffPageCache(context);
+        return tableSink;
+    }
+
+    private void turnOffPageCache(StatementContext context) {
+        SessionVariable sessionVariable = context.getConnectContext().getSessionVariable();
+        // set temporary session value, and then revert value in the 'finally block' of StmtExecutor#execute
+        sessionVariable.setIsSingleSetVar(true);
+        try {
+            VariableMgr.setVar(sessionVariable,
+                new SetVar(SessionVariable.ENABLE_PAGE_CACHE, new StringLiteral("false")));
+        } catch (Throwable t) {
+            throw new AnalysisException("Can not set turn off page cache for insert into select", t);
+        }
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 2851304dc372e1..25f2f2e2e28448 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -886,8 +886,14 @@ private void analyzeVariablesInStmt(StatementBase statement) throws DdlException
         if (statement instanceof SelectStmt) {
             SelectStmt selectStmt = (SelectStmt) statement;
             Map<String, String> optHints = selectStmt.getSelectList().getOptHints();
+            if (optHints == null) {
+                optHints = new HashMap<>();
+            }
             if (optHints != null) {
                 sessionVariable.setIsSingleSetVar(true);
+                if (selectStmt.isFromInsert()) {
+                    optHints.put("enable_page_cache", "false");
+                }
                 for (String key : optHints.keySet()) {
                     VariableMgr.setVar(sessionVariable, new SetVar(key, new StringLiteral(optHints.get(key))));
                 }

From c3c34e10bb1b63b856044bb09b72dbbb65eff4fe Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@apache.org>
Date: Fri, 29 Dec 2023 15:41:16 +0800
Subject: [PATCH 166/299] [feature](executor) Add some check when create
 workload group/workload schedule policy (#29236)

---
 .../workload_sched_policy_mgr.cpp             |  3 +
 .../java/org/apache/doris/common/Config.java  | 12 +++
 .../workloadgroup/WorkloadGroupMgr.java       | 17 +---
 .../WorkloadMetricType.java                   |  2 +-
 .../WorkloadSchedPolicy.java                  | 17 ++++
 .../WorkloadSchedPolicyMgr.java               | 81 ++++++++++++++++++-
 .../test_workload_sched_policy.out            |  7 +-
 .../test_workload_sched_policy.groovy         | 51 ++++++------
 8 files changed, 147 insertions(+), 43 deletions(-)

diff --git a/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp b/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp
index 8a30b5395eb3e2..2398bff465c82c 100644
--- a/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp
+++ b/be/src/runtime/workload_management/workload_sched_policy_mgr.cpp
@@ -78,6 +78,9 @@ void WorkloadSchedPolicyMgr::_schedule_workload() {
         std::vector<WorkloadQueryInfo> list;
         _exec_env->fragment_mgr()->get_runtime_query_info(&list);
         // todo: add timer
+        if (list.size() == 0) {
+            continue;
+        }
         LOG(INFO) << "[workload_schedule] get query list size=" << list.size();
 
         for (int i = 0; i < list.size(); i++) {
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 0f4d0193470efe..4e053524552043 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2318,6 +2318,18 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true)
     public static int workload_action_interval_ms = 10000; // 10s
 
+    @ConfField(mutable = true, masterOnly = true)
+    public static int workload_max_policy_num = 25;
+
+    @ConfField(mutable = true, masterOnly = true)
+    public static int workload_max_condition_num_in_policy = 5;
+
+    @ConfField(mutable = true, masterOnly = true)
+    public static int workload_max_action_num_in_policy = 5; // mainly used to limit set session var action
+
+    @ConfField(mutable = true, masterOnly = true)
+    public static int workload_group_max_num = 15;
+
     @ConfField(description = {"查询be wal_queue 的超时阈值(ms)",
             "the timeout threshold of checking wal_queue on be(ms)"})
     public static int check_wal_queue_timeout_threshold = 180000;   // 3 min
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
index c8b49a5ebaa267..85a87f610e1f8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadgroup/WorkloadGroupMgr.java
@@ -153,13 +153,6 @@ private void writeUnlock() {
         lock.writeLock().unlock();
     }
 
-    private void checkWorkloadGroupEnabled() throws DdlException {
-        if (!Config.enable_workload_group) {
-            throw new DdlException(
-                    "WorkloadGroup is disabled, you can set config enable_workload_group = true to enable it");
-        }
-    }
-
     public void init() {
         if (Config.enable_workload_group || Config.use_fuzzy_session_variable /* for github workflow */) {
             checkAndCreateDefaultGroup();
@@ -265,8 +258,6 @@ private void checkAndCreateDefaultGroup() {
     }
 
     public void createWorkloadGroup(CreateWorkloadGroupStmt stmt) throws DdlException {
-        checkWorkloadGroupEnabled();
-
         WorkloadGroup workloadGroup = WorkloadGroup.create(stmt.getWorkloadGroupName(), stmt.getProperties());
         String workloadGroupName = workloadGroup.getName();
         writeLock();
@@ -277,6 +268,10 @@ public void createWorkloadGroup(CreateWorkloadGroupStmt stmt) throws DdlExceptio
                 }
                 throw new DdlException("workload group " + workloadGroupName + " already exist");
             }
+            if (idToWorkloadGroup.size() >= Config.workload_group_max_num) {
+                throw new DdlException(
+                        "workload group number can not be exceed " + Config.workload_group_max_num);
+            }
             checkGlobalUnlock(workloadGroup, null);
             nameToWorkloadGroup.put(workloadGroupName, workloadGroup);
             idToWorkloadGroup.put(workloadGroup.getId(), workloadGroup);
@@ -323,8 +318,6 @@ private void checkGlobalUnlock(WorkloadGroup workloadGroup, WorkloadGroup old) t
     }
 
     public void alterWorkloadGroup(AlterWorkloadGroupStmt stmt) throws DdlException {
-        checkWorkloadGroupEnabled();
-
         String workloadGroupName = stmt.getWorkloadGroupName();
         Map<String, String> properties = stmt.getProperties();
         WorkloadGroup newWorkloadGroup;
@@ -346,8 +339,6 @@ public void alterWorkloadGroup(AlterWorkloadGroupStmt stmt) throws DdlException
     }
 
     public void dropWorkloadGroup(DropWorkloadGroupStmt stmt) throws DdlException {
-        checkWorkloadGroupEnabled();
-
         String workloadGroupName = stmt.getWorkloadGroupName();
         if (DEFAULT_GROUP_NAME.equals(workloadGroupName)) {
             throw new DdlException("Dropping default workload group " + workloadGroupName + " is not allowed");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadMetricType.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadMetricType.java
index 89dfde9eba38a5..f81d75c675f273 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadMetricType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadMetricType.java
@@ -18,5 +18,5 @@
 package org.apache.doris.resource.workloadschedpolicy;
 
 public enum WorkloadMetricType {
-    USERNAME, QUERY_TIME
+    USERNAME, QUERY_TIME, SCAN_ROWS, SCAN_BYTES
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java
index d514ea62d2d0ff..13637b5b8be1ea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicy.java
@@ -87,6 +87,8 @@ public class WorkloadSchedPolicy implements Writable, GsonPostProcessable {
     private List<WorkloadCondition> workloadConditionList;
     private List<WorkloadAction> workloadActionList;
 
+    private Boolean isFePolicy = null;
+
     // for ut
     public WorkloadSchedPolicy() {
     }
@@ -198,6 +200,21 @@ public List<WorkloadActionMeta> getActionMetaList() {
         return actionMetaList;
     }
 
+    // true, current policy can only run in FE;
+    // false, current policy can only run in BE
+    public boolean isFePolicy() {
+        if (isFePolicy == null) {
+            isFePolicy = false;
+            for (WorkloadAction action : workloadActionList) {
+                if (WorkloadSchedPolicyMgr.FE_ACTION_SET.contains(action.getWorkloadActionType())) {
+                    isFePolicy = true;
+                    break;
+                }
+            }
+        }
+        return isFePolicy;
+    }
+
     public TopicInfo toTopicInfo() {
         TWorkloadSchedPolicy tPolicy = new TWorkloadSchedPolicy();
         tPolicy.setId(id);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
index 45ba3a35de469b..0074020735e113 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyMgr.java
@@ -39,6 +39,7 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Maps;
 import com.google.gson.annotations.SerializedName;
 import org.apache.logging.log4j.LogManager;
@@ -73,6 +74,22 @@ public class WorkloadSchedPolicyMgr implements Writable, GsonPostProcessable {
             .add("Id").add("Name").add("Condition").add("Action").add("Priority").add("Enabled").add("Version")
             .build();
 
+    public static final ImmutableSet<WorkloadActionType> FE_ACTION_SET
+            = new ImmutableSet.Builder<WorkloadActionType>().add(WorkloadActionType.SET_SESSION_VARIABLE).build();
+
+    public static final ImmutableSet<WorkloadMetricType> FE_METRIC_SET
+            = new ImmutableSet.Builder<WorkloadMetricType>().add(WorkloadMetricType.USERNAME)
+            .build();
+
+    public static final ImmutableSet<WorkloadActionType> BE_ACTION_SET
+            = new ImmutableSet.Builder<WorkloadActionType>().add(WorkloadActionType.MOVE_QUERY_TO_GROUP)
+            .add(WorkloadActionType.CANCEL_QUERY).build();
+
+    public static final ImmutableSet<WorkloadMetricType> BE_METRIC_SET
+            = new ImmutableSet.Builder<WorkloadMetricType>().add(WorkloadMetricType.SCAN_ROWS)
+            .add(WorkloadMetricType.SCAN_BYTES).add(WorkloadMetricType.QUERY_TIME)
+            .build();
+
     private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
 
     public static Comparator<WorkloadSchedPolicy> policyComparator = new Comparator<WorkloadSchedPolicy>() {
@@ -151,6 +168,7 @@ public void createWorkloadSchedPolicy(CreateWorkloadSchedPolicyStmt createStmt)
             WorkloadCondition cond = WorkloadCondition.createWorkloadCondition(cm);
             policyConditionList.add(cond);
         }
+        boolean feCondition = checkPolicyCondition(policyConditionList);
 
         // 2 create action
         List<WorkloadActionMeta> originActions = createStmt.getActions();
@@ -172,7 +190,11 @@ public void createWorkloadSchedPolicy(CreateWorkloadSchedPolicyStmt createStmt)
             WorkloadAction ret = WorkloadAction.createWorkloadAction(workloadActionMeta);
             policyActionList.add(ret);
         }
-        checkPolicyActionConflicts(policyActionList);
+
+        boolean feAction = checkPolicyAction(policyActionList);
+        if (feAction != feCondition) {
+            throw new UserException("action and metric must run in FE together or run in BE together");
+        }
 
         // 3 create policy
         Map<String, String> propMap = createStmt.getProperties();
@@ -191,6 +213,10 @@ public void createWorkloadSchedPolicy(CreateWorkloadSchedPolicyStmt createStmt)
                     throw new UserException("workload schedule policy " + policyName + " already exists ");
                 }
             }
+            if (idToPolicy.size() >= Config.workload_max_policy_num) {
+                throw new UserException(
+                        "workload scheduler policy num can not exceed " + Config.workload_max_policy_num);
+            }
             long id = Env.getCurrentEnv().getNextId();
             WorkloadSchedPolicy policy = new WorkloadSchedPolicy(id, policyName,
                     policyConditionList, policyActionList, propMap);
@@ -204,10 +230,51 @@ public void createWorkloadSchedPolicy(CreateWorkloadSchedPolicyStmt createStmt)
         }
     }
 
-    private void checkPolicyActionConflicts(List<WorkloadAction> actionList) throws UserException {
+    private boolean checkPolicyCondition(List<WorkloadCondition> conditionList) throws UserException {
+        if (conditionList.size() > Config.workload_max_condition_num_in_policy) {
+            throw new UserException(
+                    "condition num in a policy can not exceed " + Config.workload_max_condition_num_in_policy);
+        }
+        boolean containsFeMetric = false;
+        boolean containsBeMetric = false;
+        for (WorkloadCondition cond : conditionList) {
+            if (FE_METRIC_SET.contains(cond.getMetricType())) {
+                containsFeMetric = true;
+            }
+            if (BE_METRIC_SET.contains(cond.getMetricType())) {
+                containsBeMetric = true;
+            }
+            if (containsFeMetric && containsBeMetric) {
+                throw new UserException(
+                        "one policy can not contains fe and be metric, FE metric list is " + FE_METRIC_SET
+                                + ", BE metric list is " + BE_METRIC_SET);
+            }
+        }
+        return containsFeMetric;
+    }
+
+    private boolean checkPolicyAction(List<WorkloadAction> actionList) throws UserException {
+        if (actionList.size() > Config.workload_max_action_num_in_policy) {
+            throw new UserException(
+                    "action num in one policy can not exceed " + Config.workload_max_action_num_in_policy);
+        }
+
         Set<WorkloadActionType> actionTypeSet = new HashSet<>();
         Set<String> setSessionVarSet = new HashSet<>();
+        boolean containsFeAction = false;
+        boolean containsBeAction = false;
         for (WorkloadAction action : actionList) {
+            if (FE_ACTION_SET.contains(action.getWorkloadActionType())) {
+                containsFeAction = true;
+            }
+            if (BE_ACTION_SET.contains(action.getWorkloadActionType())) {
+                containsBeAction = true;
+            }
+            if (containsFeAction && containsBeAction) {
+                throw new UserException(
+                        "one policy can not contains fe and be action, FE action list is " + FE_ACTION_SET
+                                + ", BE action list is " + BE_ACTION_SET);
+            }
             // set session var cmd can be duplicate, but args can not be duplicate
             if (action.getWorkloadActionType().equals(WorkloadActionType.SET_SESSION_VARIABLE)) {
                 WorkloadActionSetSessionVar setAction = (WorkloadActionSetSessionVar) action;
@@ -225,6 +292,7 @@ private void checkPolicyActionConflicts(List<WorkloadAction> actionList) throws
             throw new UserException(String.format("%s and %s can not exist in one policy at same time",
                     WorkloadActionType.CANCEL_QUERY, WorkloadActionType.MOVE_QUERY_TO_GROUP));
         }
+        return containsFeAction;
     }
 
     public void execPolicy(List<WorkloadQueryInfo> queryInfoList) {
@@ -232,7 +300,11 @@ public void execPolicy(List<WorkloadQueryInfo> queryInfoList) {
         Set<Long> policyIdSet = new HashSet<>();
         readLock();
         try {
-            policyIdSet.addAll(idToPolicy.keySet());
+            for (Map.Entry<Long, WorkloadSchedPolicy> entry : idToPolicy.entrySet()) {
+                if (entry.getValue().isFePolicy()) {
+                    policyIdSet.add(entry.getKey());
+                }
+            }
         } finally {
             readUnlock();
         }
@@ -418,6 +490,9 @@ public List<TopicInfo> getPublishTopicInfoList() {
         readLock();
         try {
             for (Map.Entry<Long, WorkloadSchedPolicy> entry : idToPolicy.entrySet()) {
+                if (entry.getValue().isFePolicy()) {
+                    continue;
+                }
                 TopicInfo tInfo = entry.getValue().toTopicInfo();
                 if (tInfo != null) {
                     topicInfoList.add(tInfo);
diff --git a/regression-test/data/workload_manager_p0/test_workload_sched_policy.out b/regression-test/data/workload_manager_p0/test_workload_sched_policy.out
index 4e8482384c46a2..c5028f8363eeb6 100644
--- a/regression-test/data/workload_manager_p0/test_workload_sched_policy.out
+++ b/regression-test/data/workload_manager_p0/test_workload_sched_policy.out
@@ -1,8 +1,9 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !select_policy_tvf --
-full_policy_policy	query_time > 10;username = root	set_session_variable "workload_group=normal"	10	false	0
-move_action_policy	username = root	move_query_to_group "normal"	0	true	0
-set_action_policy	query_time > 10;username = root	set_session_variable "workload_group=normal"	0	true	0
+be_policy	query_time > 10	cancel_query	10	true	0
+fe_policy	username = root	set_session_variable "workload_group=normal"	10	false	0
+move_action_policy	query_time > 10	move_query_to_group "normal"	0	true	0
+set_action_policy	username = root	set_session_variable "workload_group=normal"	0	true	0
 test_cancel_policy	query_time > 10	cancel_query	0	false	0
 
 -- !select_policy_tvf_after_drop --
diff --git a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
index 2c51c79ae0099d..d59bb76dd5ef17 100644
--- a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
+++ b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
@@ -19,12 +19,11 @@ suite("test_workload_sched_policy") {
 
     sql "set experimental_enable_nereids_planner = false;"
 
-    sql "drop workload schedule policy if exists full_policy_policy;"
-    sql "drop workload schedule policy if exists set_action_policy;"
-    sql "drop workload schedule policy if exists move_action_policy;"
     sql "drop workload schedule policy if exists test_cancel_policy;"
-    sql "drop workload schedule policy if exists test_set_var_policy;"
-    sql "drop workload schedule policy if exists test_set_var_policy2;"
+    sql "drop workload schedule policy if exists move_action_policy;"
+    sql "drop workload schedule policy if exists set_action_policy;"
+    sql "drop workload schedule policy if exists fe_policy;"
+    sql "drop workload schedule policy if exists be_policy;"
 
     // 1 create cancel policy
     sql "create workload schedule policy test_cancel_policy " +
@@ -33,27 +32,36 @@ suite("test_workload_sched_policy") {
 
     // 2 create cancel policy
     sql "create workload schedule policy move_action_policy " +
-            "conditions(username='root') " +
+            "conditions(query_time > 10) " +
             "actions(move_query_to_group 'normal');"
 
     // 3 create set policy
     sql "create workload schedule policy set_action_policy " +
-            "conditions(query_time > 10, username='root') " +
+            "conditions(username='root') " +
             "actions(set_session_variable 'workload_group=normal');"
 
-    // 4 create policy with property
-    sql "create workload schedule policy full_policy_policy " +
-            "conditions(query_time > 10, username='root') " +
+    // 4 create policy run in fe
+    sql "create workload schedule policy fe_policy " +
+            "conditions(username='root') " +
             "actions(set_session_variable 'workload_group=normal') " +
             "properties( " +
             "'enabled' = 'false', " +
             "'priority'='10' " +
             ");"
 
+    // 5 create policy run in be
+    sql "create workload schedule policy be_policy " +
+            "conditions(query_time > 10) " +
+            "actions(cancel_query) " +
+            "properties( " +
+            "'enabled' = 'true', " +
+            "'priority'='10' " +
+            ");"
+
     qt_select_policy_tvf "select name,condition,action,priority,enabled,version from workload_schedule_policy() order by name;"
 
     // test_alter
-    sql "alter workload schedule policy full_policy_policy properties('priority'='2', 'enabled'='false');"
+    sql "alter workload schedule policy fe_policy properties('priority'='2', 'enabled'='false');"
 
     // create failed check
     test {
@@ -71,26 +79,26 @@ suite("test_workload_sched_policy") {
     }
 
     test {
-        sql "alter workload schedule policy full_policy_policy properties('priority'='abc');"
+        sql "alter workload schedule policy fe_policy properties('priority'='abc');"
 
         exception "invalid priority property value"
     }
 
     test {
-        sql "alter workload schedule policy full_policy_policy properties('enabled'='abc');"
+        sql "alter workload schedule policy fe_policy properties('enabled'='abc');"
 
         exception "invalid enabled property value"
     }
 
     test {
-        sql "alter workload schedule policy full_policy_policy properties('priority'='10000');"
+        sql "alter workload schedule policy fe_policy properties('priority'='10000');"
 
         exception "priority can only between"
     }
 
     test {
         sql "create workload schedule policy conflict_policy " +
-                "conditions (username = 'root')" +
+                "conditions (query_time > 0)" +
                 "actions(cancel_query, move_query_to_group 'normal');"
 
         exception "can not exist in one policy at same time"
@@ -98,7 +106,7 @@ suite("test_workload_sched_policy") {
 
     test {
         sql "create workload schedule policy conflict_policy " +
-                "conditions (username = 'root') " +
+                "conditions (query_time > 0) " +
                 "actions(cancel_query, cancel_query);"
 
         exception "duplicate action in one policy"
@@ -113,10 +121,11 @@ suite("test_workload_sched_policy") {
     }
 
     // drop
-    sql "drop workload schedule policy full_policy_policy;"
-    sql "drop workload schedule policy set_action_policy;"
-    sql "drop workload schedule policy move_action_policy;"
     sql "drop workload schedule policy test_cancel_policy;"
+    sql "drop workload schedule policy move_action_policy;"
+    sql "drop workload schedule policy set_action_policy;"
+    sql "drop workload schedule policy fe_policy;"
+    sql "drop workload schedule policy be_policy;"
 
     qt_select_policy_tvf_after_drop "select name,condition,action,priority,enabled,version from workload_schedule_policy() order by name;"
 
@@ -158,10 +167,6 @@ suite("test_workload_sched_policy") {
 
     sql "ADMIN SET FRONTEND CONFIG ('workload_sched_policy_interval_ms' = '10000');"
 
-    sql "drop workload schedule policy if exists full_policy_policy;"
-    sql "drop workload schedule policy if exists set_action_policy;"
-    sql "drop workload schedule policy if exists move_action_policy;"
-    sql "drop workload schedule policy if exists test_cancel_policy;"
     sql "drop workload schedule policy if exists test_set_var_policy;"
     sql "drop workload schedule policy if exists test_set_var_policy2;"
 

From 9925f7be8e6559f92830b8dd9fe3baac2c051376 Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Fri, 29 Dec 2023 16:12:05 +0800
Subject: [PATCH 167/299] [fix](nereids)FillUpMissingSlots rule didn't process
 standalone having clause correctly (#29143)

---
 .../rules/analysis/FillUpMissingSlots.java    | 19 ++++++++++++++
 .../nereids/rules/rewrite/ReorderJoin.java    | 25 ++++++++-----------
 .../suites/nereids_syntax_p0/having.groovy    |  8 ++++++
 3 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java
index afee320f6f591f..a500d9bb6683c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/FillUpMissingSlots.java
@@ -157,6 +157,25 @@ public List<Rule> buildRules() {
                 })
             ),
             // Convert having to filter
+            RuleType.FILL_UP_HAVING_PROJECT.build(
+                    logicalHaving(logicalProject()).then(having -> {
+                        LogicalProject<Plan> project = having.child();
+                        Set<Slot> projectOutputSet = project.getOutputSet();
+                        Set<Slot> notExistedInProject = having.getExpressions().stream()
+                                .map(Expression::getInputSlots)
+                                .flatMap(Set::stream)
+                                .filter(s -> !projectOutputSet.contains(s))
+                                .collect(Collectors.toSet());
+                        if (notExistedInProject.size() == 0) {
+                            return null;
+                        }
+                        List<NamedExpression> projects = ImmutableList.<NamedExpression>builder()
+                                .addAll(project.getProjects()).addAll(notExistedInProject).build();
+                        return new LogicalProject<>(ImmutableList.copyOf(project.getOutput()),
+                                having.withChildren(new LogicalProject<>(projects, project.child())));
+                    })
+            ),
+            // Convert having to filter
             RuleType.FILL_UP_HAVING_PROJECT.build(
                 logicalHaving().then(having -> new LogicalFilter<>(having.getConjuncts(), having.child()))
             )
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java
index 31b0dc904ab8ce..03c4e7e8ed81c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ReorderJoin.java
@@ -329,10 +329,12 @@ private static boolean canCombine(Plan input, boolean changeChildren) {
             Set<Expression> joinFilter, Set<Integer> usedPlansIndex, Map<Plan, JoinHintType> planToHintType) {
         List<Expression> otherJoinConditions = Lists.newArrayList();
         Set<ExprId> leftOutputExprIdSet = left.getOutputExprIdSet();
+        int candidateIndex = 0;
         for (int i = 0; i < candidates.size(); i++) {
             if (usedPlansIndex.contains(i)) {
                 continue;
             }
+            candidateIndex = i;
 
             Plan candidate = candidates.get(i);
             Set<ExprId> rightOutputExprIdSet = candidate.getOutputExprIdSet();
@@ -362,21 +364,14 @@ private static boolean canCombine(Plan input, boolean changeChildren) {
         }
         // All { left -> one in [candidates] } is CrossJoin
         // Generate a CrossJoin
-        for (int i = 0; i < candidates.size(); i++) {
-            if (usedPlansIndex.contains(i)) {
-                continue;
-            }
-            usedPlansIndex.add(i);
-            Plan right = candidates.get(i);
-            return new LogicalJoin<>(JoinType.CROSS_JOIN,
-                    ExpressionUtils.EMPTY_CONDITION,
-                    otherJoinConditions,
-                    JoinHint.fromRightPlanHintType(planToHintType.getOrDefault(right, JoinHintType.NONE)),
-                    Optional.empty(),
-                    left, right);
-        }
-
-        throw new RuntimeException("findInnerJoin: can't reach here");
+        usedPlansIndex.add(candidateIndex);
+        Plan right = candidates.get(candidateIndex);
+        return new LogicalJoin<>(JoinType.CROSS_JOIN,
+                ExpressionUtils.EMPTY_CONDITION,
+                otherJoinConditions,
+                JoinHint.fromRightPlanHintType(planToHintType.getOrDefault(right, JoinHintType.NONE)),
+                Optional.empty(),
+                left, right);
     }
 
     private boolean nonJoinAndNonFilter(Plan plan) {
diff --git a/regression-test/suites/nereids_syntax_p0/having.groovy b/regression-test/suites/nereids_syntax_p0/having.groovy
index a9a4b3ae25379f..3db076ea7dfc3b 100644
--- a/regression-test/suites/nereids_syntax_p0/having.groovy
+++ b/regression-test/suites/nereids_syntax_p0/having.groovy
@@ -68,4 +68,12 @@ suite("test_nereids_having") {
     order_qt_select "SELECT SUM(a1 + a2) FROM test_nereids_having_tbl HAVING SUM(a1 + a2) > 0";
     order_qt_select "SELECT SUM(a1 + a2) FROM test_nereids_having_tbl HAVING SUM(a1 + a2 + 3) > 0";
     order_qt_select "SELECT COUNT(*) FROM test_nereids_having_tbl HAVING COUNT(*) > 0";
+    sql """SELECT alias2.`pk` AS field4
+                            FROM 
+                                (SELECT pk
+                                FROM test_nereids_having_tbl AS SQ1_alias1 ) AS alias2
+                            HAVING alias2.`pk` <> 
+                                (SELECT *
+                                FROM 
+                                    (SELECT "xAbfcUSAOy") __DORIS_DUAL__ );"""
 }

From 866fd5d32a459303ba48d64d6a02ebe18e40f33c Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Fri, 29 Dec 2023 16:12:32 +0800
Subject: [PATCH 168/299] [feature](fe)support last column or index definition
 end with comma in create table statement (#29167)

allow the column list end with COMMA in create table statement. This is a issue rooted from history. So nereids has to keep the same behavior as old planner.
CREATE TABLE t
(
k1 int,
)
---
 .../org/apache/doris/nereids/DorisParser.g4   |  2 +-
 fe/fe-core/src/main/cup/sql_parser.cup        | 26 +++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 98260c77dbd583..0aab15c884a925 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -41,7 +41,7 @@ statement
         TO (user=userIdentify | ROLE roleName=identifier)
         USING LEFT_PAREN booleanExpression RIGHT_PAREN                 #createRowPolicy
     | CREATE (EXTERNAL)? TABLE (IF NOT EXISTS)? name=multipartIdentifier
-        ((ctasCols=identifierList)? | (LEFT_PAREN columnDefs (COMMA indexDefs)? RIGHT_PAREN))
+        ((ctasCols=identifierList)? | (LEFT_PAREN columnDefs (COMMA indexDefs)? COMMA? RIGHT_PAREN))
         (ENGINE EQ engine=identifier)?
         ((AGGREGATE | UNIQUE | DUPLICATE) KEY keys=identifierList (CLUSTER BY clusterKeys=identifierList)?)?
         (COMMENT STRING_LITERAL)?
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 7e8f4dd96b569a..26b221c128c6e3 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -1800,6 +1800,19 @@ create_stmt ::=
         RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, engineName, keys, partition,
         distribution, tblProperties, extProperties, tableComment, index);
     :}
+    | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
+            LPAREN column_definition_list:columns COMMA RPAREN opt_engine:engineName
+            opt_keys:keys
+            opt_comment:tableComment
+            opt_partition:partition
+            opt_distribution:distribution
+            opt_rollup:index
+            opt_properties:tblProperties
+            opt_ext_properties:extProperties
+    {:
+        RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, null, engineName, keys, partition,
+        distribution, tblProperties, extProperties, tableComment, index);
+    :}
     | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
             LPAREN column_definition_list:columns COMMA index_definition_list:indexes RPAREN opt_engine:engineName
             opt_keys:keys
@@ -1813,6 +1826,19 @@ create_stmt ::=
         RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, keys, partition,
         distribution, tblProperties, extProperties, tableComment, index);
     :}
+    | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
+            LPAREN column_definition_list:columns COMMA index_definition_list:indexes COMMA RPAREN opt_engine:engineName
+            opt_keys:keys
+            opt_comment:tableComment
+            opt_partition:partition
+            opt_distribution:distribution
+            opt_rollup:index
+            opt_properties:tblProperties
+            opt_ext_properties:extProperties
+    {:
+        RESULT = new CreateTableStmt(ifNotExists, isExternal, name, columns, indexes, engineName, keys, partition,
+        distribution, tblProperties, extProperties, tableComment, index);
+    :}
     | KW_CREATE opt_external:isExternal KW_TABLE opt_if_not_exists:ifNotExists table_name:name
             opt_col_list:columns
             opt_engine:engineName

From 82635d4b59e951975988a28fe83ecc97340fc74c Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi02@gmail.com>
Date: Fri, 29 Dec 2023 16:15:56 +0800
Subject: [PATCH 169/299] [opt](memory) All LRU Cache inherit from
 LRUCachePolicy (#28940)

After all LRU Cache inherits from LRUCachePolicy, this will allow prune stale entry, eviction when memory exceeds limit, and define common properties. LRUCache constructor change to private, only allow LRUCachePolicy to construct it.

Impl DummyLRUCache, when LRU Cache capacity is 0, will no longer be meaningless insert and evict.
---
 be/src/common/config.cpp                      |  6 +-
 be/src/common/config.h                        |  7 +-
 be/src/olap/lru_cache.cpp                     | 35 ++++++++++
 be/src/olap/lru_cache.h                       | 50 +++++++++++---
 be/src/olap/page_cache.cpp                    |  6 +-
 be/src/olap/page_cache.h                      | 25 ++-----
 .../segment_v2/inverted_index_cache.cpp       | 52 ++++++--------
 .../rowset/segment_v2/inverted_index_cache.h  | 32 ++++++---
 be/src/olap/rowset/segment_v2/page_io.cpp     |  5 +-
 be/src/olap/schema_cache.h                    | 10 +--
 be/src/olap/segment_loader.cpp                | 12 ++--
 be/src/olap/tablet_meta.cpp                   |  2 +-
 be/src/olap/tablet_meta.h                     | 16 +++--
 be/src/olap/txn_manager.cpp                   | 15 ++--
 be/src/olap/txn_manager.h                     | 11 ++-
 be/src/runtime/exec_env.h                     |  6 ++
 be/src/runtime/exec_env_init.cpp              | 14 ++--
 be/src/runtime/load_channel_mgr.cpp           | 12 ++--
 be/src/runtime/load_channel_mgr.h             | 11 ++-
 be/src/runtime/memory/cache_manager.cpp       |  3 +
 be/src/runtime/memory/cache_policy.cpp        |  4 +-
 be/src/runtime/memory/cache_policy.h          | 26 ++++++-
 be/src/runtime/memory/lru_cache_policy.h      | 68 +++++++++++++++----
 be/src/service/point_query_executor.cpp       | 19 +++---
 be/src/service/point_query_executor.h         | 22 +++---
 be/src/util/obj_lru_cache.cpp                 | 15 ++--
 be/src/util/obj_lru_cache.h                   | 14 ++--
 be/test/olap/lru_cache_test.cpp               | 46 ++++++++-----
 .../inverted_index_searcher_cache_test.cpp    | 29 +++-----
 be/test/testutil/run_all_tests.cpp            |  1 +
 30 files changed, 367 insertions(+), 207 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 1d297ece456cd5..120ca28be7a26a 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -275,7 +275,8 @@ DEFINE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes, "1000000
 
 DEFINE_mInt32(cache_prune_stale_interval, "10");
 // the clean interval of tablet lookup cache
-DEFINE_mInt32(tablet_lookup_cache_clean_interval, "30");
+DEFINE_mInt32(tablet_lookup_cache_stale_sweep_time_sec, "30");
+DEFINE_mInt32(point_query_row_cache_stale_sweep_time_sec, "300");
 DEFINE_mInt32(disk_stat_monitor_interval, "5");
 DEFINE_mInt32(unused_rowset_monitor_interval, "30");
 DEFINE_String(storage_root_path, "${DORIS_HOME}/storage");
@@ -809,6 +810,9 @@ DEFINE_mInt32(external_table_connect_timeout_sec, "30");
 
 // Global bitmap cache capacity for aggregation cache, size in bytes
 DEFINE_Int64(delete_bitmap_agg_cache_capacity, "104857600");
+DEFINE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec, "1800");
+
+DEFINE_mInt32(common_obj_lru_cache_stale_sweep_time_sec, "900");
 
 // s3 config
 DEFINE_mInt32(max_remote_storage_count, "10");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 4e09bb2b08bfab..68c02941cdcb08 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -318,7 +318,8 @@ DECLARE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes);
 // the prune stale interval of all cache
 DECLARE_mInt32(cache_prune_stale_interval);
 // the clean interval of tablet lookup cache
-DECLARE_mInt32(tablet_lookup_cache_clean_interval);
+DECLARE_mInt32(tablet_lookup_cache_stale_sweep_time_sec);
+DECLARE_mInt32(point_query_row_cache_stale_sweep_time_sec);
 DECLARE_mInt32(disk_stat_monitor_interval);
 DECLARE_mInt32(unused_rowset_monitor_interval);
 DECLARE_String(storage_root_path);
@@ -863,6 +864,10 @@ DECLARE_mInt32(external_table_connect_timeout_sec);
 
 // Global bitmap cache capacity for aggregation cache, size in bytes
 DECLARE_Int64(delete_bitmap_agg_cache_capacity);
+DECLARE_mInt32(delete_bitmap_agg_cache_stale_sweep_time_sec);
+
+// A common object cache depends on an Sharded LRU Cache.
+DECLARE_mInt32(common_obj_lru_cache_stale_sweep_time_sec);
 
 // s3 config
 DECLARE_mInt32(max_remote_storage_count);
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index 0de99e20b205e1..efc928704707ab 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -667,4 +667,39 @@ void ShardedLRUCache::update_cache_metrics() const {
             total_lookup_count == 0 ? 0 : ((double)total_hit_count / total_lookup_count));
 }
 
+Cache::Handle* DummyLRUCache::insert(const CacheKey& key, void* value, size_t charge,
+                                     void (*deleter)(const CacheKey& key, void* value),
+                                     CachePriority priority, size_t bytes) {
+    size_t handle_size = sizeof(LRUHandle) - 1 + key.size();
+    auto* e = reinterpret_cast<LRUHandle*>(malloc(handle_size));
+    e->value = value;
+    e->deleter = deleter;
+    e->charge = charge;
+    e->key_length = 0;
+    e->total_size = 0;
+    e->bytes = 0;
+    e->hash = 0;
+    e->refs = 1; // only one for the returned handle
+    e->next = e->prev = nullptr;
+    e->in_cache = false;
+    return reinterpret_cast<Cache::Handle*>(e);
+}
+
+void DummyLRUCache::release(Cache::Handle* handle) {
+    if (handle == nullptr) {
+        return;
+    }
+    auto* e = reinterpret_cast<LRUHandle*>(handle);
+    e->free();
+}
+
+void* DummyLRUCache::value(Handle* handle) {
+    return reinterpret_cast<LRUHandle*>(handle)->value;
+}
+
+Slice DummyLRUCache::value_slice(Handle* handle) {
+    auto* lru_handle = reinterpret_cast<LRUHandle*>(handle);
+    return Slice((char*)lru_handle->value, lru_handle->charge);
+}
+
 } // namespace doris
diff --git a/be/src/olap/lru_cache.h b/be/src/olap/lru_cache.h
index 8608a85bf2d7f4..6ecab80022dafd 100644
--- a/be/src/olap/lru_cache.h
+++ b/be/src/olap/lru_cache.h
@@ -52,12 +52,17 @@ namespace doris {
     } while (0)
 
 class Cache;
+class LRUCachePolicy;
 
 enum LRUCacheType {
     SIZE,  // The capacity of cache is based on the size of cache entry.
     NUMBER // The capacity of cache is based on the number of cache entry.
 };
 
+static constexpr LRUCacheType DEFAULT_LRU_CACHE_TYPE = LRUCacheType::SIZE;
+static constexpr uint32_t DEFAULT_LRU_CACHE_NUM_SHARDS = 16;
+static constexpr size_t DEFAULT_LRU_CACHE_ELEMENT_COUNT_CAPACITY = 0;
+
 class CacheKey {
 public:
     CacheKey() : _data(nullptr), _size(0) {}
@@ -263,8 +268,10 @@ struct LRUHandle {
 
     void free() {
         (*deleter)(key(), value);
-        THREAD_MEM_TRACKER_TRANSFER_FROM(bytes, mem_tracker);
-        DorisMetrics::instance()->lru_cache_memory_bytes->increment(-bytes);
+        if (bytes != 0) { // DummyLRUCache bytes always equal to 0
+            THREAD_MEM_TRACKER_TRANSFER_FROM(bytes, mem_tracker);
+            DorisMetrics::instance()->lru_cache_memory_bytes->increment(-bytes);
+        }
         ::free(this);
     }
 };
@@ -330,6 +337,7 @@ class LRUCache {
     }
 
     // Like Cache methods, but with an extra "hash" parameter.
+    // Must call release on the returned handle pointer.
     Cache::Handle* insert(const CacheKey& key, uint32_t hash, void* value, size_t charge,
                           void (*deleter)(const CacheKey& key, void* value),
                           MemTrackerLimiter* tracker,
@@ -389,14 +397,6 @@ class LRUCache {
 
 class ShardedLRUCache : public Cache {
 public:
-    explicit ShardedLRUCache(const std::string& name, size_t total_capacity,
-                             LRUCacheType type = LRUCacheType::SIZE, uint32_t num_shards = 16,
-                             uint32_t element_count_capacity = 0);
-    explicit ShardedLRUCache(const std::string& name, size_t total_capacity, LRUCacheType type,
-                             uint32_t num_shards,
-                             CacheValueTimeExtractor cache_value_time_extractor,
-                             bool cache_value_check_timestamp, uint32_t element_count_capacity = 0);
-    // TODO(fdy): 析构时清除所有cache元素
     virtual ~ShardedLRUCache();
     virtual Handle* insert(const CacheKey& key, void* value, size_t charge,
                            void (*deleter)(const CacheKey& key, void* value),
@@ -415,6 +415,16 @@ class ShardedLRUCache : public Cache {
     size_t get_total_capacity() override { return _total_capacity; };
 
 private:
+    // LRUCache can only be created and managed with LRUCachePolicy.
+    friend class LRUCachePolicy;
+
+    explicit ShardedLRUCache(const std::string& name, size_t total_capacity, LRUCacheType type,
+                             uint32_t num_shards, uint32_t element_count_capacity);
+    explicit ShardedLRUCache(const std::string& name, size_t total_capacity, LRUCacheType type,
+                             uint32_t num_shards,
+                             CacheValueTimeExtractor cache_value_time_extractor,
+                             bool cache_value_check_timestamp, uint32_t element_count_capacity);
+
     void update_cache_metrics() const;
 
     static std::string lru_cache_type_string(LRUCacheType type) {
@@ -456,4 +466,24 @@ class ShardedLRUCache : public Cache {
     std::unique_ptr<bvar::PerSecond<bvar::Adder<uint64_t>>> _lookup_count_per_second;
 };
 
+// Compatible with ShardedLRUCache usage, but will not actually cache.
+class DummyLRUCache : public Cache {
+public:
+    // Must call release on the returned handle pointer.
+    Handle* insert(const CacheKey& key, void* value, size_t charge,
+                   void (*deleter)(const CacheKey& key, void* value),
+                   CachePriority priority = CachePriority::NORMAL, size_t bytes = -1) override;
+    Handle* lookup(const CacheKey& key) override { return nullptr; };
+    void release(Handle* handle) override;
+    void erase(const CacheKey& key) override {};
+    void* value(Handle* handle) override;
+    Slice value_slice(Handle* handle) override;
+    uint64_t new_id() override { return 0; };
+    int64_t prune() override { return 0; };
+    int64_t prune_if(CacheValuePredicate pred, bool lazy_mode = false) override { return 0; };
+    int64_t mem_consumption() override { return 0; };
+    int64_t get_usage() override { return 0; };
+    size_t get_total_capacity() override { return 0; };
+};
+
 } // namespace doris
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index 1779c293f8957e..6fbcbbe19f35b2 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -48,10 +48,8 @@ StoragePageCache::StoragePageCache(size_t capacity, int32_t index_cache_percenta
     } else {
         CHECK(false) << "invalid index page cache percentage";
     }
-    if (pk_index_cache_capacity > 0) {
-        _pk_index_page_cache =
-                std::make_unique<PKIndexPageCache>(pk_index_cache_capacity, num_shards);
-    }
+
+    _pk_index_page_cache = std::make_unique<PKIndexPageCache>(pk_index_cache_capacity, num_shards);
 }
 
 bool StoragePageCache::lookup(const CacheKey& key, PageCacheHandle* handle,
diff --git a/be/src/olap/page_cache.h b/be/src/olap/page_cache.h
index 2d9357445592fd..a604d205503a9a 100644
--- a/be/src/olap/page_cache.h
+++ b/be/src/olap/page_cache.h
@@ -157,12 +157,6 @@ class StoragePageCache {
     void insert(const CacheKey& key, DataPage* data, PageCacheHandle* handle,
                 segment_v2::PageTypePB page_type, bool in_memory = false);
 
-    // Page cache available check.
-    // When percentage is set to 0 or 100, the index or data cache will not be allocated.
-    bool is_cache_available(segment_v2::PageTypePB page_type) {
-        return _get_page_cache(page_type) != nullptr;
-    }
-
 private:
     StoragePageCache();
 
@@ -177,26 +171,19 @@ class StoragePageCache {
     Cache* _get_page_cache(segment_v2::PageTypePB page_type) {
         switch (page_type) {
         case segment_v2::DATA_PAGE: {
-            if (_data_page_cache) {
-                return _data_page_cache->get();
-            }
-            return nullptr;
+            return _data_page_cache->cache();
         }
         case segment_v2::INDEX_PAGE: {
-            if (_index_page_cache) {
-                return _index_page_cache->get();
-            }
-            return nullptr;
+            return _index_page_cache->cache();
         }
         case segment_v2::PRIMARY_KEY_INDEX_PAGE: {
-            if (_pk_index_page_cache) {
-                return _pk_index_page_cache->get();
-            }
-            return nullptr;
+            return _pk_index_page_cache->cache();
         }
         default:
-            return nullptr;
+            LOG(FATAL) << "get error type page cache";
         }
+        LOG(FATAL) << "__builtin_unreachable";
+        __builtin_unreachable();
     }
 };
 
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
index e88573c1be84ac..f5259e8139ceb2 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
@@ -86,11 +86,7 @@ InvertedIndexSearcherCache* InvertedIndexSearcherCache::create_global_instance(
     return new InvertedIndexSearcherCache(capacity, num_shards);
 }
 
-InvertedIndexSearcherCache::InvertedIndexSearcherCache(size_t capacity, uint32_t num_shards)
-        : LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE,
-                         config::inverted_index_cache_stale_sweep_time_sec),
-          _mem_tracker(std::make_unique<MemTracker>("InvertedIndexSearcherCache")) {
-    SCOPED_CONSUME_MEM_TRACKER(_mem_tracker.get());
+InvertedIndexSearcherCache::InvertedIndexSearcherCache(size_t capacity, uint32_t num_shards) {
     uint64_t fd_number = config::min_file_descriptor_number;
     struct rlimit l;
     int ret = getrlimit(RLIMIT_NOFILE, &l);
@@ -113,13 +109,11 @@ InvertedIndexSearcherCache::InvertedIndexSearcherCache(size_t capacity, uint32_t
             auto* cache_value = (InvertedIndexSearcherCache::CacheValue*)value;
             return cache_value->last_visit_time;
         };
-        _cache = std::unique_ptr<Cache>(
-                new ShardedLRUCache("InvertedIndexSearcherCache", capacity, LRUCacheType::SIZE,
-                                    num_shards, get_last_visit_time, true, open_searcher_limit));
+        _policy = std::make_unique<InvertedIndexSearcherCachePolicy>(
+                capacity, num_shards, open_searcher_limit, get_last_visit_time, true);
     } else {
-        _cache = std::unique_ptr<Cache>(new ShardedLRUCache("InvertedIndexSearcherCache", capacity,
-                                                            LRUCacheType::SIZE, num_shards,
-                                                            open_searcher_limit));
+        _policy = std::make_unique<InvertedIndexSearcherCachePolicy>(capacity, num_shards,
+                                                                     open_searcher_limit);
     }
 }
 
@@ -208,8 +202,8 @@ Status InvertedIndexSearcherCache::get_index_searcher(
         IndexCacheValuePtr cache_value = std::make_unique<InvertedIndexSearcherCache::CacheValue>();
         cache_value->index_searcher = std::move(index_searcher);
         cache_value->size = mem_tracker->consumption();
-        *cache_handle =
-                InvertedIndexCacheHandle(_cache.get(), _insert(cache_key, cache_value.release()));
+        *cache_handle = InvertedIndexCacheHandle(_policy->cache(),
+                                                 _insert(cache_key, cache_value.release()));
     } else {
         cache_handle->index_searcher = std::move(index_searcher);
     }
@@ -280,30 +274,27 @@ Status InvertedIndexSearcherCache::insert(const io::FileSystemSPtr& fs,
     cache_value->size = mem_tracker->consumption();
     cache_value->last_visit_time = UnixMillis();
     auto* lru_handle = _insert(cache_key, cache_value.release());
-    _cache->release(lru_handle);
+    _policy->cache()->release(lru_handle);
     return Status::OK();
 }
 
 Status InvertedIndexSearcherCache::erase(const std::string& index_file_path) {
     InvertedIndexSearcherCache::CacheKey cache_key(index_file_path);
-    _cache->erase(cache_key.index_file_path);
+    _policy->cache()->erase(cache_key.index_file_path);
     return Status::OK();
 }
 
 int64_t InvertedIndexSearcherCache::mem_consumption() {
-    if (_cache) {
-        return _cache->mem_consumption();
-    }
-    return 0L;
+    return _policy->cache()->mem_consumption();
 }
 
 bool InvertedIndexSearcherCache::_lookup(const InvertedIndexSearcherCache::CacheKey& key,
                                          InvertedIndexCacheHandle* handle) {
-    auto* lru_handle = _cache->lookup(key.index_file_path);
+    auto* lru_handle = _policy->cache()->lookup(key.index_file_path);
     if (lru_handle == nullptr) {
         return false;
     }
-    *handle = InvertedIndexCacheHandle(_cache.get(), lru_handle);
+    *handle = InvertedIndexCacheHandle(_policy->cache(), lru_handle);
     return true;
 }
 
@@ -314,8 +305,8 @@ Cache::Handle* InvertedIndexSearcherCache::_insert(const InvertedIndexSearcherCa
         delete cache_value;
     };
 
-    Cache::Handle* lru_handle =
-            _cache->insert(key.index_file_path, value, value->size, deleter, CachePriority::NORMAL);
+    Cache::Handle* lru_handle = _policy->cache()->insert(key.index_file_path, value, value->size,
+                                                         deleter, CachePriority::NORMAL);
     return lru_handle;
 }
 
@@ -323,11 +314,11 @@ bool InvertedIndexQueryCache::lookup(const CacheKey& key, InvertedIndexQueryCach
     if (key.encode().empty()) {
         return false;
     }
-    auto* lru_handle = _cache->lookup(key.encode());
+    auto* lru_handle = cache()->lookup(key.encode());
     if (lru_handle == nullptr) {
         return false;
     }
-    *handle = InvertedIndexQueryCacheHandle(_cache.get(), lru_handle);
+    *handle = InvertedIndexQueryCacheHandle(cache(), lru_handle);
     return true;
 }
 
@@ -346,16 +337,13 @@ void InvertedIndexQueryCache::insert(const CacheKey& key, std::shared_ptr<roarin
         return;
     }
 
-    auto* lru_handle = _cache->insert(key.encode(), (void*)cache_value_ptr.release(),
-                                      bitmap->getSizeInBytes(), deleter, CachePriority::NORMAL);
-    *handle = InvertedIndexQueryCacheHandle(_cache.get(), lru_handle);
+    auto* lru_handle = cache()->insert(key.encode(), (void*)cache_value_ptr.release(),
+                                       bitmap->getSizeInBytes(), deleter, CachePriority::NORMAL);
+    *handle = InvertedIndexQueryCacheHandle(cache(), lru_handle);
 }
 
 int64_t InvertedIndexQueryCache::mem_consumption() {
-    if (_cache) {
-        return _cache->mem_consumption();
-    }
-    return 0L;
+    return cache()->mem_consumption();
 }
 
 } // namespace doris::segment_v2
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_cache.h b/be/src/olap/rowset/segment_v2/inverted_index_cache.h
index 8d7d90eea2d96f..51791f0e10a667 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_cache.h
+++ b/be/src/olap/rowset/segment_v2/inverted_index_cache.h
@@ -94,7 +94,7 @@ class BKDIndexSearcherBuilder : public IndexSearcherBuilder {
                  OptionalIndexSearcherPtr& output_searcher) override;
 };
 
-class InvertedIndexSearcherCache : public LRUCachePolicy {
+class InvertedIndexSearcherCache {
 public:
     // The cache key of index_searcher lru cache
     struct CacheKey {
@@ -113,12 +113,6 @@ class InvertedIndexSearcherCache : public LRUCachePolicy {
     static InvertedIndexSearcherCache* create_global_instance(size_t capacity,
                                                               uint32_t num_shards = 16);
 
-    void reset() {
-        _cache.reset();
-        _mem_tracker.reset();
-        // Reset or clear the state of the object.
-    }
-
     // Return global instance.
     // Client should call create_global_cache before.
     static InvertedIndexSearcherCache* instance() {
@@ -139,11 +133,32 @@ class InvertedIndexSearcherCache : public LRUCachePolicy {
     // function `erase` called after compaction remove segment
     Status erase(const std::string& index_file_path);
 
+    void release(Cache::Handle* handle) { _policy->cache()->release(handle); }
+
     int64_t mem_consumption();
 
 private:
     InvertedIndexSearcherCache();
 
+    class InvertedIndexSearcherCachePolicy : public LRUCachePolicy {
+    public:
+        InvertedIndexSearcherCachePolicy(size_t capacity, uint32_t num_shards,
+                                         uint32_t element_count_capacity)
+                : LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
+                                 LRUCacheType::SIZE,
+                                 config::inverted_index_cache_stale_sweep_time_sec, num_shards,
+                                 element_count_capacity, true) {}
+        InvertedIndexSearcherCachePolicy(size_t capacity, uint32_t num_shards,
+                                         uint32_t element_count_capacity,
+                                         CacheValueTimeExtractor cache_value_time_extractor,
+                                         bool cache_value_check_timestamp)
+                : LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
+                                 LRUCacheType::SIZE,
+                                 config::inverted_index_cache_stale_sweep_time_sec, num_shards,
+                                 element_count_capacity, cache_value_time_extractor,
+                                 cache_value_check_timestamp, true) {}
+    };
+
     // Lookup the given index_searcher in the cache.
     // If the index_searcher is found, the cache entry will be written into handle.
     // Return true if entry is found, otherwise return false.
@@ -154,8 +169,7 @@ class InvertedIndexSearcherCache : public LRUCachePolicy {
     // This function is thread-safe.
     Cache::Handle* _insert(const InvertedIndexSearcherCache::CacheKey& key, CacheValue* value);
 
-private:
-    std::unique_ptr<MemTracker> _mem_tracker;
+    std::unique_ptr<InvertedIndexSearcherCachePolicy> _policy;
 };
 
 using IndexCacheValuePtr = std::unique_ptr<InvertedIndexSearcherCache::CacheValue>;
diff --git a/be/src/olap/rowset/segment_v2/page_io.cpp b/be/src/olap/rowset/segment_v2/page_io.cpp
index c712478850a6c5..dac9012abcd377 100644
--- a/be/src/olap/rowset/segment_v2/page_io.cpp
+++ b/be/src/olap/rowset/segment_v2/page_io.cpp
@@ -120,8 +120,7 @@ Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle*
     PageCacheHandle cache_handle;
     StoragePageCache::CacheKey cache_key(opts.file_reader->path().native(),
                                          opts.file_reader->size(), opts.page_pointer.offset);
-    if (opts.use_page_cache && cache->is_cache_available(opts.type) &&
-        cache->lookup(cache_key, &cache_handle, opts.type)) {
+    if (opts.use_page_cache && cache && cache->lookup(cache_key, &cache_handle, opts.type)) {
         // we find page in cache, use it
         *handle = PageHandle(std::move(cache_handle));
         opts.stats->cached_pages_num++;
@@ -218,7 +217,7 @@ Status PageIO::read_and_decompress_page(const PageReadOptions& opts, PageHandle*
 
     *body = Slice(page_slice.data, page_slice.size - 4 - footer_size);
     page->reset_size(page_slice.size);
-    if (opts.use_page_cache && cache->is_cache_available(opts.type)) {
+    if (opts.use_page_cache && cache) {
         // insert this page into cache and return the cache handle
         cache->insert(cache_key, page.get(), &cache_handle, opts.type, opts.kept_in_memory);
         *handle = PageHandle(std::move(cache_handle));
diff --git a/be/src/olap/schema_cache.h b/be/src/olap/schema_cache.h
index b7f640551d4b7b..58bf05bbfc87a0 100644
--- a/be/src/olap/schema_cache.h
+++ b/be/src/olap/schema_cache.h
@@ -65,10 +65,10 @@ class SchemaCache : public LRUCachePolicy {
         if (!instance() || schema_key.empty()) {
             return {};
         }
-        auto lru_handle = _cache->lookup(schema_key);
+        auto lru_handle = cache()->lookup(schema_key);
         if (lru_handle) {
-            Defer release([cache = _cache.get(), lru_handle] { cache->release(lru_handle); });
-            auto value = (CacheValue*)_cache->value(lru_handle);
+            Defer release([cache = cache(), lru_handle] { cache->release(lru_handle); });
+            auto value = (CacheValue*)cache()->value(lru_handle);
             value->last_visit_time = UnixMillis();
             VLOG_DEBUG << "use cache schema";
             if constexpr (std::is_same_v<SchemaType, TabletSchemaSPtr>) {
@@ -101,8 +101,8 @@ class SchemaCache : public LRUCachePolicy {
             delete cache_value;
         };
         auto lru_handle =
-                _cache->insert(key, value, 1, deleter, CachePriority::NORMAL, schema->mem_size());
-        _cache->release(lru_handle);
+                cache()->insert(key, value, 1, deleter, CachePriority::NORMAL, schema->mem_size());
+        cache()->release(lru_handle);
     }
 
     // Try to prune the cache if expired.
diff --git a/be/src/olap/segment_loader.cpp b/be/src/olap/segment_loader.cpp
index 02539a3f1690e1..0d068bfcbe6271 100644
--- a/be/src/olap/segment_loader.cpp
+++ b/be/src/olap/segment_loader.cpp
@@ -29,11 +29,11 @@ SegmentLoader* SegmentLoader::instance() {
 }
 
 bool SegmentCache::lookup(const SegmentCache::CacheKey& key, SegmentCacheHandle* handle) {
-    auto lru_handle = _cache->lookup(key.encode());
+    auto lru_handle = cache()->lookup(key.encode());
     if (lru_handle == nullptr) {
         return false;
     }
-    handle->push_segment(_cache.get(), lru_handle);
+    handle->push_segment(cache(), lru_handle);
     return true;
 }
 
@@ -45,13 +45,13 @@ void SegmentCache::insert(const SegmentCache::CacheKey& key, SegmentCache::Cache
         delete cache_value;
     };
 
-    auto lru_handle = _cache->insert(key.encode(), &value, 1, deleter, CachePriority::NORMAL,
-                                     value.segment->meta_mem_usage());
-    handle->push_segment(_cache.get(), lru_handle);
+    auto lru_handle = cache()->insert(key.encode(), &value, 1, deleter, CachePriority::NORMAL,
+                                      value.segment->meta_mem_usage());
+    handle->push_segment(cache(), lru_handle);
 }
 
 void SegmentCache::erase(const SegmentCache::CacheKey& key) {
-    _cache->erase(key.encode());
+    cache()->erase(key.encode());
 }
 
 Status SegmentLoader::load_segments(const BetaRowsetSharedPtr& rowset,
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index a67103a8939264..793bde7b081189 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1074,7 +1074,7 @@ std::shared_ptr<roaring::Roaring> DeleteBitmap::get_agg(const BitmapKey& bmk) co
             &val->bitmap, [this, handle](...) { _agg_cache->repr()->release(handle); });
 }
 
-std::atomic<ShardedLRUCache*> DeleteBitmap::AggCache::s_repr {nullptr};
+std::atomic<DeleteBitmap::AggCachePolicy*> DeleteBitmap::AggCache::s_repr {nullptr};
 
 std::string tablet_state_name(TabletState state) {
     switch (state) {
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 00886bb848b4fc..2d14ef599f3b10 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -46,6 +46,7 @@
 #include "olap/olap_common.h"
 #include "olap/rowset/rowset_meta.h"
 #include "olap/tablet_schema.h"
+#include "runtime/memory/lru_cache_policy.h"
 #include "util/uid_util.h"
 
 namespace json2pb {
@@ -469,6 +470,14 @@ class DeleteBitmap {
      */
     std::shared_ptr<roaring::Roaring> get_agg(const BitmapKey& bmk) const;
 
+    class AggCachePolicy : public LRUCachePolicy {
+    public:
+        AggCachePolicy(size_t capacity)
+                : LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity,
+                                 LRUCacheType::SIZE,
+                                 config::delete_bitmap_agg_cache_stale_sweep_time_sec, 256) {}
+    };
+
     class AggCache {
     public:
         struct Value {
@@ -478,8 +487,7 @@ class DeleteBitmap {
         AggCache(size_t size_in_bytes) {
             static std::once_flag once;
             std::call_once(once, [size_in_bytes] {
-                auto tmp = new ShardedLRUCache("DeleteBitmap AggCache", size_in_bytes,
-                                               LRUCacheType::SIZE, 256);
+                auto* tmp = new AggCachePolicy(size_in_bytes);
                 AggCache::s_repr.store(tmp, std::memory_order_release);
             });
 
@@ -487,8 +495,8 @@ class DeleteBitmap {
             }
         }
 
-        static ShardedLRUCache* repr() { return s_repr.load(std::memory_order_acquire); }
-        static std::atomic<ShardedLRUCache*> s_repr;
+        static Cache* repr() { return s_repr.load(std::memory_order_acquire)->cache(); }
+        static std::atomic<AggCachePolicy*> s_repr;
     };
 
 private:
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index 2b1e57925fd14f..90c277d8ad15e7 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -116,8 +116,7 @@ TxnManager::TxnManager(int32_t txn_map_shard_size, int32_t txn_shard_size)
     _txn_tablet_delta_writer_map = new txn_tablet_delta_writer_map_t[_txn_map_shard_size];
     _txn_tablet_delta_writer_map_locks = new std::shared_mutex[_txn_map_shard_size];
     // For debugging
-    _tablet_version_cache = std::unique_ptr<Cache>(
-            new ShardedLRUCache("TabletVersionCache", 100000, LRUCacheType::NUMBER, 32));
+    _tablet_version_cache = std::make_unique<TabletVersionCache>(100000);
 }
 
 // prepare txn should always be allowed because ingest task will be retried
@@ -889,12 +888,12 @@ int64_t TxnManager::get_txn_by_tablet_version(int64_t tablet_id, int64_t version
     memcpy(key + sizeof(int64_t), &version, sizeof(int64_t));
     CacheKey cache_key((const char*)&key, sizeof(key));
 
-    auto handle = _tablet_version_cache->lookup(cache_key);
+    auto* handle = _tablet_version_cache->cache()->lookup(cache_key);
     if (handle == nullptr) {
         return -1;
     }
-    int64_t res = *(int64_t*)_tablet_version_cache->value(handle);
-    _tablet_version_cache->release(handle);
+    int64_t res = *(int64_t*)_tablet_version_cache->cache()->value(handle);
+    _tablet_version_cache->cache()->release(handle);
     return res;
 }
 
@@ -911,9 +910,9 @@ void TxnManager::update_tablet_version_txn(int64_t tablet_id, int64_t version, i
         delete cache_value;
     };
 
-    auto handle = _tablet_version_cache->insert(cache_key, value, 1, deleter, CachePriority::NORMAL,
-                                                sizeof(txn_id));
-    _tablet_version_cache->release(handle);
+    auto* handle = _tablet_version_cache->cache()->insert(cache_key, value, 1, deleter,
+                                                          CachePriority::NORMAL, sizeof(txn_id));
+    _tablet_version_cache->cache()->release(handle);
 }
 
 TxnState TxnManager::get_txn_state(TPartitionId partition_id, TTransactionId transaction_id,
diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h
index 74ad589cba03b0..25b8da90b71972 100644
--- a/be/src/olap/txn_manager.h
+++ b/be/src/olap/txn_manager.h
@@ -44,6 +44,7 @@
 #include "olap/rowset/segment_v2/segment_writer.h"
 #include "olap/tablet.h"
 #include "olap/tablet_meta.h"
+#include "runtime/memory/lru_cache_policy.h"
 #include "util/time.h"
 #include "vec/core/block.h"
 
@@ -220,6 +221,14 @@ class TxnManager {
     void _insert_txn_partition_map_unlocked(int64_t transaction_id, int64_t partition_id);
     void _clear_txn_partition_map_unlocked(int64_t transaction_id, int64_t partition_id);
 
+    class TabletVersionCache : public LRUCachePolicy {
+    public:
+        TabletVersionCache(size_t capacity)
+                : LRUCachePolicy(CachePolicy::CacheType::TABLET_VERSION_CACHE, capacity,
+                                 LRUCacheType::NUMBER, -1, DEFAULT_LRU_CACHE_NUM_SHARDS,
+                                 DEFAULT_LRU_CACHE_ELEMENT_COUNT_CAPACITY, false) {}
+    };
+
 private:
     const int32_t _txn_map_shard_size;
 
@@ -238,7 +247,7 @@ class TxnManager {
     std::shared_mutex* _txn_mutex = nullptr;
 
     txn_tablet_delta_writer_map_t* _txn_tablet_delta_writer_map = nullptr;
-    std::unique_ptr<Cache> _tablet_version_cache;
+    std::unique_ptr<TabletVersionCache> _tablet_version_cache;
     std::shared_mutex* _txn_tablet_delta_writer_map_locks = nullptr;
     DISALLOW_COPY_AND_ASSIGN(TxnManager);
 }; // TxnManager
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index f8a52ad1ec402b..c97ff55a20647f 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -102,6 +102,7 @@ class StoragePageCache;
 class SegmentLoader;
 class LookupConnectionCache;
 class RowCache;
+class DummyLRUCache;
 class CacheManager;
 class WalManager;
 
@@ -232,6 +233,9 @@ class ExecEnv {
         this->_routine_load_task_executor = r;
     }
     void set_wal_mgr(std::shared_ptr<WalManager> wm) { this->_wal_manager = wm; }
+    void set_dummy_lru_cache(std::shared_ptr<DummyLRUCache> dummy_lru_cache) {
+        this->_dummy_lru_cache = dummy_lru_cache;
+    }
 
 #endif
     stream_load::LoadStreamStubPool* load_stream_stub_pool() {
@@ -261,6 +265,7 @@ class ExecEnv {
     segment_v2::InvertedIndexQueryCache* get_inverted_index_query_cache() {
         return _inverted_index_query_cache;
     }
+    std::shared_ptr<DummyLRUCache> get_dummy_lru_cache() { return _dummy_lru_cache; }
 
     std::shared_ptr<doris::pipeline::BlockedTaskScheduler> get_global_block_scheduler() {
         return _global_block_scheduler;
@@ -371,6 +376,7 @@ class ExecEnv {
     CacheManager* _cache_manager = nullptr;
     segment_v2::InvertedIndexSearcherCache* _inverted_index_searcher_cache = nullptr;
     segment_v2::InvertedIndexQueryCache* _inverted_index_query_cache = nullptr;
+    std::shared_ptr<DummyLRUCache> _dummy_lru_cache = nullptr;
 
     // used for query with group cpu hard limit
     std::shared_ptr<doris::pipeline::BlockedTaskScheduler> _global_block_scheduler;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index c204ead069d120..e612cb816b32a8 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -216,7 +216,6 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
     _small_file_mgr = new SmallFileMgr(this, config::small_file_dir);
     _block_spill_mgr = new BlockSpillManager(store_paths);
     _group_commit_mgr = new GroupCommitMgr(this);
-    _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num);
     _memtable_memory_limiter = std::make_unique<MemTableMemoryLimiter>();
     _load_stream_stub_pool = std::make_unique<stream_load::LoadStreamStubPool>();
     _delta_writer_v2_pool = std::make_unique<vectorized::DeltaWriterV2Pool>();
@@ -371,13 +370,13 @@ Status ExecEnv::_init_mem_env() {
     init_hook();
 #endif
 
-    // 2. init buffer pool
     if (!BitUtil::IsPowerOf2(config::min_buffer_size)) {
         ss << "Config min_buffer_size must be a power-of-two: " << config::min_buffer_size;
         return Status::InternalError(ss.str());
     }
 
-    // 3. init storage page cache
+    _dummy_lru_cache = std::make_shared<DummyLRUCache>();
+
     _cache_manager = CacheManager::create_global_instance();
 
     int64_t storage_cache_limit =
@@ -396,6 +395,12 @@ Status ExecEnv::_init_mem_env() {
                      << ". Please modify the 'storage_page_cache_shard_size' parameter in your "
                         "conf file to be a power of two for better performance.";
     }
+    if (storage_cache_limit < num_shards * 2) {
+        LOG(WARNING) << "storage_cache_limit(" << storage_cache_limit << ") less than num_shards("
+                     << num_shards
+                     << ") * 2, cache capacity will be 0, continuing to use "
+                        "cache will only have negative effects, will be disabled.";
+    }
     int64_t pk_storage_page_cache_limit =
             ParseUtil::parse_mem_spec(config::pk_storage_page_cache_limit, MemInfo::mem_limit(),
                                       MemInfo::physical_mem(), &is_percent);
@@ -442,6 +447,8 @@ Status ExecEnv::_init_mem_env() {
 
     _schema_cache = new SchemaCache(config::schema_cache_capacity);
 
+    _file_meta_cache = new FileMetaCache(config::max_external_file_meta_cache_num);
+
     _lookup_connection_cache = LookupConnectionCache::create_global_instance(
             config::lookup_connection_cache_bytes_limit);
 
@@ -473,7 +480,6 @@ Status ExecEnv::_init_mem_env() {
               << PrettyPrinter::print(inverted_index_cache_limit, TUnit::BYTES)
               << ", origin config value: " << config::inverted_index_query_cache_limit;
 
-    // 4. init other managers
     RETURN_IF_ERROR(_block_spill_mgr->init());
     return Status::OK();
 }
diff --git a/be/src/runtime/load_channel_mgr.cpp b/be/src/runtime/load_channel_mgr.cpp
index 2eb6812ac52a59..4d98276f0d2626 100644
--- a/be/src/runtime/load_channel_mgr.cpp
+++ b/be/src/runtime/load_channel_mgr.cpp
@@ -82,8 +82,7 @@ void LoadChannelMgr::stop() {
 }
 
 Status LoadChannelMgr::init(int64_t process_mem_limit) {
-    _last_success_channel =
-            std::unique_ptr<Cache>(new ShardedLRUCache("LastSuccessChannelCache", 1024));
+    _last_success_channels = std::make_unique<LastSuccessChannelCache>(1024);
     RETURN_IF_ERROR(_start_bg_worker());
     return Status::OK();
 }
@@ -124,10 +123,10 @@ Status LoadChannelMgr::_get_load_channel(std::shared_ptr<LoadChannel>& channel,
     std::lock_guard<std::mutex> l(_lock);
     auto it = _load_channels.find(load_id);
     if (it == _load_channels.end()) {
-        auto handle = _last_success_channel->lookup(load_id.to_string());
+        auto handle = _last_success_channels->cache()->lookup(load_id.to_string());
         // success only when eos be true
         if (handle != nullptr) {
-            _last_success_channel->release(handle);
+            _last_success_channels->cache()->release(handle);
             if (request.has_eos() && request.eos()) {
                 is_eof = true;
                 return Status::OK();
@@ -183,8 +182,9 @@ void LoadChannelMgr::_finish_load_channel(const UniqueId load_id) {
         if (_load_channels.find(load_id) != _load_channels.end()) {
             _load_channels.erase(load_id);
         }
-        auto handle = _last_success_channel->insert(load_id.to_string(), nullptr, 1, dummy_deleter);
-        _last_success_channel->release(handle);
+        auto handle = _last_success_channels->cache()->insert(load_id.to_string(), nullptr, 1,
+                                                              dummy_deleter);
+        _last_success_channels->cache()->release(handle);
     }
     VLOG_CRITICAL << "removed load channel " << load_id;
 }
diff --git a/be/src/runtime/load_channel_mgr.h b/be/src/runtime/load_channel_mgr.h
index 0aeec52f245ae2..94bd210f262557 100644
--- a/be/src/runtime/load_channel_mgr.h
+++ b/be/src/runtime/load_channel_mgr.h
@@ -33,6 +33,7 @@
 #include "olap/lru_cache.h"
 #include "olap/memtable_memory_limiter.h"
 #include "runtime/load_channel.h"
+#include "runtime/memory/lru_cache_policy.h"
 #include "runtime/memory/mem_tracker_limiter.h"
 #include "runtime/thread_context.h"
 #include "util/countdown_latch.h"
@@ -71,12 +72,20 @@ class LoadChannelMgr {
 
     Status _start_bg_worker();
 
+    class LastSuccessChannelCache : public LRUCachePolicy {
+    public:
+        LastSuccessChannelCache(size_t capacity)
+                : LRUCachePolicy(CachePolicy::CacheType::LAST_SUCCESS_CHANNEL_CACHE, capacity,
+                                 LRUCacheType::SIZE, -1, DEFAULT_LRU_CACHE_NUM_SHARDS,
+                                 DEFAULT_LRU_CACHE_ELEMENT_COUNT_CAPACITY, false) {}
+    };
+
 protected:
     // lock protect the load channel map
     std::mutex _lock;
     // load id -> load channel
     std::unordered_map<UniqueId, std::shared_ptr<LoadChannel>> _load_channels;
-    std::unique_ptr<Cache> _last_success_channel;
+    std::unique_ptr<LastSuccessChannelCache> _last_success_channels;
 
     MemTableMemoryLimiter* _memtable_memory_limiter = nullptr;
 
diff --git a/be/src/runtime/memory/cache_manager.cpp b/be/src/runtime/memory/cache_manager.cpp
index eada36836d5518..09107817086f72 100644
--- a/be/src/runtime/memory/cache_manager.cpp
+++ b/be/src/runtime/memory/cache_manager.cpp
@@ -27,6 +27,9 @@ int64_t CacheManager::for_each_cache_prune_stale_wrap(
     int64_t freed_size = 0;
     std::lock_guard<std::mutex> l(_caches_lock);
     for (auto cache_policy : _caches) {
+        if (!cache_policy->enable_prune()) {
+            continue;
+        }
         func(cache_policy);
         freed_size += cache_policy->profile()->get_counter("FreedMemory")->value();
         if (cache_policy->profile()->get_counter("FreedMemory")->value() != 0 && profile) {
diff --git a/be/src/runtime/memory/cache_policy.cpp b/be/src/runtime/memory/cache_policy.cpp
index e79beaffa806ad..99af4857038613 100644
--- a/be/src/runtime/memory/cache_policy.cpp
+++ b/be/src/runtime/memory/cache_policy.cpp
@@ -21,8 +21,8 @@
 
 namespace doris {
 
-CachePolicy::CachePolicy(CacheType type, uint32_t stale_sweep_time_s)
-        : _type(type), _stale_sweep_time_s(stale_sweep_time_s) {
+CachePolicy::CachePolicy(CacheType type, uint32_t stale_sweep_time_s, bool enable_prune)
+        : _type(type), _stale_sweep_time_s(stale_sweep_time_s), _enable_prune(enable_prune) {
     _it = CacheManager::instance()->register_cache(this);
     init_profile();
 }
diff --git a/be/src/runtime/memory/cache_policy.h b/be/src/runtime/memory/cache_policy.h
index e931b8f1317ac0..0a57f838bfa485 100644
--- a/be/src/runtime/memory/cache_policy.h
+++ b/be/src/runtime/memory/cache_policy.h
@@ -34,7 +34,13 @@ class CachePolicy {
         SEGMENT_CACHE = 4,
         INVERTEDINDEX_SEARCHER_CACHE = 5,
         INVERTEDINDEX_QUERY_CACHE = 6,
-        LOOKUP_CONNECTION_CACHE = 7
+        LOOKUP_CONNECTION_CACHE = 7,
+        POINT_QUERY_ROW_CACHE = 8,
+        DELETE_BITMAP_AGG_CACHE = 9,
+        TABLET_VERSION_CACHE = 10,
+        LAST_SUCCESS_CHANNEL_CACHE = 11,
+        COMMON_OBJ_LRU_CACHE = 12,
+        FOR_UT = 13
     };
 
     static std::string type_string(CacheType type) {
@@ -54,7 +60,19 @@ class CachePolicy {
         case CacheType::INVERTEDINDEX_QUERY_CACHE:
             return "InvertedIndexQueryCache";
         case CacheType::LOOKUP_CONNECTION_CACHE:
-            return "LookupConnectionCache";
+            return "PointQueryLookupConnectionCache";
+        case CacheType::POINT_QUERY_ROW_CACHE:
+            return "PointQueryRowCache";
+        case CacheType::DELETE_BITMAP_AGG_CACHE:
+            return "MowDeleteBitmapAggCache";
+        case CacheType::TABLET_VERSION_CACHE:
+            return "MowTabletVersionCache";
+        case CacheType::LAST_SUCCESS_CHANNEL_CACHE:
+            return "LastSuccessChannelCache";
+        case CacheType::COMMON_OBJ_LRU_CACHE:
+            return "CommonObjLRUCache";
+        case CacheType::FOR_UT:
+            return "ForUT";
         default:
             LOG(FATAL) << "not match type of cache policy :" << static_cast<int>(type);
         }
@@ -62,13 +80,14 @@ class CachePolicy {
         __builtin_unreachable();
     }
 
-    CachePolicy(CacheType type, uint32_t stale_sweep_time_s);
+    CachePolicy(CacheType type, uint32_t stale_sweep_time_s, bool enable_prune);
     virtual ~CachePolicy();
 
     virtual void prune_stale() = 0;
     virtual void prune_all(bool clear) = 0;
 
     CacheType type() { return _type; }
+    bool enable_prune() const { return _enable_prune; }
     RuntimeProfile* profile() { return _profile.get(); }
 
 protected:
@@ -94,6 +113,7 @@ class CachePolicy {
     RuntimeProfile::Counter* _cost_timer = nullptr;
 
     uint32_t _stale_sweep_time_s;
+    bool _enable_prune = true;
 };
 
 } // namespace doris
diff --git a/be/src/runtime/memory/lru_cache_policy.h b/be/src/runtime/memory/lru_cache_policy.h
index bd301cd47b5c29..717c90d9ef2a71 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -17,6 +17,8 @@
 
 #pragma once
 
+#include <fmt/format.h>
+
 #include "olap/lru_cache.h"
 #include "runtime/memory/cache_policy.h"
 #include "util/time.h"
@@ -34,29 +36,64 @@ struct LRUCacheValueBase {
 // Base of lru cache, allow prune stale entry and prune all entry.
 class LRUCachePolicy : public CachePolicy {
 public:
-    LRUCachePolicy(CacheType type, uint32_t stale_sweep_time_s)
-            : CachePolicy(type, stale_sweep_time_s) {};
     LRUCachePolicy(CacheType type, size_t capacity, LRUCacheType lru_cache_type,
-                   uint32_t stale_sweep_time_s, uint32_t num_shards = -1)
-            : CachePolicy(type, stale_sweep_time_s) {
-        _cache = num_shards == -1
-                         ? std::unique_ptr<Cache>(
-                                   new ShardedLRUCache(type_string(type), capacity, lru_cache_type))
-                         : std::unique_ptr<Cache>(new ShardedLRUCache(type_string(type), capacity,
-                                                                      lru_cache_type, num_shards));
+                   uint32_t stale_sweep_time_s, uint32_t num_shards = DEFAULT_LRU_CACHE_NUM_SHARDS,
+                   uint32_t element_count_capacity = DEFAULT_LRU_CACHE_ELEMENT_COUNT_CAPACITY,
+                   bool enable_prune = true)
+            : CachePolicy(type, stale_sweep_time_s, enable_prune) {
+        if (check_capacity(capacity, num_shards)) {
+            _cache = std::shared_ptr<ShardedLRUCache>(
+                    new ShardedLRUCache(type_string(type), capacity, lru_cache_type, num_shards,
+                                        element_count_capacity));
+        } else {
+            CHECK(ExecEnv::GetInstance()->get_dummy_lru_cache());
+            _cache = ExecEnv::GetInstance()->get_dummy_lru_cache();
+        }
+    }
+
+    LRUCachePolicy(CacheType type, size_t capacity, LRUCacheType lru_cache_type,
+                   uint32_t stale_sweep_time_s, uint32_t num_shards,
+                   uint32_t element_count_capacity,
+                   CacheValueTimeExtractor cache_value_time_extractor,
+                   bool cache_value_check_timestamp, bool enable_prune = true)
+            : CachePolicy(type, stale_sweep_time_s, enable_prune) {
+        if (check_capacity(capacity, num_shards)) {
+            _cache = std::shared_ptr<ShardedLRUCache>(
+                    new ShardedLRUCache(type_string(type), capacity, lru_cache_type, num_shards,
+                                        cache_value_time_extractor, cache_value_check_timestamp,
+                                        element_count_capacity));
+        } else {
+            CHECK(ExecEnv::GetInstance()->get_dummy_lru_cache());
+            _cache = ExecEnv::GetInstance()->get_dummy_lru_cache();
+        }
+    }
+
+    bool check_capacity(size_t capacity, uint32_t num_shards) {
+        if (capacity < num_shards) {
+            LOG(INFO) << fmt::format(
+                    "{} lru cache capacity({} B) less than num_shards({}), init failed, will be "
+                    "disabled.",
+                    type_string(type()), capacity, num_shards);
+            _enable_prune = false;
+            return false;
+        }
+        return true;
     }
 
     ~LRUCachePolicy() override = default;
 
     // Try to prune the cache if expired.
     void prune_stale() override {
+        if (_stale_sweep_time_s <= 0 && _cache == ExecEnv::GetInstance()->get_dummy_lru_cache()) {
+            return;
+        }
         if (_cache->mem_consumption() > CACHE_MIN_FREE_SIZE) {
             COUNTER_SET(_cost_timer, (int64_t)0);
             SCOPED_TIMER(_cost_timer);
             const int64_t curtime = UnixMillis();
             int64_t byte_size = 0L;
             auto pred = [this, curtime, &byte_size](const void* value) -> bool {
-                LRUCacheValueBase* cache_value = (LRUCacheValueBase*)value;
+                auto* cache_value = (LRUCacheValueBase*)value;
                 if ((cache_value->last_visit_time + _stale_sweep_time_s * 1000) < curtime) {
                     byte_size += cache_value->size;
                     return true;
@@ -76,6 +113,9 @@ class LRUCachePolicy : public CachePolicy {
     }
 
     void prune_all(bool clear) override {
+        if (_cache == ExecEnv::GetInstance()->get_dummy_lru_cache()) {
+            return;
+        }
         if ((clear && _cache->mem_consumption() != 0) ||
             _cache->mem_consumption() > CACHE_MIN_FREE_SIZE) {
             COUNTER_SET(_cost_timer, (int64_t)0);
@@ -91,10 +131,12 @@ class LRUCachePolicy : public CachePolicy {
         }
     }
 
-    Cache* get() { return _cache.get(); }
+    // if check_capacity failed, will return dummy lru cache,
+    // compatible with ShardedLRUCache usage, but will not actually cache.
+    Cache* cache() const { return _cache.get(); }
 
-protected:
-    std::unique_ptr<Cache> _cache;
+private:
+    std::shared_ptr<Cache> _cache;
 };
 
 } // namespace doris
diff --git a/be/src/service/point_query_executor.cpp b/be/src/service/point_query_executor.cpp
index 6a0000a3c4a491..d122a9d3cb738c 100644
--- a/be/src/service/point_query_executor.cpp
+++ b/be/src/service/point_query_executor.cpp
@@ -111,11 +111,10 @@ LookupConnectionCache* LookupConnectionCache::create_global_instance(size_t capa
     return res;
 }
 
-RowCache::RowCache(int64_t capacity, int num_shards) {
-    // Create Row Cache
-    _cache = std::unique_ptr<Cache>(
-            new ShardedLRUCache("RowCache", capacity, LRUCacheType::SIZE, num_shards));
-}
+RowCache::RowCache(int64_t capacity, int num_shards)
+        : LRUCachePolicy(CachePolicy::CacheType::POINT_QUERY_ROW_CACHE, capacity,
+                         LRUCacheType::SIZE, config::point_query_row_cache_stale_sweep_time_sec,
+                         num_shards) {}
 
 // Create global instance of this class
 RowCache* RowCache::create_global_cache(int64_t capacity, uint32_t num_shards) {
@@ -130,12 +129,12 @@ RowCache* RowCache::instance() {
 
 bool RowCache::lookup(const RowCacheKey& key, CacheHandle* handle) {
     const std::string& encoded_key = key.encode();
-    auto lru_handle = _cache->lookup(encoded_key);
+    auto lru_handle = cache()->lookup(encoded_key);
     if (!lru_handle) {
         // cache miss
         return false;
     }
-    *handle = CacheHandle(_cache.get(), lru_handle);
+    *handle = CacheHandle(cache(), lru_handle);
     return true;
 }
 
@@ -145,14 +144,14 @@ void RowCache::insert(const RowCacheKey& key, const Slice& value) {
     memcpy(cache_value, value.data, value.size);
     const std::string& encoded_key = key.encode();
     auto handle =
-            _cache->insert(encoded_key, cache_value, value.size, deleter, CachePriority::NORMAL);
+            cache()->insert(encoded_key, cache_value, value.size, deleter, CachePriority::NORMAL);
     // handle will released
-    auto tmp = CacheHandle {_cache.get(), handle};
+    auto tmp = CacheHandle {cache(), handle};
 }
 
 void RowCache::erase(const RowCacheKey& key) {
     const std::string& encoded_key = key.encode();
-    _cache->erase(encoded_key);
+    cache()->erase(encoded_key);
 }
 
 Status PointQueryExecutor::init(const PTabletKeyLookupRequest* request,
diff --git a/be/src/service/point_query_executor.h b/be/src/service/point_query_executor.h
index c951cc67462cc7..77fa8c69cdf4ea 100644
--- a/be/src/service/point_query_executor.h
+++ b/be/src/service/point_query_executor.h
@@ -109,7 +109,7 @@ class Reusable {
 };
 
 // RowCache is a LRU cache for row store
-class RowCache {
+class RowCache : public LRUCachePolicy {
 public:
     // The cache key for row lru cache
     struct RowCacheKey {
@@ -187,7 +187,6 @@ class RowCache {
 private:
     static constexpr uint32_t kDefaultNumShards = 128;
     RowCache(int64_t capacity, int num_shards = kDefaultNumShards);
-    std::unique_ptr<Cache> _cache;
 };
 
 // A cache used for prepare stmt.
@@ -204,7 +203,8 @@ class LookupConnectionCache : public LRUCachePolicy {
     friend class PointQueryExecutor;
     LookupConnectionCache(size_t capacity)
             : LRUCachePolicy(CachePolicy::CacheType::LOOKUP_CONNECTION_CACHE, capacity,
-                             LRUCacheType::SIZE, config::tablet_lookup_cache_clean_interval) {}
+                             LRUCacheType::SIZE, config::tablet_lookup_cache_stale_sweep_time_sec) {
+    }
 
     std::string encode_key(__int128_t cache_id) {
         fmt::memory_buffer buffer;
@@ -222,20 +222,20 @@ class LookupConnectionCache : public LRUCachePolicy {
             delete cache_value;
         };
         LOG(INFO) << "Add item mem size " << item->mem_size()
-                  << ", cache_capacity: " << _cache->get_total_capacity()
-                  << ", cache_usage: " << _cache->get_usage()
-                  << ", mem_consum: " << _cache->mem_consumption();
+                  << ", cache_capacity: " << cache()->get_total_capacity()
+                  << ", cache_usage: " << cache()->get_usage()
+                  << ", mem_consum: " << cache()->mem_consumption();
         auto lru_handle =
-                _cache->insert(key, value, item->mem_size(), deleter, CachePriority::NORMAL);
-        _cache->release(lru_handle);
+                cache()->insert(key, value, item->mem_size(), deleter, CachePriority::NORMAL);
+        cache()->release(lru_handle);
     }
 
     std::shared_ptr<Reusable> get(__int128_t cache_id) {
         std::string key = encode_key(cache_id);
-        auto lru_handle = _cache->lookup(key);
+        auto lru_handle = cache()->lookup(key);
         if (lru_handle) {
-            Defer release([cache = _cache.get(), lru_handle] { cache->release(lru_handle); });
-            auto value = (CacheValue*)_cache->value(lru_handle);
+            Defer release([cache = cache(), lru_handle] { cache->release(lru_handle); });
+            auto value = (CacheValue*)cache()->value(lru_handle);
             value->last_visit_time = UnixMillis();
             return value->item;
         }
diff --git a/be/src/util/obj_lru_cache.cpp b/be/src/util/obj_lru_cache.cpp
index b1f2b8025831e2..4b61b245f2381a 100644
--- a/be/src/util/obj_lru_cache.cpp
+++ b/be/src/util/obj_lru_cache.cpp
@@ -19,30 +19,29 @@
 
 namespace doris {
 
-ObjLRUCache::ObjLRUCache(int64_t capacity, uint32_t num_shards) {
+ObjLRUCache::ObjLRUCache(int64_t capacity, uint32_t num_shards)
+        : LRUCachePolicy(CachePolicy::CacheType::COMMON_OBJ_LRU_CACHE, capacity,
+                         LRUCacheType::NUMBER, config::common_obj_lru_cache_stale_sweep_time_sec,
+                         num_shards) {
     _enabled = (capacity > 0);
-    if (_enabled) {
-        _cache = std::unique_ptr<Cache>(
-                new ShardedLRUCache("ObjLRUCache", capacity, LRUCacheType::NUMBER, num_shards));
-    }
 }
 
 bool ObjLRUCache::lookup(const ObjKey& key, CacheHandle* handle) {
     if (!_enabled) {
         return false;
     }
-    auto lru_handle = _cache->lookup(key.key);
+    auto lru_handle = cache()->lookup(key.key);
     if (!lru_handle) {
         // cache miss
         return false;
     }
-    *handle = CacheHandle(_cache.get(), lru_handle);
+    *handle = CacheHandle(cache(), lru_handle);
     return true;
 }
 
 void ObjLRUCache::erase(const ObjKey& key) {
     if (_enabled) {
-        _cache->erase(key.key);
+        cache()->erase(key.key);
     }
 }
 
diff --git a/be/src/util/obj_lru_cache.h b/be/src/util/obj_lru_cache.h
index d280567650cc13..f43e7a1e8d27e3 100644
--- a/be/src/util/obj_lru_cache.h
+++ b/be/src/util/obj_lru_cache.h
@@ -18,14 +18,14 @@
 #pragma once
 
 #include "olap/lru_cache.h"
+#include "runtime/memory/lru_cache_policy.h"
 
 namespace doris {
 
 // A common object cache depends on an Sharded LRU Cache.
 // It has a certain capacity, which determin how many objects it can cache.
 // Caller must hold a CacheHandle instance when visiting the cached object.
-// TODO shouble add gc prune
-class ObjLRUCache {
+class ObjLRUCache : public LRUCachePolicy {
 public:
     struct ObjKey {
         ObjKey(const std::string& key_) : key(key_) {}
@@ -67,7 +67,7 @@ class ObjLRUCache {
         DISALLOW_COPY_AND_ASSIGN(CacheHandle);
     };
 
-    ObjLRUCache(int64_t capacity, uint32_t num_shards = kDefaultNumShards);
+    ObjLRUCache(int64_t capacity, uint32_t num_shards = DEFAULT_LRU_CACHE_NUM_SHARDS);
 
     bool lookup(const ObjKey& key, CacheHandle* handle);
 
@@ -85,9 +85,9 @@ class ObjLRUCache {
                 void (*deleter)(const CacheKey& key, void* value)) {
         if (_enabled) {
             const std::string& encoded_key = key.key;
-            auto handle = _cache->insert(encoded_key, (void*)value, 1, deleter,
-                                         CachePriority::NORMAL, sizeof(T));
-            *cache_handle = CacheHandle {_cache.get(), handle};
+            auto handle = cache()->insert(encoded_key, (void*)value, 1, deleter,
+                                          CachePriority::NORMAL, sizeof(T));
+            *cache_handle = CacheHandle {cache(), handle};
         } else {
             cache_handle = nullptr;
         }
@@ -96,8 +96,6 @@ class ObjLRUCache {
     void erase(const ObjKey& key);
 
 private:
-    static constexpr uint32_t kDefaultNumShards = 16;
-    std::unique_ptr<Cache> _cache;
     bool _enabled;
 };
 
diff --git a/be/test/olap/lru_cache_test.cpp b/be/test/olap/lru_cache_test.cpp
index 4f9c661e43d50a..60dbd0402b181f 100644
--- a/be/test/olap/lru_cache_test.cpp
+++ b/be/test/olap/lru_cache_test.cpp
@@ -25,6 +25,7 @@
 #include <vector>
 
 #include "gtest/gtest_pred_impl.h"
+#include "runtime/memory/lru_cache_policy.h"
 #include "runtime/memory/mem_tracker_limiter.h"
 #include "testutil/test_util.h"
 
@@ -72,25 +73,34 @@ class CacheTest : public testing::Test {
         _s_current->_deleted_values.push_back(DecodeValue(v));
     }
 
+    class CacheTestPolicy : public LRUCachePolicy {
+    public:
+        CacheTestPolicy(size_t capacity)
+                : LRUCachePolicy(CachePolicy::CacheType::FOR_UT, capacity, LRUCacheType::SIZE, -1) {
+        }
+    };
+
     // there is 16 shards in ShardedLRUCache
     // And the LRUHandle size is about 100B. So the cache size should big enough
     // to run the UT.
     static const int kCacheSize = 1000 * 16;
     std::vector<int> _deleted_keys;
     std::vector<int> _deleted_values;
-    Cache* _cache;
+    CacheTestPolicy* _cache;
 
-    CacheTest() : _cache(new ShardedLRUCache("test", kCacheSize)) { _s_current = this; }
+    CacheTest() : _cache(new CacheTestPolicy(kCacheSize)) { _s_current = this; }
 
     ~CacheTest() { delete _cache; }
 
+    Cache* cache() { return _cache->cache(); }
+
     int Lookup(int key) {
         std::string result;
-        Cache::Handle* handle = _cache->lookup(EncodeKey(&result, key));
-        const int r = (handle == nullptr) ? -1 : DecodeValue(_cache->value(handle));
+        Cache::Handle* handle = cache()->lookup(EncodeKey(&result, key));
+        const int r = (handle == nullptr) ? -1 : DecodeValue(cache()->value(handle));
 
         if (handle != nullptr) {
-            _cache->release(handle);
+            cache()->release(handle);
         }
 
         return r;
@@ -98,19 +108,19 @@ class CacheTest : public testing::Test {
 
     void Insert(int key, int value, int charge) {
         std::string result;
-        _cache->release(_cache->insert(EncodeKey(&result, key), EncodeValue(value), charge,
-                                       &CacheTest::Deleter));
+        cache()->release(cache()->insert(EncodeKey(&result, key), EncodeValue(value), charge,
+                                         &CacheTest::Deleter));
     }
 
     void InsertDurable(int key, int value, int charge) {
         std::string result;
-        _cache->release(_cache->insert(EncodeKey(&result, key), EncodeValue(value), charge,
-                                       &CacheTest::Deleter, CachePriority::DURABLE));
+        cache()->release(cache()->insert(EncodeKey(&result, key), EncodeValue(value), charge,
+                                         &CacheTest::Deleter, CachePriority::DURABLE));
     }
 
     void Erase(int key) {
         std::string result;
-        _cache->erase(EncodeKey(&result, key));
+        cache()->erase(EncodeKey(&result, key));
     }
 
     void SetUp() {}
@@ -164,16 +174,16 @@ TEST_F(CacheTest, Erase) {
 TEST_F(CacheTest, EntriesArePinned) {
     Insert(100, 101, 1);
     std::string result1;
-    Cache::Handle* h1 = _cache->lookup(EncodeKey(&result1, 100));
-    EXPECT_EQ(101, DecodeValue(_cache->value(h1)));
+    Cache::Handle* h1 = cache()->lookup(EncodeKey(&result1, 100));
+    EXPECT_EQ(101, DecodeValue(cache()->value(h1)));
 
     Insert(100, 102, 1);
     std::string result2;
-    Cache::Handle* h2 = _cache->lookup(EncodeKey(&result2, 100));
-    EXPECT_EQ(102, DecodeValue(_cache->value(h2)));
+    Cache::Handle* h2 = cache()->lookup(EncodeKey(&result2, 100));
+    EXPECT_EQ(102, DecodeValue(cache()->value(h2)));
     EXPECT_EQ(0, _deleted_keys.size());
 
-    _cache->release(h1);
+    cache()->release(h1);
     EXPECT_EQ(1, _deleted_keys.size());
     EXPECT_EQ(100, _deleted_keys[0]);
     EXPECT_EQ(101, _deleted_values[0]);
@@ -182,7 +192,7 @@ TEST_F(CacheTest, EntriesArePinned) {
     EXPECT_EQ(-1, Lookup(100));
     EXPECT_EQ(1, _deleted_keys.size());
 
-    _cache->release(h2);
+    cache()->release(h2);
     EXPECT_EQ(2, _deleted_keys.size());
     EXPECT_EQ(100, _deleted_keys[1]);
     EXPECT_EQ(102, _deleted_values[1]);
@@ -400,8 +410,8 @@ TEST_F(CacheTest, HeavyEntries) {
 }
 
 TEST_F(CacheTest, NewId) {
-    uint64_t a = _cache->new_id();
-    uint64_t b = _cache->new_id();
+    uint64_t a = cache()->new_id();
+    uint64_t b = cache()->new_id();
     EXPECT_NE(a, b);
 }
 
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_searcher_cache_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_searcher_cache_test.cpp
index a9e381b169d8b6..4ea9ac62a1d562 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_searcher_cache_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_searcher_cache_test.cpp
@@ -161,8 +161,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_usage) {
     cache_value_1->size = 200;
     //cache_value_1->index_searcher;
     cache_value_1->last_visit_time = 10;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_1, cache_value_1.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_1, cache_value_1.release()));
 
     // should evict {key_1, cache_value_1}
     std::string file_name_2 = "test_2.idx";
@@ -171,8 +170,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_usage) {
     cache_value_2->size = 800;
     //cache_value_2->index_searcher;
     cache_value_2->last_visit_time = 20;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_2, cache_value_2.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_2, cache_value_2.release()));
     {
         InvertedIndexCacheHandle cache_handle;
         // lookup key_1
@@ -188,8 +186,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_usage) {
     cache_value_3->size = 400;
     //cache_value_3->index_searcher;
     cache_value_3->last_visit_time = 30;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_3, cache_value_3.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_3, cache_value_3.release()));
     {
         InvertedIndexCacheHandle cache_handle;
         // lookup key_2
@@ -205,8 +202,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_usage) {
     cache_value_4->size = 100;
     //cache_value_4->index_searcher;
     cache_value_4->last_visit_time = 40;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_4, cache_value_4.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_4, cache_value_4.release()));
     {
         InvertedIndexCacheHandle cache_handle;
         // lookup key_3
@@ -228,8 +224,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_element_count_limit) {
     cache_value_1->size = 20;
     //cache_value_1->index_searcher;
     cache_value_1->last_visit_time = 10;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_1, cache_value_1.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_1, cache_value_1.release()));
 
     // no need evict
     std::string file_name_2 = "test_2.idx";
@@ -238,8 +233,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_element_count_limit) {
     cache_value_2->size = 50;
     //cache_value_2->index_searcher;
     cache_value_2->last_visit_time = 20;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_2, cache_value_2.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_2, cache_value_2.release()));
     {
         InvertedIndexCacheHandle cache_handle;
         // lookup key_1
@@ -259,8 +253,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_element_count_limit) {
     cache_value_3->size = 80;
     //cache_value_3->index_searcher;
     cache_value_3->last_visit_time = 30;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_3, cache_value_3.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_3, cache_value_3.release()));
     {
         InvertedIndexCacheHandle cache_handle;
         // lookup key_1
@@ -286,8 +279,7 @@ TEST_F(InvertedIndexSearcherCacheTest, evict_by_element_count_limit) {
     cache_value_4->size = 100;
     //cache_value_4->index_searcher;
     cache_value_4->last_visit_time = 40;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_4, cache_value_4.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_4, cache_value_4.release()));
     {
         InvertedIndexCacheHandle cache_handle;
         // lookup key_1
@@ -321,8 +313,7 @@ TEST_F(InvertedIndexSearcherCacheTest, remove_element_only_in_table) {
     cache_value_1->size = 200;
     //cache_value_1->index_searcher;
     cache_value_1->last_visit_time = 10;
-    index_searcher_cache->_cache->release(
-            index_searcher_cache->_insert(key_1, cache_value_1.release()));
+    index_searcher_cache->release(index_searcher_cache->_insert(key_1, cache_value_1.release()));
 
     std::string file_name_2 = "test_2.idx";
     InvertedIndexSearcherCache::CacheKey key_2(file_name_2);
@@ -338,7 +329,7 @@ TEST_F(InvertedIndexSearcherCacheTest, remove_element_only_in_table) {
         cache_value_2->size = 800;
         //cache_value_2->index_searcher;
         cache_value_2->last_visit_time = 20;
-        index_searcher_cache->_cache->release(
+        index_searcher_cache->release(
                 index_searcher_cache->_insert(key_2, cache_value_2.release()));
 
         // lookup key_2, key_2 has removed from table due to cache is full
diff --git a/be/test/testutil/run_all_tests.cpp b/be/test/testutil/run_all_tests.cpp
index c9ea1c552e724f..fe514d1a3871b1 100644
--- a/be/test/testutil/run_all_tests.cpp
+++ b/be/test/testutil/run_all_tests.cpp
@@ -46,6 +46,7 @@ int main(int argc, char** argv) {
     doris::ExecEnv::GetInstance()->set_tablet_schema_cache(
             doris::TabletSchemaCache::create_global_schema_cache());
     doris::ExecEnv::GetInstance()->get_tablet_schema_cache()->start();
+    doris::ExecEnv::GetInstance()->set_dummy_lru_cache(std::make_shared<doris::DummyLRUCache>());
     doris::ExecEnv::GetInstance()->set_storage_page_cache(
             doris::StoragePageCache::create_global_cache(1 << 30, 10, 0));
     doris::ExecEnv::GetInstance()->set_segment_loader(new doris::SegmentLoader(1000));

From 236276e9c147f47fffdba2d540a93e31394256aa Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Fri, 29 Dec 2023 16:27:48 +0800
Subject: [PATCH 170/299] [typo](docs) fix lakehouse doc 404 (#29299)

---
 docs/en/docs/lakehouse/multi-catalog/iceberg.md    | 2 +-
 docs/en/docs/lakehouse/multi-catalog/jdbc.md       | 2 +-
 docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md | 2 +-
 docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/en/docs/lakehouse/multi-catalog/iceberg.md b/docs/en/docs/lakehouse/multi-catalog/iceberg.md
index 07325114ca1d56..8386fe4792469b 100644
--- a/docs/en/docs/lakehouse/multi-catalog/iceberg.md
+++ b/docs/en/docs/lakehouse/multi-catalog/iceberg.md
@@ -214,4 +214,4 @@ You can use the `FOR TIME AS OF` and `FOR VERSION AS OF` statements to read hist
 
 `SELECT * FROM iceberg_tbl FOR VERSION AS OF 868895038966572;`
 
-In addition, you can use the [iceberg_meta](../../sql-manual/sql-functions/table-functions/iceberg_meta.md) table function to query the snapshot information of the specified table.
+In addition, you can use the [iceberg_meta](../../sql-manual/sql-functions/table-functions/iceberg-meta.md) table function to query the snapshot information of the specified table.
diff --git a/docs/en/docs/lakehouse/multi-catalog/jdbc.md b/docs/en/docs/lakehouse/multi-catalog/jdbc.md
index 971eb316eb49f7..11a02b049a600b 100644
--- a/docs/en/docs/lakehouse/multi-catalog/jdbc.md
+++ b/docs/en/docs/lakehouse/multi-catalog/jdbc.md
@@ -701,7 +701,7 @@ CREATE CATALOG jdbc_oceanbase PROPERTIES (
 ```
 
 :::tip
-When Doris connects to OceanBase, it will automatically recognize that OceanBase is in MySQL or Oracle mode. Hierarchical correspondence and type mapping refer to [MySQL](#MySQL) and [Oracle](#Oracle)
+When Doris connects to OceanBase, it will automatically recognize that OceanBase is in MySQL or Oracle mode. Hierarchical correspondence and type mapping refer to [MySQL](#mysql) and [Oracle](#oracle)
 :::
 
 ## JDBC Drivers
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
index 9bf61f6d9f7646..7e32fbd90333ab 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/iceberg.md
@@ -214,5 +214,5 @@ CREATE CATALOG iceberg PROPERTIES (
 
 `SELECT * FROM iceberg_tbl FOR VERSION AS OF 868895038966572;`
 
-另外，可以使用 [iceberg_meta](../../sql-manual/sql-functions/table-functions/iceberg_meta.md) 表函数查询指定表的 snapshot 信息。
+另外，可以使用 [iceberg_meta](../../sql-manual/sql-functions/table-functions/iceberg-meta.md) 表函数查询指定表的 snapshot 信息。
 
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
index 2c2e084ab7d082..e57acbe688f80c 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/jdbc.md
@@ -702,7 +702,7 @@ CREATE CATALOG jdbc_oceanbase PROPERTIES (
 ```
 
 :::tip
- Doris 在连接 OceanBase 时，会自动识别 OceanBase 处于 MySQL 或者 Oracle 模式，层级对应和类型映射参考 [MySQL](#MySQL) 与 [Oracle](#Oracle)
+ Doris 在连接 OceanBase 时，会自动识别 OceanBase 处于 MySQL 或者 Oracle 模式，层级对应和类型映射参考 [MySQL](#mysql) 与 [Oracle](#oracle)
 :::
 
 ## JDBC Driver 列表

From 1a2ec2fbd92cf1e211a626cda7cac61b0b1cc8dc Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Fri, 29 Dec 2023 13:58:04 +0530
Subject: [PATCH 171/299] [fix](meta)Fix  error code for unknown table #27751
 (#27976)

This fix will improve the error code for unknown table
exception cases.
---
 .../doris/alter/AlterCancelException.java      |  6 ++++++
 .../org/apache/doris/catalog/DatabaseIf.java   | 18 ++++++++++++------
 .../java/org/apache/doris/catalog/Table.java   | 13 +++++++++----
 .../doris/catalog/external/ExternalTable.java  | 13 +++++++++----
 .../org/apache/doris/common/ErrorReport.java   |  2 +-
 .../doris/datasource/InternalCatalog.java      |  2 +-
 6 files changed, 38 insertions(+), 16 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterCancelException.java b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterCancelException.java
index 2c22322240f980..b4170e41a863b0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterCancelException.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterCancelException.java
@@ -18,6 +18,7 @@
 package org.apache.doris.alter;
 
 import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
 
 /*
  * This exception will be thrown when the alter job(v2) being cancelled due to
@@ -30,4 +31,9 @@ public class AlterCancelException extends DdlException {
     public AlterCancelException(String msg) {
         super(msg);
     }
+
+    public AlterCancelException(String msg, ErrorCode mysqlErrorCode) {
+        super(msg);
+        setMysqlErrorCode(mysqlErrorCode);
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
index 985296c366fe5f..3ba295733740a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
@@ -151,11 +151,13 @@ default <E extends Exception> T getTableOrException(long tableId, Function<Long,
     }
 
     default T getTableOrMetaException(String tableName) throws MetaNotFoundException {
-        return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
+        return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t,
+                                                        ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     default T getTableOrMetaException(long tableId) throws MetaNotFoundException {
-        return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t));
+        return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t,
+                                                        ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     default T getTableOrMetaException(String tableName, TableIf.TableType tableType) throws MetaNotFoundException {
@@ -206,7 +208,8 @@ default T getTableOrMetaException(long tableId, List<TableIf.TableType> tableTyp
     }
 
     default T getTableOrDdlException(String tableName) throws DdlException {
-        return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+        return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t),
+                                                            ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     default T getTableOrDdlException(String tableName, TableIf.TableType tableType) throws DdlException {
@@ -220,7 +223,8 @@ default T getTableOrDdlException(String tableName, TableIf.TableType tableType)
     }
 
     default T getTableOrDdlException(long tableId) throws DdlException {
-        return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+        return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t),
+                                                            ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     default T getTableOrDdlException(long tableId, TableIf.TableType tableType) throws DdlException {
@@ -235,12 +239,14 @@ default T getTableOrDdlException(long tableId, TableIf.TableType tableType) thro
 
     default T getTableOrAnalysisException(String tableName) throws AnalysisException {
         return getTableOrException(tableName,
-                t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+                t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t),
+                                ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     default T getTableOrAnalysisException(long tableId) throws AnalysisException {
         return getTableOrException(tableId,
-                t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+                t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t),
+                                ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     default OlapTable getOlapTableOrDdlException(String tableName) throws DdlException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index f6481aea2d51f2..4abf490f90f440 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -22,6 +22,7 @@
 import org.apache.doris.catalog.constraint.Constraint;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
 import org.apache.doris.common.MetaNotFoundException;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
@@ -257,19 +258,23 @@ public <E extends Exception> void writeLockOrException(E e) throws E {
     }
 
     public void writeLockOrDdlException() throws DdlException {
-        writeLockOrException(new DdlException("unknown table, tableName=" + name));
+        writeLockOrException(new DdlException("unknown table, tableName=" + name,
+                                            ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     public void writeLockOrMetaException() throws MetaNotFoundException {
-        writeLockOrException(new MetaNotFoundException("unknown table, tableName=" + name));
+        writeLockOrException(new MetaNotFoundException("unknown table, tableName=" + name,
+                                            ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     public void writeLockOrAlterCancelException() throws AlterCancelException {
-        writeLockOrException(new AlterCancelException("unknown table, tableName=" + name));
+        writeLockOrException(new AlterCancelException("unknown table, tableName=" + name,
+                                            ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     public boolean tryWriteLockOrMetaException(long timeout, TimeUnit unit) throws MetaNotFoundException {
-        return tryWriteLockOrException(timeout, unit, new MetaNotFoundException("unknown table, tableName=" + name));
+        return tryWriteLockOrException(timeout, unit, new MetaNotFoundException("unknown table, tableName=" + name,
+                                                                ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     public <E extends Exception> boolean tryWriteLockOrException(long timeout, TimeUnit unit, E e) throws E {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
index 2e0cc8ac565044..49bf28589055db 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
@@ -24,6 +24,7 @@
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
 import org.apache.doris.common.MetaNotFoundException;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.io.Writable;
@@ -186,22 +187,26 @@ public <E extends Exception> void writeLockOrException(E e) throws E {
 
     @Override
     public void writeLockOrDdlException() throws DdlException {
-        writeLockOrException(new DdlException("unknown table, tableName=" + name));
+        writeLockOrException(new DdlException("unknown table, tableName=" + name,
+                                        ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     @Override
     public void writeLockOrMetaException() throws MetaNotFoundException {
-        writeLockOrException(new MetaNotFoundException("unknown table, tableName=" + name));
+        writeLockOrException(new MetaNotFoundException("unknown table, tableName=" + name,
+                                        ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     @Override
     public void writeLockOrAlterCancelException() throws AlterCancelException {
-        writeLockOrException(new AlterCancelException("unknown table, tableName=" + name));
+        writeLockOrException(new AlterCancelException("unknown table, tableName=" + name,
+                                        ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     @Override
     public boolean tryWriteLockOrMetaException(long timeout, TimeUnit unit) throws MetaNotFoundException {
-        return tryWriteLockOrException(timeout, unit, new MetaNotFoundException("unknown table, tableName=" + name));
+        return tryWriteLockOrException(timeout, unit, new MetaNotFoundException("unknown table, tableName=" + name,
+                                        ErrorCode.ERR_BAD_TABLE_ERROR));
     }
 
     @Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorReport.java b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorReport.java
index b92b9cbe54788a..35fa6d65141953 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/ErrorReport.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/ErrorReport.java
@@ -65,7 +65,7 @@ public static void reportDdlException(ErrorCode errorCode, Object... objs)
 
     public static void reportDdlException(String pattern, ErrorCode errorCode, Object... objs)
             throws DdlException {
-        throw new DdlException(reportCommon(pattern, errorCode, objs));
+        throw new DdlException(reportCommon(pattern, errorCode, objs), errorCode);
     }
 
     public static void report(ErrorCode errorCode, Object... objs) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 2e031faede2674..a3ced0b72fefdf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -912,7 +912,7 @@ public void dropTable(DropTableStmt stmt) throws DdlException {
             Env.getCurrentEnv().getAnalysisManager().removeTableStats(table.getId());
             Env.getCurrentEnv().getMtmvService().dropTable(table);
         } catch (UserException e) {
-            throw new DdlException(e.getMessage(), e);
+            throw new DdlException(e.getMessage(), e.getMysqlErrorCode());
         } finally {
             db.writeUnlock();
         }

From ada5cf4b2e1abf5081b233598705bffbdb653657 Mon Sep 17 00:00:00 2001
From: Siyang Tang <82279870+TangSiyang2001@users.noreply.github.com>
Date: Fri, 29 Dec 2023 17:00:02 +0800
Subject: [PATCH 172/299] [fix](chore)  fix session var memtable on sink for
 bulk load (#29102)

---
 .../main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
index 97abfb0db906ca..cbe293a8ab4145 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
@@ -154,6 +154,7 @@ private void executeOnce() throws Exception {
          * here we use exec_mem_limit to directly override the load_mem_limit property.
          */
         curCoordinator.setLoadMemLimit(execMemLimit);
+        curCoordinator.setMemTableOnSinkNode(enableMemTableOnSinkNode);
 
         long leftTimeMs = getLeftTimeMs();
         if (leftTimeMs <= 0) {

From 9c57483aa3da37f88c51daa3d93699c25c3eec11 Mon Sep 17 00:00:00 2001
From: Dongyang Li <hello_stephen@qq.com>
Date: Fri, 29 Dec 2023 17:01:31 +0800
Subject: [PATCH 173/299] [ci](trigger) only master run performance check
 (#29290)

Co-authored-by: stephen <hello-stephen@qq.com>
---
 .../workflows/comment-to-trigger-teamcity.yml | 23 ++++++++++++++-----
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/comment-to-trigger-teamcity.yml b/.github/workflows/comment-to-trigger-teamcity.yml
index 7634a8ead5b0c6..954054fe35404c 100644
--- a/.github/workflows/comment-to-trigger-teamcity.yml
+++ b/.github/workflows/comment-to-trigger-teamcity.yml
@@ -279,9 +279,20 @@ jobs:
       run: |
         source ./regression-test/pipeline/common/teamcity-utils.sh
         set -x
-        trigger_or_skip_build \
-          "${{ steps.changes.outputs.changed_performance }}" \
-          "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
-          "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
-          "performance" \
-          "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        if [[ "${{ steps.parse.outputs.TARGET_BRANCH }}" == "'master'" ]]; then
+          echo "PR target branch is master, need run performance"
+          trigger_or_skip_build \
+            "${{ steps.changes.outputs.changed_performance }}" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "performance" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        else
+          echo "PR target branch is not master, skip run performance"
+          trigger_or_skip_build \
+            "false" \
+            "${{ steps.parse.outputs.PULL_REQUEST_NUM }}" \
+            "${{ steps.parse.outputs.COMMIT_ID_FROM_TRIGGER }}" \
+            "performance" \
+            "${{ steps.parse.outputs.COMMENT_REPEAT_TIMES }}"
+        fi

From 7e0616ba7755538ba9f53a462773a2069df8dff3 Mon Sep 17 00:00:00 2001
From: yujun <yu.jun.reach@gmail.com>
Date: Fri, 29 Dec 2023 17:31:55 +0800
Subject: [PATCH 174/299] [fix](regression test) fix test_publish_one_succ
 (#28993)

---
 regression-test/suites/load/insert/test_publish_one_succ.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/regression-test/suites/load/insert/test_publish_one_succ.groovy b/regression-test/suites/load/insert/test_publish_one_succ.groovy
index 4e331b5c8d2515..22f78c64300222 100644
--- a/regression-test/suites/load/insert/test_publish_one_succ.groovy
+++ b/regression-test/suites/load/insert/test_publish_one_succ.groovy
@@ -26,7 +26,7 @@ suite('test_publish_one_succ') {
 
         sql 'SET GLOBAL insert_visible_timeout_ms = 1000'
         sql "ADMIN SET FRONTEND CONFIG ('publish_wait_time_second' = '1000000')"
-        sql 'CREATE TABLE tbl (k1 int, k2 int)'
+        sql 'CREATE TABLE tbl (k1 int, k2 int) DISTRIBUTED BY HASH(`k1`) BUCKETS 10'
         for (def i = 1; i <= 5; i++) {
             sql "INSERT INTO tbl VALUES (${i}, ${10 * i})"
         }

From 7604401b06d90df988a4c9c623cb606fc595dc00 Mon Sep 17 00:00:00 2001
From: zhangguoqiang <18372634969@163.com>
Date: Fri, 29 Dec 2023 17:59:20 +0800
Subject: [PATCH 175/299] [Enhance](regression)Do path creation ahead of time
 for case test_export_external_table (#28616)

Do path creation ahead of time for case test_export_external_table
---
 .../org/apache/doris/regression/suite/Suite.groovy     |  8 ++++++++
 .../export/test_export_external_table.groovy           | 10 ++++++++++
 2 files changed, 18 insertions(+)

diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index a299f0a01d49f0..942f44f8d8d278 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -575,6 +575,14 @@ class Suite implements GroovyInterceptable {
         Assert.assertEquals(0, code)
     }
 
+    void mkdirRemote(String username, String host, String path) {
+        String cmd = "ssh ${username}@${host} 'mkdir -p ${path}'"
+        logger.info("Execute: ${cmd}".toString())
+        Process process = cmd.execute()
+        def code = process.waitFor()
+        Assert.assertEquals(0, code)
+    }
+
     void sshExec(String username, String host, String cmd) {
         String command = "ssh ${username}@${host} '${cmd}'"
         def cmds = ["/bin/bash", "-c", command]
diff --git a/regression-test/suites/external_table_p0/export/test_export_external_table.groovy b/regression-test/suites/external_table_p0/export/test_export_external_table.groovy
index 48cfb9e83e8abc..69b29b14e24b5c 100644
--- a/regression-test/suites/external_table_p0/export/test_export_external_table.groovy
+++ b/regression-test/suites/external_table_p0/export/test_export_external_table.groovy
@@ -55,12 +55,22 @@ suite("test_export_external_table", "p0,external,mysql,external_docker,external_
     }
 
     def check_path_exists = { dir_path ->
+        List<List<Object>> backends =  sql """ show backends """
+        assertTrue(backends.size() > 0)
         File path = new File(dir_path)
         if (!path.exists()) {
             assert path.mkdirs()
         } else {
             throw new IllegalStateException("""${dir_path} already exists! """)
         }
+        if (backends.size() > 1) {
+            for (List<Object> backend : backends) {
+                def be_host = backend[1]
+                def cmd="""mkdir -p ${dir_path}"""
+                sshExec("root", be_host, cmd.toString())
+            }
+        }
+
     }
     def check_file_amounts = { dir_path, amount ->
         File path = new File(dir_path)

From 4266432d11160c66d52c2e2b4c67b824556dc831 Mon Sep 17 00:00:00 2001
From: Xiangyu Wang <dut.xiangyu@gmail.com>
Date: Fri, 29 Dec 2023 18:02:57 +0800
Subject: [PATCH 176/299] [Fix](dialect) Fix trino dialect converter when sql
 does not end with delimiter. (#29106)

Co-authored-by: wangxiangyu <wangxiangyu@360shuke.com>
---
 .../nereids/parser/trino/TrinoParser.java     | 20 +++++++++++++++----
 .../nereids/parser/NereidsParserTest.java     | 14 +++++++++++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java
index 671af5e7f05dfb..82d9b9e93a6d43 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoParser.java
@@ -42,9 +42,8 @@ public class TrinoParser {
 
     public static final Logger LOG = LogManager.getLogger(TrinoParser.class);
 
-    private static final io.trino.sql.parser.ParsingOptions PARSING_OPTIONS =
-            new io.trino.sql.parser.ParsingOptions(
-                    io.trino.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL);
+    private static final io.trino.sql.parser.ParsingOptions PARSING_OPTIONS = new io.trino.sql.parser.ParsingOptions(
+                io.trino.sql.parser.ParsingOptions.DecimalLiteralTreatment.AS_DECIMAL);
 
     /**
      * Parse with trino syntax, return null if parse failed
@@ -52,7 +51,8 @@ public class TrinoParser {
     public static @Nullable List<StatementBase> parse(String sql, SessionVariable sessionVariable) {
         final List<StatementBase> logicalPlans = new ArrayList<>();
         try {
-            io.trino.sql.parser.StatementSplitter splitter = new io.trino.sql.parser.StatementSplitter(sql);
+            io.trino.sql.parser.StatementSplitter splitter = new io.trino.sql.parser.StatementSplitter(
+                        addDelimiterIfNeeded(sql));
             ParserContext parserContext = new ParserContext(ParseDialect.TRINO_395);
             StatementContext statementContext = new StatementContext();
             for (io.trino.sql.parser.StatementSplitter.Statement statement : splitter.getCompleteStatements()) {
@@ -87,4 +87,16 @@ public static <T> T parseSingle(String sql, ParserContext parserContext) {
         io.trino.sql.tree.Statement statement = TrinoParser.parse(sql);
         return (T) new TrinoLogicalPlanBuilder().visit(statement, parserContext);
     }
+
+    /**
+     * {@link io.trino.sql.parser.StatementSplitter} use ";" as the delimiter if not set
+     * So add ";" if sql does not end with ";",
+     * otherwise {@link io.trino.sql.parser.StatementSplitter#getCompleteStatements()} will return empty list
+     */
+    private static String addDelimiterIfNeeded(String sql) {
+        if (!sql.trim().endsWith(";")) {
+            return sql + ";";
+        }
+        return sql;
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index 9baee503cfc491..dc7e336433f245 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -206,6 +206,20 @@ public void testParseSQLWithTrinoDialect() {
         Assertions.assertTrue(logicalPlan1 instanceof ExplainCommand);
     }
 
+    @Test
+    public void testParseSingleStmtWithTrinoDialect() {
+        String sql = "select `AD``D` from t1 where a = 1";
+        NereidsParser nereidsParser = new NereidsParser();
+        SessionVariable sessionVariable = new SessionVariable();
+        sessionVariable.setSqlDialect("trino");
+        // test fall back to doris parser
+        List<StatementBase> statementBases = nereidsParser.parseSQL(sql, sessionVariable);
+        Assertions.assertEquals(1, statementBases.size());
+        Assertions.assertTrue(statementBases.get(0) instanceof LogicalPlanAdapter);
+        LogicalPlan logicalPlan0 = ((LogicalPlanAdapter) statementBases.get(0)).getLogicalPlan();
+        Assertions.assertTrue(logicalPlan0 instanceof UnboundResultSink);
+    }
+
     @Test
     public void testParseSQLWithSparkSqlDialect() {
         // doris parser will throw a ParseException when derived table does not have alias

From f0629da10a674c04f819a60338d279db704ff5dd Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Fri, 29 Dec 2023 18:35:41 +0800
Subject: [PATCH 177/299] [pipelineX](fix) Fix illegal memory access (#29283)

---
 be/src/pipeline/exec/aggregation_source_operator.cpp |  2 +-
 be/src/pipeline/exec/data_queue.h                    | 10 ++++------
 be/src/pipeline/exec/union_source_operator.cpp       |  2 +-
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/be/src/pipeline/exec/aggregation_source_operator.cpp b/be/src/pipeline/exec/aggregation_source_operator.cpp
index e80f1e3d5e8301..dc476cf63a0634 100644
--- a/be/src/pipeline/exec/aggregation_source_operator.cpp
+++ b/be/src/pipeline/exec/aggregation_source_operator.cpp
@@ -50,7 +50,7 @@ Status AggLocalState::init(RuntimeState* state, LocalStateInfo& info) {
     _hash_table_size_counter = ADD_COUNTER(profile(), "HashTableSize", TUnit::UNIT);
     auto& p = _parent->template cast<AggSourceOperatorX>();
     if (p._is_streaming) {
-        _shared_state->data_queue->set_source_dependency(_dependency);
+        _shared_state->data_queue->set_source_dependency(info.dependency);
     }
     if (p._without_key) {
         if (p._needs_finalize) {
diff --git a/be/src/pipeline/exec/data_queue.h b/be/src/pipeline/exec/data_queue.h
index f0d3511a86f8fd..4eaa768fb8c97b 100644
--- a/be/src/pipeline/exec/data_queue.h
+++ b/be/src/pipeline/exec/data_queue.h
@@ -28,8 +28,7 @@
 #include "util/spinlock.h"
 #include "vec/core/block.h"
 
-namespace doris {
-namespace pipeline {
+namespace doris::pipeline {
 
 class Dependency;
 
@@ -61,7 +60,7 @@ class DataQueue {
     int64_t max_size_of_queue() const { return _max_size_of_queue; }
 
     bool data_exhausted() const { return _data_exhausted; }
-    void set_source_dependency(Dependency* source_dependency) {
+    void set_source_dependency(std::shared_ptr<Dependency> source_dependency) {
         _source_dependency = source_dependency;
     }
     void set_sink_dependency(Dependency* sink_dependency, int child_idx) {
@@ -105,10 +104,9 @@ class DataQueue {
     static constexpr int64_t MAX_BYTE_OF_QUEUE = 1024l * 1024 * 1024 / 10;
 
     // data queue is multi sink one source
-    Dependency* _source_dependency = nullptr;
+    std::shared_ptr<Dependency> _source_dependency = nullptr;
     std::vector<Dependency*> _sink_dependencies;
     SpinLock _source_lock;
 };
 
-} // namespace pipeline
-} // namespace doris
+} // namespace doris::pipeline
diff --git a/be/src/pipeline/exec/union_source_operator.cpp b/be/src/pipeline/exec/union_source_operator.cpp
index c9deae0d03767c..b42f941d60804f 100644
--- a/be/src/pipeline/exec/union_source_operator.cpp
+++ b/be/src/pipeline/exec/union_source_operator.cpp
@@ -120,7 +120,7 @@ Status UnionSourceLocalState::init(RuntimeState* state, LocalStateInfo& info) {
         ((UnionSourceDependency*)deps.front().get())->set_shared_state(ss);
     }
     RETURN_IF_ERROR(Base::init(state, info));
-    ss->data_queue.set_source_dependency(_dependency);
+    ss->data_queue.set_source_dependency(info.dependency);
     SCOPED_TIMER(exec_time_counter());
     SCOPED_TIMER(_open_timer);
     // Const exprs materialized by this node. These exprs don't refer to any children.

From 7c5fda11694af93cbb7bae3d0f16dbc8c8335ba1 Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@hotmail.com>
Date: Fri, 29 Dec 2023 19:22:07 +0800
Subject: [PATCH 178/299] [thirdparty](lib) Add streamvbyte thirdparty lib
 (#29308)

---
 thirdparty/build-thirdparty.sh | 15 +++++++++++++++
 thirdparty/vars.sh             |  7 +++++++
 2 files changed, 22 insertions(+)

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 43dbe33bd4ab1e..65224886d7b8a1 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -1720,6 +1720,20 @@ build_libdeflate() {
     "${BUILD_SYSTEM}" install
 }
 
+# streamvbyte
+build_streamvbyte() {
+    check_if_source_exist "${STREAMVBYTE_SOURCE}"
+    cd "${TP_SOURCE_DIR}/${STREAMVBYTE_SOURCE}"
+
+    rm -rf "${BUILD_DIR}"
+    mkdir -p "${BUILD_DIR}"
+    cd "${BUILD_DIR}"
+
+    "${CMAKE_CMD}" -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" -DCMAKE_BUILD_TYPE=Release ..
+    "${BUILD_SYSTEM}" -j "${PARALLEL}"
+    "${BUILD_SYSTEM}" install
+}
+
 if [[ "${#packages[@]}" -eq 0 ]]; then
     packages=(
         libunixodbc
@@ -1784,6 +1798,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
         dragonbox
         avx2neon
         libdeflate
+        streamvbyte
     )
     if [[ "$(uname -s)" == 'Darwin' ]]; then
         read -r -a packages <<<"binutils gettext ${packages[*]}"
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 5581204d914b15..7e2d5bf9dc680d 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -484,6 +484,12 @@ LIBDEFLATE_NAME=libdeflate-1.19.tar.gz
 LIBDEFLATE_SOURCE=libdeflate-1.19
 LIBDEFLATE_MD5SUM="c69e9193d2975a729068ffa862c81fb6"
 
+# streamvbyte
+STREAMVBYTE_DOWNLOAD="https://github.com/lemire/streamvbyte/archive/refs/tags/v1.0.0.tar.gz"
+STREAMVBYTE_NAME=streamvbyte-1.0.0.tar.gz
+STREAMVBYTE_SOURCE=streamvbyte-1.0.0
+STREAMVBYTE_MD5SUM="f334219db5a832b6dae3589a56a29563"
+
 # all thirdparties which need to be downloaded is set in array TP_ARCHIVES
 export TP_ARCHIVES=(
     'LIBEVENT'
@@ -555,6 +561,7 @@ export TP_ARCHIVES=(
     'DRAGONBOX'
     'AVX2NEON'
     'LIBDEFLATE'
+    'STREAMVBYTE'
 )
 
 if [[ "$(uname -s)" == 'Darwin' ]]; then

From 07bd65abe085a2f644f1c3533e03a851b36d2505 Mon Sep 17 00:00:00 2001
From: Mryange <59914473+Mryange@users.noreply.github.com>
Date: Fri, 29 Dec 2023 20:30:28 +0800
Subject: [PATCH 179/299] [pipelineX](local shuffle) remove unused code in
 local shuffle to improve performance #29292

---
 .../local_exchange/local_exchanger.cpp        | 141 ++++--------------
 .../local_exchange/local_exchanger.h          |  15 +-
 2 files changed, 31 insertions(+), 125 deletions(-)

diff --git a/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp b/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp
index fbe5098f6391eb..b2f1bb00ebeb2c 100644
--- a/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp
+++ b/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp
@@ -283,38 +283,12 @@ Status AdaptivePassthroughExchanger::_passthrough_sink(RuntimeState* state,
     new_block.swap(*in_block);
     auto channel_id = (local_state._channel_id++) % _num_partitions;
     local_state._shared_state->add_mem_usage(channel_id, new_block.allocated_bytes());
-    _passthrough_data_queue[channel_id].enqueue(std::move(new_block));
+    _data_queue[channel_id].enqueue(std::move(new_block));
     local_state._shared_state->set_ready_to_read(channel_id);
 
     return Status::OK();
 }
 
-bool AdaptivePassthroughExchanger::_passthrough_get_block(
-        RuntimeState* state, vectorized::Block* block, SourceState& source_state,
-        LocalExchangeSourceLocalState& local_state) {
-    vectorized::Block next_block;
-    if (_running_sink_operators == 0) {
-        if (_passthrough_data_queue[local_state._channel_id].try_dequeue(next_block)) {
-            block->swap(next_block);
-            _free_blocks.enqueue(std::move(next_block));
-            local_state._shared_state->sub_mem_usage(local_state._channel_id,
-                                                     block->allocated_bytes());
-        } else {
-            COUNTER_UPDATE(local_state._get_block_failed_counter, 1);
-            source_state = SourceState::FINISHED;
-            return false;
-        }
-    } else if (_passthrough_data_queue[local_state._channel_id].try_dequeue(next_block)) {
-        block->swap(next_block);
-        _free_blocks.enqueue(std::move(next_block));
-        local_state._shared_state->sub_mem_usage(local_state._channel_id, block->allocated_bytes());
-    } else {
-        COUNTER_UPDATE(local_state._get_block_failed_counter, 1);
-        return false;
-    }
-    return true;
-}
-
 Status AdaptivePassthroughExchanger::_shuffle_sink(RuntimeState* state, vectorized::Block* block,
                                                    SourceState source_state,
                                                    LocalExchangeSinkLocalState& local_state) {
@@ -339,7 +313,7 @@ Status AdaptivePassthroughExchanger::_split_rows(RuntimeState* state,
                                                  const uint32_t* __restrict channel_ids,
                                                  vectorized::Block* block, SourceState source_state,
                                                  LocalExchangeSinkLocalState& local_state) {
-    auto& data_queue = _shuffle_data_queue;
+    auto& data_queue = _data_queue;
     const auto rows = block->rows();
     auto row_idx = std::make_shared<std::vector<uint32_t>>(rows);
     {
@@ -357,79 +331,21 @@ Status AdaptivePassthroughExchanger::_split_rows(RuntimeState* state,
             local_state._partition_rows_histogram[channel_ids[i]]--;
         }
     }
-
-    vectorized::Block data_block;
-    std::shared_ptr<ShuffleBlockWrapper> new_block_wrapper;
-    if (_free_blocks.try_enqueue(data_block)) {
-        new_block_wrapper = ShuffleBlockWrapper::create_shared(std::move(data_block));
-    } else {
-        new_block_wrapper = ShuffleBlockWrapper::create_shared(block->clone_empty());
-    }
-
-    new_block_wrapper->data_block.swap(*block);
-    if (new_block_wrapper->data_block.empty()) {
-        return Status::OK();
-    }
-    local_state._shared_state->add_total_mem_usage(new_block_wrapper->data_block.allocated_bytes());
-    new_block_wrapper->ref(_num_partitions);
-
     for (size_t i = 0; i < _num_partitions; i++) {
-        size_t start = local_state._partition_rows_histogram[i];
-        size_t size = local_state._partition_rows_histogram[i + 1] - start;
+        const size_t start = local_state._partition_rows_histogram[i];
+        const size_t size = local_state._partition_rows_histogram[i + 1] - start;
         if (size > 0) {
-            local_state._shared_state->add_mem_usage(
-                    i, new_block_wrapper->data_block.allocated_bytes(), false);
-            data_queue[i].enqueue({new_block_wrapper, {row_idx, start, size}});
-            local_state._shared_state->set_ready_to_read(i);
-        } else {
-            new_block_wrapper->unref(local_state._shared_state);
+            std::unique_ptr<vectorized::MutableBlock> mutable_block =
+                    vectorized::MutableBlock::create_unique(block->clone_empty());
+            mutable_block->add_rows(block, start, size);
+            auto new_block = mutable_block->to_block();
+            local_state._shared_state->add_mem_usage(i, new_block.allocated_bytes());
+            data_queue[i].enqueue(std::move(new_block));
         }
+        local_state._shared_state->set_ready_to_read(i);
     }
-
     return Status::OK();
 }
-bool AdaptivePassthroughExchanger::_shuffle_get_block(RuntimeState* state, vectorized::Block* block,
-                                                      SourceState& source_state,
-                                                      LocalExchangeSourceLocalState& local_state) {
-    PartitionedBlock partitioned_block;
-    std::unique_ptr<vectorized::MutableBlock> mutable_block = nullptr;
-
-    auto get_data = [&](vectorized::Block* result_block) {
-        do {
-            const auto* offset_start = &((
-                    *std::get<0>(partitioned_block.second))[std::get<1>(partitioned_block.second)]);
-            auto block_wrapper = partitioned_block.first;
-            local_state._shared_state->sub_mem_usage(
-                    local_state._channel_id, block_wrapper->data_block.allocated_bytes(), false);
-            mutable_block->add_rows(&block_wrapper->data_block, offset_start,
-                                    offset_start + std::get<2>(partitioned_block.second));
-            block_wrapper->unref(local_state._shared_state);
-        } while (mutable_block->rows() < state->batch_size() &&
-                 _shuffle_data_queue[local_state._channel_id].try_dequeue(partitioned_block));
-        *result_block = mutable_block->to_block();
-    };
-    if (_running_sink_operators == 0) {
-        if (_shuffle_data_queue[local_state._channel_id].try_dequeue(partitioned_block)) {
-            SCOPED_TIMER(local_state._copy_data_timer);
-            mutable_block = vectorized::MutableBlock::create_unique(
-                    partitioned_block.first->data_block.clone_empty());
-            get_data(block);
-        } else {
-            COUNTER_UPDATE(local_state._get_block_failed_counter, 1);
-            source_state = SourceState::FINISHED;
-            return false;
-        }
-    } else if (_shuffle_data_queue[local_state._channel_id].try_dequeue(partitioned_block)) {
-        SCOPED_TIMER(local_state._copy_data_timer);
-        mutable_block = vectorized::MutableBlock::create_unique(
-                partitioned_block.first->data_block.clone_empty());
-        get_data(block);
-    } else {
-        COUNTER_UPDATE(local_state._get_block_failed_counter, 1);
-        return false;
-    }
-    return true;
-}
 
 Status AdaptivePassthroughExchanger::sink(RuntimeState* state, vectorized::Block* in_block,
                                           SourceState source_state,
@@ -447,23 +363,24 @@ Status AdaptivePassthroughExchanger::sink(RuntimeState* state, vectorized::Block
 Status AdaptivePassthroughExchanger::get_block(RuntimeState* state, vectorized::Block* block,
                                                SourceState& source_state,
                                                LocalExchangeSourceLocalState& local_state) {
-    auto is_shuffle_success = false, is_passthrough_success = false;
-    SourceState shuffle_state = SourceState::MORE_DATA, passthrough_state = SourceState::MORE_DATA;
-
-    is_shuffle_success = _shuffle_get_block(state, block, shuffle_state, local_state);
-
-    if (is_shuffle_success) {
-        return Status::OK();
-    }
-
-    is_passthrough_success = _passthrough_get_block(state, block, passthrough_state, local_state);
-
-    if (is_passthrough_success) {
-        return Status::OK();
-    }
-
-    if (shuffle_state == SourceState::FINISHED && passthrough_state == SourceState::FINISHED) {
-        source_state = SourceState::FINISHED;
+    vectorized::Block next_block;
+    if (_running_sink_operators == 0) {
+        if (_data_queue[local_state._channel_id].try_dequeue(next_block)) {
+            block->swap(next_block);
+            _free_blocks.enqueue(std::move(next_block));
+            local_state._shared_state->sub_mem_usage(local_state._channel_id,
+                                                     block->allocated_bytes());
+        } else {
+            COUNTER_UPDATE(local_state._get_block_failed_counter, 1);
+            source_state = SourceState::FINISHED;
+        }
+    } else if (_data_queue[local_state._channel_id].try_dequeue(next_block)) {
+        block->swap(next_block);
+        _free_blocks.enqueue(std::move(next_block));
+        local_state._shared_state->sub_mem_usage(local_state._channel_id, block->allocated_bytes());
+    } else {
+        COUNTER_UPDATE(local_state._get_block_failed_counter, 1);
+        local_state._dependency->block();
     }
     return Status::OK();
 }
diff --git a/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.h b/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.h
index ff64c88d5aafe1..28c99e751ea98b 100644
--- a/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.h
+++ b/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.h
@@ -183,14 +183,10 @@ class BroadcastExchanger final : public Exchanger {
 // a copy of ShuffleExchanger and PassthroughExchanger.
 class AdaptivePassthroughExchanger : public Exchanger {
 public:
-    using PartitionedBlock =
-            std::pair<std::shared_ptr<ShuffleBlockWrapper>,
-                      std::tuple<std::shared_ptr<std::vector<uint32_t>>, size_t, size_t>>;
     ENABLE_FACTORY_CREATOR(AdaptivePassthroughExchanger);
     AdaptivePassthroughExchanger(int running_sink_operators, int num_partitions)
             : Exchanger(running_sink_operators, num_partitions) {
-        _passthrough_data_queue.resize(num_partitions);
-        _shuffle_data_queue.resize(num_partitions);
+        _data_queue.resize(num_partitions);
     }
     Status sink(RuntimeState* state, vectorized::Block* in_block, SourceState source_state,
                 LocalExchangeSinkLocalState& local_state) override;
@@ -204,17 +200,10 @@ class AdaptivePassthroughExchanger : public Exchanger {
                              SourceState source_state, LocalExchangeSinkLocalState& local_state);
     Status _shuffle_sink(RuntimeState* state, vectorized::Block* in_block, SourceState source_state,
                          LocalExchangeSinkLocalState& local_state);
-
-    bool _passthrough_get_block(RuntimeState* state, vectorized::Block* block,
-                                SourceState& source_state,
-                                LocalExchangeSourceLocalState& local_state);
-    bool _shuffle_get_block(RuntimeState* state, vectorized::Block* block,
-                            SourceState& source_state, LocalExchangeSourceLocalState& local_state);
     Status _split_rows(RuntimeState* state, const uint32_t* __restrict channel_ids,
                        vectorized::Block* block, SourceState source_state,
                        LocalExchangeSinkLocalState& local_state);
-    std::vector<moodycamel::ConcurrentQueue<vectorized::Block>> _passthrough_data_queue;
-    std::vector<moodycamel::ConcurrentQueue<PartitionedBlock>> _shuffle_data_queue;
+    std::vector<moodycamel::ConcurrentQueue<vectorized::Block>> _data_queue;
 
     std::atomic_bool _is_pass_through = false;
     std::atomic_int32_t _total_block = 0;

From fcb90dc808f880a225aafe1a58ef5fd7227b31bb Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Fri, 29 Dec 2023 20:47:22 +0800
Subject: [PATCH 180/299] [opt](Nereids) use enable_nereids_dml to control ctas
 and create table (#29284)

---
 .../nereids/trees/plans/commands/CreateTableCommand.java  | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
index 9bdbc949cba1d0..4d1e3937162247 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateTableCommand.java
@@ -73,6 +73,14 @@ public CreateTableCommand(Optional<LogicalPlan> ctasQuery, CreateTableInfo creat
 
     @Override
     public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        if (!ctx.getSessionVariable().isEnableNereidsDML()) {
+            try {
+                ctx.getSessionVariable().enableFallbackToOriginalPlannerOnce();
+            } catch (Exception e) {
+                throw new AnalysisException("failed to set fallback to original planner to true", e);
+            }
+            throw new AnalysisException("Nereids DML is disabled, will try to fall back to the original planner");
+        }
         if (!ctasQuery.isPresent()) {
             createTableInfo.validate(ctx);
             CreateTableStmt createTableStmt = createTableInfo.translateToLegacyStmt();

From 59fdd5e42b24c3549b013bc361a19b7a4984fcf1 Mon Sep 17 00:00:00 2001
From: Sun Chenyang <csun5285@gmail.com>
Date: Fri, 29 Dec 2023 22:20:41 +0800
Subject: [PATCH 181/299] [fix](compaction) time series compaction policy ->
 compact consecutive empty rowsets (#27299)

Sometimes we need to merge a large number of empty versions to reduce meta-information.
---
 be/src/agent/task_worker_pool.cpp             | 10 ++++
 ...mulative_compaction_time_series_policy.cpp | 36 +++++++++++-
 be/src/olap/tablet.cpp                        | 46 ++++++++++++++++
 be/src/olap/tablet.h                          |  2 +
 be/src/olap/tablet_meta.cpp                   | 19 ++++++-
 be/src/olap/tablet_meta.h                     | 10 +++-
 .../java/org/apache/doris/alter/Alter.java    |  4 +-
 .../org/apache/doris/alter/RollupJobV2.java   |  1 +
 .../doris/alter/SchemaChangeHandler.java      |  7 +++
 .../apache/doris/alter/SchemaChangeJobV2.java |  1 +
 .../analysis/ModifyTablePropertiesClause.java | 16 ++++++
 .../org/apache/doris/backup/RestoreJob.java   |  1 +
 .../java/org/apache/doris/catalog/Env.java    | 11 +++-
 .../org/apache/doris/catalog/OlapTable.java   | 14 +++++
 .../apache/doris/catalog/TableProperty.java   | 18 +++++-
 .../doris/common/util/PropertyAnalyzer.java   | 55 ++++++++++++++-----
 .../doris/datasource/InternalCatalog.java     | 25 ++++++++-
 .../apache/doris/master/ReportHandler.java    |  1 +
 .../apache/doris/task/CreateReplicaTask.java  |  5 ++
 .../doris/task/UpdateTabletMetaInfoTask.java  |  5 ++
 .../org/apache/doris/task/AgentTaskTest.java  |  2 +-
 gensrc/proto/olap_file.proto                  |  1 +
 gensrc/thrift/AgentService.thrift             |  2 +
 23 files changed, 267 insertions(+), 25 deletions(-)

diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index a90bbec7d8941f..97bf7f700a1ae7 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -731,6 +731,16 @@ void update_tablet_meta_callback(StorageEngine& engine, const TAgentTaskRequest&
                     tablet_meta_info.time_series_compaction_time_threshold_seconds);
             need_to_save = true;
         }
+        if (tablet_meta_info.__isset.time_series_compaction_empty_rowsets_threshold) {
+            if (tablet->tablet_meta()->compaction_policy() != "time_series") {
+                status = Status::InvalidArgument(
+                        "only time series compaction policy support time series config");
+                continue;
+            }
+            tablet->tablet_meta()->set_time_series_compaction_empty_rowsets_threshold(
+                    tablet_meta_info.time_series_compaction_empty_rowsets_threshold);
+            need_to_save = true;
+        }
         if (tablet_meta_info.__isset.replica_id) {
             tablet->tablet_meta()->set_replica_id(tablet_meta_info.replica_id);
         }
diff --git a/be/src/olap/cumulative_compaction_time_series_policy.cpp b/be/src/olap/cumulative_compaction_time_series_policy.cpp
index c5733df3eafe48..6f7cb8e2e3598f 100644
--- a/be/src/olap/cumulative_compaction_time_series_policy.cpp
+++ b/be/src/olap/cumulative_compaction_time_series_policy.cpp
@@ -63,6 +63,13 @@ uint32_t TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(
         return 0;
     }
 
+    // If there is a continuous set of empty rowsets, prioritize merging.
+    auto consecutive_empty_rowsets = tablet->pick_first_consecutive_empty_rowsets(
+            tablet->tablet_meta()->time_series_compaction_empty_rowsets_threshold());
+    if (!consecutive_empty_rowsets.empty()) {
+        return score;
+    }
+
     // Condition 1: the size of input files for compaction meets the requirement of parameter compaction_goal_size
     int64_t compaction_goal_size_mbytes =
             tablet->tablet_meta()->time_series_compaction_goal_size_mbytes();
@@ -149,6 +156,13 @@ void TimeSeriesCumulativeCompactionPolicy::calculate_cumulative_point(
             break;
         }
 
+        // check if the rowset has been compacted, but it is a empty rowset
+        if (!is_delete && rs->version().first != 0 && rs->version().first != rs->version().second &&
+            rs->num_segments() == 0) {
+            *ret_cumulative_point = rs->version().first;
+            break;
+        }
+
         // include one situation: When the segment is not deleted, and is singleton delta, and is NONOVERLAPPING, ret_cumulative_point increase
         prev_version = rs->version().second;
         *ret_cumulative_point = prev_version + 1;
@@ -166,6 +180,19 @@ int TimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
         return 0;
     }
 
+    // If their are many empty rowsets, maybe should be compacted
+    auto consecutive_empty_rowsets = tablet->pick_first_consecutive_empty_rowsets(
+            tablet->tablet_meta()->time_series_compaction_empty_rowsets_threshold());
+    if (!consecutive_empty_rowsets.empty()) {
+        VLOG_NOTICE << "tablet is " << tablet->tablet_id()
+                    << ", there are too many consecutive empty rowsets, size is "
+                    << consecutive_empty_rowsets.size();
+        input_rowsets->clear();
+        input_rowsets->insert(input_rowsets->end(), consecutive_empty_rowsets.begin(),
+                              consecutive_empty_rowsets.end());
+        return 0;
+    }
+
     int transient_size = 0;
     *compaction_score = 0;
     input_rowsets->clear();
@@ -175,8 +202,9 @@ int TimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
     // BE1 should performs compaction on its own, the time series compaction may re-compact previously fetched rowsets.
     // time series compaction policy needs to skip over the fetched rowset
     const auto& first_rowset_iter = std::find_if(
-            candidate_rowsets.begin(), candidate_rowsets.end(),
-            [](const RowsetSharedPtr& rs) { return rs->start_version() == rs->end_version(); });
+            candidate_rowsets.begin(), candidate_rowsets.end(), [](const RowsetSharedPtr& rs) {
+                return rs->start_version() == rs->end_version() || rs->num_segments() == 0;
+            });
     for (auto it = first_rowset_iter; it != candidate_rowsets.end(); ++it) {
         const auto& rowset = *it;
         // check whether this rowset is delete version
@@ -254,10 +282,12 @@ int TimeSeriesCumulativeCompactionPolicy::pick_input_rowsets(
 void TimeSeriesCumulativeCompactionPolicy::update_cumulative_point(
         Tablet* tablet, const std::vector<RowsetSharedPtr>& input_rowsets,
         RowsetSharedPtr output_rowset, Version& last_delete_version) {
-    if (tablet->tablet_state() != TABLET_RUNNING) {
+    if (tablet->tablet_state() != TABLET_RUNNING || output_rowset->num_segments() == 0) {
         // if tablet under alter process, do not update cumulative point
+        // if the merged output rowset is empty, do not update cumulative point
         return;
     }
+
     tablet->set_cumulative_layer_point(output_rowset->end_version() + 1);
 }
 
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 2d4b430334b04f..4f1c3229958e0b 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1349,6 +1349,52 @@ std::vector<RowsetSharedPtr> Tablet::pick_candidate_rowsets_to_full_compaction()
     return pick_candidate_rowsets_to_single_replica_compaction();
 }
 
+std::vector<RowsetSharedPtr> Tablet::pick_first_consecutive_empty_rowsets(int limit) {
+    std::vector<RowsetSharedPtr> consecutive_empty_rowsets;
+    std::vector<RowsetSharedPtr> candidate_rowsets;
+    traverse_rowsets([&candidate_rowsets, this](const auto& rs) {
+        if (rs->is_local() && rs->start_version() >= _cumulative_point) {
+            candidate_rowsets.emplace_back(rs);
+        }
+    });
+    std::sort(candidate_rowsets.begin(), candidate_rowsets.end(), Rowset::comparator);
+    int len = candidate_rowsets.size();
+    for (int i = 0; i < len - 1; ++i) {
+        auto rowset = candidate_rowsets[i];
+        auto next_rowset = candidate_rowsets[i + 1];
+
+        // identify two consecutive rowsets that are empty
+        if (rowset->num_segments() == 0 && next_rowset->num_segments() == 0 &&
+            !rowset->rowset_meta()->has_delete_predicate() &&
+            !next_rowset->rowset_meta()->has_delete_predicate() &&
+            rowset->end_version() == next_rowset->start_version() - 1) {
+            consecutive_empty_rowsets.emplace_back(rowset);
+            consecutive_empty_rowsets.emplace_back(next_rowset);
+            rowset = next_rowset;
+            int next_index = i + 2;
+
+            // keep searching for consecutive empty rowsets
+            while (next_index < len && candidate_rowsets[next_index]->num_segments() == 0 &&
+                   !candidate_rowsets[next_index]->rowset_meta()->has_delete_predicate() &&
+                   rowset->end_version() == candidate_rowsets[next_index]->start_version() - 1) {
+                consecutive_empty_rowsets.emplace_back(candidate_rowsets[next_index]);
+                rowset = candidate_rowsets[next_index++];
+            }
+            // if the number of consecutive empty rowset reach the limit,
+            // and there are still rowsets following them
+            if (consecutive_empty_rowsets.size() >= limit && next_index < len) {
+                return consecutive_empty_rowsets;
+            } else {
+                // current rowset is not empty, start searching from that rowset in the next
+                i = next_index - 1;
+                consecutive_empty_rowsets.clear();
+            }
+        }
+    }
+
+    return consecutive_empty_rowsets;
+}
+
 std::vector<RowsetSharedPtr> Tablet::pick_candidate_rowsets_to_build_inverted_index(
         const std::set<int32_t>& alter_index_uids, bool is_drop_op) {
     std::vector<RowsetSharedPtr> candidate_rowsets;
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 9f7141b404f04e..03737ea241bf81 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -271,6 +271,8 @@ class Tablet final : public BaseTablet {
     std::vector<RowsetSharedPtr> pick_candidate_rowsets_to_single_replica_compaction();
     std::vector<Version> get_all_versions();
 
+    std::vector<RowsetSharedPtr> pick_first_consecutive_empty_rowsets(int limit);
+
     void calculate_cumulative_point();
     // TODO(ygl):
     bool is_primary_replica() { return false; }
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 793bde7b081189..fc4a22a6175fb2 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -68,7 +68,8 @@ TabletMetaSharedPtr TabletMeta::create(
             std::move(binlog_config), request.compaction_policy,
             request.time_series_compaction_goal_size_mbytes,
             request.time_series_compaction_file_count_threshold,
-            request.time_series_compaction_time_threshold_seconds);
+            request.time_series_compaction_time_threshold_seconds,
+            request.time_series_compaction_empty_rowsets_threshold);
 }
 
 TabletMeta::TabletMeta()
@@ -86,7 +87,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
                        std::optional<TBinlogConfig> binlog_config, std::string compaction_policy,
                        int64_t time_series_compaction_goal_size_mbytes,
                        int64_t time_series_compaction_file_count_threshold,
-                       int64_t time_series_compaction_time_threshold_seconds)
+                       int64_t time_series_compaction_time_threshold_seconds,
+                       int64_t time_series_compaction_empty_rowsets_threshold)
         : _tablet_uid(0, 0),
           _schema(new TabletSchema),
           _delete_bitmap(new DeleteBitmap(tablet_id)) {
@@ -114,6 +116,8 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, int64_t tablet_id
             time_series_compaction_file_count_threshold);
     tablet_meta_pb.set_time_series_compaction_time_threshold_seconds(
             time_series_compaction_time_threshold_seconds);
+    tablet_meta_pb.set_time_series_compaction_empty_rowsets_threshold(
+            time_series_compaction_empty_rowsets_threshold);
     TabletSchemaPB* schema = tablet_meta_pb.mutable_schema();
     schema->set_num_short_key_columns(tablet_schema.short_key_column_count);
     schema->set_num_rows_per_row_block(config::default_num_rows_per_column_file_block);
@@ -313,7 +317,9 @@ TabletMeta::TabletMeta(const TabletMeta& b)
           _time_series_compaction_file_count_threshold(
                   b._time_series_compaction_file_count_threshold),
           _time_series_compaction_time_threshold_seconds(
-                  b._time_series_compaction_time_threshold_seconds) {};
+                  b._time_series_compaction_time_threshold_seconds),
+          _time_series_compaction_empty_rowsets_threshold(
+                  b._time_series_compaction_empty_rowsets_threshold) {};
 
 void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tcolumn,
                                           ColumnPB* column) {
@@ -595,6 +601,8 @@ void TabletMeta::init_from_pb(const TabletMetaPB& tablet_meta_pb) {
             tablet_meta_pb.time_series_compaction_file_count_threshold();
     _time_series_compaction_time_threshold_seconds =
             tablet_meta_pb.time_series_compaction_time_threshold_seconds();
+    _time_series_compaction_empty_rowsets_threshold =
+            tablet_meta_pb.time_series_compaction_empty_rowsets_threshold();
 }
 
 void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
@@ -676,6 +684,8 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
             time_series_compaction_file_count_threshold());
     tablet_meta_pb->set_time_series_compaction_time_threshold_seconds(
             time_series_compaction_time_threshold_seconds());
+    tablet_meta_pb->set_time_series_compaction_empty_rowsets_threshold(
+            time_series_compaction_empty_rowsets_threshold());
 }
 
 int64_t TabletMeta::mem_size() const {
@@ -861,6 +871,9 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) {
     if (a._time_series_compaction_time_threshold_seconds !=
         b._time_series_compaction_time_threshold_seconds)
         return false;
+    if (a._time_series_compaction_empty_rowsets_threshold !=
+        b._time_series_compaction_empty_rowsets_threshold)
+        return false;
     return true;
 }
 
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index 2d14ef599f3b10..a7e284b420fe1a 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -109,7 +109,8 @@ class TabletMeta {
                std::string compaction_policy = "size_based",
                int64_t time_series_compaction_goal_size_mbytes = 1024,
                int64_t time_series_compaction_file_count_threshold = 2000,
-               int64_t time_series_compaction_time_threshold_seconds = 3600);
+               int64_t time_series_compaction_time_threshold_seconds = 3600,
+               int64_t time_series_compaction_empty_rowsets_threshold = 5);
     // If need add a filed in TableMeta, filed init copy in copy construct function
     TabletMeta(const TabletMeta& tablet_meta);
     TabletMeta(TabletMeta&& tablet_meta) = delete;
@@ -251,6 +252,12 @@ class TabletMeta {
     int64_t time_series_compaction_time_threshold_seconds() const {
         return _time_series_compaction_time_threshold_seconds;
     }
+    void set_time_series_compaction_empty_rowsets_threshold(int64_t empty_rowsets_threshold) {
+        _time_series_compaction_empty_rowsets_threshold = empty_rowsets_threshold;
+    }
+    int64_t time_series_compaction_empty_rowsets_threshold() const {
+        return _time_series_compaction_empty_rowsets_threshold;
+    }
 
 private:
     Status _save_meta(DataDir* data_dir);
@@ -303,6 +310,7 @@ class TabletMeta {
     int64_t _time_series_compaction_goal_size_mbytes = 0;
     int64_t _time_series_compaction_file_count_threshold = 0;
     int64_t _time_series_compaction_time_threshold_seconds = 0;
+    int64_t _time_series_compaction_empty_rowsets_threshold = 0;
 
     mutable std::shared_mutex _meta_lock;
 };
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index ff6bf103bc2050..c49a32548754c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -505,7 +505,9 @@ public void processAlterTable(AlterTableStmt stmt) throws UserException {
                         || properties
                             .containsKey(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION)
                         || properties
-                            .containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD));
+                            .containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)
+                        || properties
+                            .containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD));
                 ((SchemaChangeHandler) schemaChangeHandler).updateTableProperties(db, tableName, properties);
             } else {
                 throw new DdlException("Invalid alter operation: " + alterClause.getOpType());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 3abebd17d58559..aa61eaf9062a7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -274,6 +274,7 @@ protected void runPendingJob() throws AlterCancelException {
                                 tbl.getTimeSeriesCompactionGoalSizeMbytes(),
                                 tbl.getTimeSeriesCompactionFileCountThreshold(),
                                 tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
+                                tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                                 tbl.storeRowColumn(),
                                 binlogConfig);
                         createReplicaTask.setBaseTablet(tabletIdMap.get(rollupTabletId), baseSchemaHash);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 654775616b10e8..b1d08ae0c0e561 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -2201,6 +2201,13 @@ public void updateTableProperties(Database db, String tableName, Map<String, Str
                         .get(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS)));
         }
 
+        if (properties.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)) {
+            timeSeriesCompactionConfig
+                        .put(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD,
+                        Long.parseLong(properties
+                        .get(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)));
+        }
+
         if (isInMemory < 0 && storagePolicyId < 0 && compactionPolicy == null && timeSeriesCompactionConfig.isEmpty()
                 && !properties.containsKey(PropertyAnalyzer.PROPERTIES_IS_BEING_SYNCED)
                 && !properties.containsKey(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index b85f0ab764bfe4..9e7174752a84c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -282,6 +282,7 @@ protected void runPendingJob() throws AlterCancelException {
                                     tbl.getTimeSeriesCompactionGoalSizeMbytes(),
                                     tbl.getTimeSeriesCompactionFileCountThreshold(),
                                     tbl.getTimeSeriesCompactionTimeThresholdSeconds(),
+                                    tbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                                     tbl.storeRowColumn(),
                                     binlogConfig);
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
index a23a73df157a05..07529535e6dc70 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java
@@ -206,6 +206,22 @@ public void analyze(Analyzer analyzer) throws AnalysisException {
             }
             this.needTableStable = false;
             this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
+        } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)) {
+            long emptyRowsetsThreshold;
+            String emptyRowsetsThresholdStr = properties
+                                    .get(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD);
+            try {
+                emptyRowsetsThreshold = Long.parseLong(emptyRowsetsThresholdStr);
+                if (emptyRowsetsThreshold < 2) {
+                    throw new AnalysisException("time_series_compaction_empty_rowsets_threshold can not be less than 2:"
+                        + emptyRowsetsThresholdStr);
+                }
+            } catch (NumberFormatException e) {
+                throw new AnalysisException("Invalid time_series_compaction_empty_rowsets_threshold format: "
+                        + emptyRowsetsThresholdStr);
+            }
+            this.needTableStable = false;
+            this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC;
         } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) {
             if (!properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")
                     && !properties.get(PropertyAnalyzer
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 23ca4a335fdd90..c6c6beb3358050 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -1069,6 +1069,7 @@ private void createReplicas(Database db, AgentBatchTask batchTask, OlapTable loc
                             localTbl.getTimeSeriesCompactionGoalSizeMbytes(),
                             localTbl.getTimeSeriesCompactionFileCountThreshold(),
                             localTbl.getTimeSeriesCompactionTimeThresholdSeconds(),
+                            localTbl.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                             localTbl.storeRowColumn(),
                             binlogConfig);
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 1ad131abb0ae18..aba414ecdf6ce0 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -3357,6 +3357,14 @@ public static void getDdlStmt(DdlStmt ddlStmt, String dbName, TableIf table, Lis
                 sb.append(olapTable.getTimeSeriesCompactionTimeThresholdSeconds()).append("\"");
             }
 
+            // time series compaction empty rowsets threshold
+            if (olapTable.getCompactionPolicy() != null && olapTable.getCompactionPolicy()
+                                                            .equals(PropertyAnalyzer.TIME_SERIES_COMPACTION_POLICY)) {
+                sb.append(",\n\"").append(PropertyAnalyzer
+                                    .PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD).append("\" = \"");
+                sb.append(olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold()).append("\"");
+            }
+
             // disable auto compaction
             sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_DISABLE_AUTO_COMPACTION).append("\" = \"");
             sb.append(olapTable.disableAutoCompaction()).append("\"");
@@ -4771,7 +4779,8 @@ public void modifyTableProperties(Database db, OlapTable table, Map<String, Stri
                 .buildTimeSeriesCompactionTimeThresholdSeconds()
                 .buildSkipWriteIndexOnLoad()
                 .buildDisableAutoCompaction()
-                .buildEnableSingleReplicaCompaction();
+                .buildEnableSingleReplicaCompaction()
+                .buildTimeSeriesCompactionEmptyRowsetsThreshold();
 
         // need to update partition info meta
         for (Partition partition : table.getPartitions()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index d12dc9512b5b40..be1ded6eebe775 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2077,6 +2077,20 @@ public Long getTimeSeriesCompactionTimeThresholdSeconds() {
         return null;
     }
 
+    public void setTimeSeriesCompactionEmptyRowsetsThreshold(long timeSeriesCompactionEmptyRowsetsThreshold) {
+        TableProperty tableProperty = getOrCreatTableProperty();
+        tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD,
+                                                Long.valueOf(timeSeriesCompactionEmptyRowsetsThreshold).toString());
+        tableProperty.buildTimeSeriesCompactionEmptyRowsetsThreshold();
+    }
+
+    public Long getTimeSeriesCompactionEmptyRowsetsThreshold() {
+        if (tableProperty != null) {
+            return tableProperty.timeSeriesCompactionEmptyRowsetsThreshold();
+        }
+        return null;
+    }
+
     public int getBaseSchemaVersion() {
         MaterializedIndexMeta baseIndexMeta = indexIdToMeta.get(baseIndexId);
         return baseIndexMeta.getSchemaVersion();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
index 857833de0c487b..54a6041b5043f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableProperty.java
@@ -97,6 +97,9 @@ public class TableProperty implements Writable {
     private long timeSeriesCompactionTimeThresholdSeconds
                                     = PropertyAnalyzer.TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS_DEFAULT_VALUE;
 
+    private long timeSeriesCompactionEmptyRowsetsThreshold
+                                    = PropertyAnalyzer.TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE;
+
     private DataSortInfo dataSortInfo = new DataSortInfo();
 
     public TableProperty(Map<String, String> properties) {
@@ -132,6 +135,7 @@ public TableProperty buildProperty(short opCode) {
                 buildSkipWriteIndexOnLoad();
                 buildEnableSingleReplicaCompaction();
                 buildDisableAutoCompaction();
+                buildTimeSeriesCompactionEmptyRowsetsThreshold();
                 break;
             default:
                 break;
@@ -278,6 +282,17 @@ public long timeSeriesCompactionTimeThresholdSeconds() {
         return timeSeriesCompactionTimeThresholdSeconds;
     }
 
+    public TableProperty buildTimeSeriesCompactionEmptyRowsetsThreshold() {
+        timeSeriesCompactionEmptyRowsetsThreshold = Long.parseLong(properties
+                    .getOrDefault(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD,
+                    String.valueOf(PropertyAnalyzer.TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE)));
+        return this;
+    }
+
+    public long timeSeriesCompactionEmptyRowsetsThreshold() {
+        return timeSeriesCompactionEmptyRowsetsThreshold;
+    }
+
     public TableProperty buildMinLoadReplicaNum() {
         minLoadReplicaNum = Short.parseShort(
                 properties.getOrDefault(PropertyAnalyzer.PROPERTIES_MIN_LOAD_REPLICA_NUM, "-1"));
@@ -528,7 +543,8 @@ public static TableProperty read(DataInput in) throws IOException {
                 .buildTimeSeriesCompactionFileCountThreshold()
                 .buildTimeSeriesCompactionTimeThresholdSeconds()
                 .buildDisableAutoCompaction()
-                .buildEnableSingleReplicaCompaction();
+                .buildEnableSingleReplicaCompaction()
+                .buildTimeSeriesCompactionEmptyRowsetsThreshold();
         if (Env.getCurrentEnvJournalVersion() < FeMetaVersion.VERSION_105) {
             // get replica num from property map and create replica allocation
             String repNum = tableProperty.properties.remove(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index c8c0978ca74015..9c0cfe722f7a9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -144,6 +144,10 @@ public class PropertyAnalyzer {
 
     public static final String PROPERTIES_TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS =
             "time_series_compaction_time_threshold_seconds";
+
+    public static final String PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD =
+            "time_series_compaction_empty_rowsets_threshold";
+
     public static final String PROPERTIES_MUTABLE = "mutable";
 
     public static final String PROPERTIES_IS_BEING_SYNCED = "is_being_synced";
@@ -182,6 +186,7 @@ public class PropertyAnalyzer {
     public static final long TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES_DEFAULT_VALUE = 1024;
     public static final long TIME_SERIES_COMPACTION_FILE_COUNT_THRESHOLD_DEFAULT_VALUE = 2000;
     public static final long TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS_DEFAULT_VALUE = 3600;
+    public static final long TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE = 5;
 
 
     /**
@@ -711,27 +716,28 @@ public static String analyzeCompactionPolicy(Map<String, String> properties) thr
         return compactionPolicy;
     }
 
-    public static long analyzeTimeSeriesCompactionGoalSizeMbytes(Map<String, String> properties)
+    public static long analyzeTimeSeriesCompactionEmptyRowsetsThreshold(Map<String, String> properties)
             throws AnalysisException {
-        long goalSizeMbytes = TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES_DEFAULT_VALUE;
+        long emptyRowsetsThreshold = TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE;
         if (properties == null || properties.isEmpty()) {
-            return goalSizeMbytes;
+            return emptyRowsetsThreshold;
         }
-        if (properties.containsKey(PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES)) {
-            String goalSizeMbytesStr = properties.get(PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES);
-            properties.remove(PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES);
+        if (properties.containsKey(PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)) {
+            String emptyRowsetsThresholdStr = properties
+                                    .get(PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD);
+            properties.remove(PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD);
             try {
-                goalSizeMbytes = Long.parseLong(goalSizeMbytesStr);
-                if (goalSizeMbytes < 10) {
-                    throw new AnalysisException("time_series_compaction_goal_size_mbytes can not be"
-                            + " less than 10: " + goalSizeMbytesStr);
+                emptyRowsetsThreshold = Long.parseLong(emptyRowsetsThresholdStr);
+                if (emptyRowsetsThreshold < 2) {
+                    throw new AnalysisException("time_series_compaction_empty_rowsets_threshold can not"
+                            + " be less than 2: " + emptyRowsetsThresholdStr);
                 }
             } catch (NumberFormatException e) {
-                throw new AnalysisException("Invalid time_series_compaction_goal_size_mbytes format: "
-                        + goalSizeMbytesStr);
+                throw new AnalysisException("Invalid time_series_compaction_empty_rowsets_threshold: "
+                        + emptyRowsetsThresholdStr);
             }
         }
-        return goalSizeMbytes;
+        return emptyRowsetsThreshold;
     }
 
     public static long analyzeTimeSeriesCompactionFileCountThreshold(Map<String, String> properties)
@@ -781,6 +787,29 @@ public static long analyzeTimeSeriesCompactionTimeThresholdSeconds(Map<String, S
         return timeThresholdSeconds;
     }
 
+    public static long analyzeTimeSeriesCompactionGoalSizeMbytes(Map<String, String> properties)
+            throws AnalysisException {
+        long goalSizeMbytes = TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES_DEFAULT_VALUE;
+        if (properties == null || properties.isEmpty()) {
+            return goalSizeMbytes;
+        }
+        if (properties.containsKey(PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES)) {
+            String goalSizeMbytesStr = properties.get(PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES);
+            properties.remove(PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES);
+            try {
+                goalSizeMbytes = Long.parseLong(goalSizeMbytesStr);
+                if (goalSizeMbytes < 10) {
+                    throw new AnalysisException("time_series_compaction_goal_size_mbytes can not be"
+                            + " less than 10: " + goalSizeMbytesStr);
+                }
+            } catch (NumberFormatException e) {
+                throw new AnalysisException("Invalid time_series_compaction_goal_size_mbytes format: "
+                        + goalSizeMbytesStr);
+            }
+        }
+        return goalSizeMbytes;
+    }
+
     // analyzeCompressionType will parse the compression type from properties
     public static TCompressionType analyzeCompressionType(Map<String, String> properties) throws AnalysisException {
         String compressionType = "";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index a3ced0b72fefdf..5adb18e344d6c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1446,6 +1446,10 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa
                 properties.put(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS,
                                                 olapTable.getTimeSeriesCompactionTimeThresholdSeconds().toString());
             }
+            if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)) {
+                properties.put(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD,
+                                                olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold().toString());
+            }
             if (!properties.containsKey(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY)) {
                 properties.put(PropertyAnalyzer.PROPERTIES_STORAGE_POLICY, olapTable.getStoragePolicy());
             }
@@ -1542,6 +1546,7 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa
                     olapTable.getTimeSeriesCompactionGoalSizeMbytes(),
                     olapTable.getTimeSeriesCompactionFileCountThreshold(),
                     olapTable.getTimeSeriesCompactionTimeThresholdSeconds(),
+                    olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                     olapTable.storeRowColumn(),
                     binlogConfig, dataProperty.isStorageMediumSpecified(), null);
             // TODO cluster key ids
@@ -1798,6 +1803,7 @@ private Partition createPartitionWithIndices(long dbId, long tableId, String tab
             boolean enableSingleReplicaCompaction, boolean skipWriteIndexOnLoad,
             String compactionPolicy, Long timeSeriesCompactionGoalSizeMbytes,
             Long timeSeriesCompactionFileCountThreshold, Long timeSeriesCompactionTimeThresholdSeconds,
+            Long timeSeriesCompactionEmptyRowsetsThreshold,
             boolean storeRowColumn, BinlogConfig binlogConfig,
             boolean isStorageMediumSpecified, List<Integer> clusterKeyIndexes) throws DdlException {
         // create base index first.
@@ -1864,6 +1870,7 @@ private Partition createPartitionWithIndices(long dbId, long tableId, String tab
                             disableAutoCompaction, enableSingleReplicaCompaction, skipWriteIndexOnLoad,
                             compactionPolicy, timeSeriesCompactionGoalSizeMbytes,
                             timeSeriesCompactionFileCountThreshold, timeSeriesCompactionTimeThresholdSeconds,
+                            timeSeriesCompactionEmptyRowsetsThreshold,
                             storeRowColumn, binlogConfig);
 
                     task.setStorageFormat(storageFormat);
@@ -2079,7 +2086,9 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
                 && (properties.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_GOAL_SIZE_MBYTES)
                 || properties.containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_FILE_COUNT_THRESHOLD)
                 || properties
-                        .containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS))) {
+                        .containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS)
+                || properties
+                        .containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD))) {
             throw new DdlException("only time series compaction policy support for time series config");
         }
 
@@ -2116,6 +2125,17 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
         }
         olapTable.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds);
 
+        // set time series compaction empty rowsets threshold
+        long timeSeriesCompactionEmptyRowsetsThreshold
+                                     = PropertyAnalyzer.TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD_DEFAULT_VALUE;
+        try {
+            timeSeriesCompactionEmptyRowsetsThreshold = PropertyAnalyzer
+                                    .analyzeTimeSeriesCompactionEmptyRowsetsThreshold(properties);
+        } catch (AnalysisException e) {
+            throw new DdlException(e.getMessage());
+        }
+        olapTable.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold);
+
         // get storage format
         TStorageFormat storageFormat = TStorageFormat.V2; // default is segment v2
         try {
@@ -2466,6 +2486,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
                         olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(),
                         olapTable.getTimeSeriesCompactionFileCountThreshold(),
                         olapTable.getTimeSeriesCompactionTimeThresholdSeconds(),
+                        olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                         storeRowColumn, binlogConfigForTask,
                         partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified(),
                         keysDesc.getClusterKeysColumnIds());
@@ -2542,6 +2563,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep
                             olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(),
                             olapTable.getTimeSeriesCompactionFileCountThreshold(),
                             olapTable.getTimeSeriesCompactionTimeThresholdSeconds(),
+                            olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                             storeRowColumn, binlogConfigForTask,
                             dataProperty.isStorageMediumSpecified(), keysDesc.getClusterKeysColumnIds());
                     olapTable.addPartition(partition);
@@ -2990,6 +3012,7 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti
                         olapTable.getCompactionPolicy(), olapTable.getTimeSeriesCompactionGoalSizeMbytes(),
                         olapTable.getTimeSeriesCompactionFileCountThreshold(),
                         olapTable.getTimeSeriesCompactionTimeThresholdSeconds(),
+                        olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                         olapTable.storeRowColumn(), binlogConfig,
                         copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified(),
                         clusterKeyIdxes);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 71064585a310b4..2c8dc80affb7cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -835,6 +835,7 @@ private static void deleteFromMeta(ListMultimap<Long, Long> tabletDeleteFromMeta
                                             olapTable.getTimeSeriesCompactionGoalSizeMbytes(),
                                             olapTable.getTimeSeriesCompactionFileCountThreshold(),
                                             olapTable.getTimeSeriesCompactionTimeThresholdSeconds(),
+                                            olapTable.getTimeSeriesCompactionEmptyRowsetsThreshold(),
                                             olapTable.storeRowColumn(),
                                             binlogConfig);
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
index 6842e273a6bd8d..818636fe387b50 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/CreateReplicaTask.java
@@ -111,6 +111,8 @@ public class CreateReplicaTask extends AgentTask {
 
     private long timeSeriesCompactionTimeThresholdSeconds;
 
+    private long timeSeriesCompactionEmptyRowsetsThreshold;
+
     private boolean storeRowColumn;
 
     private BinlogConfig binlogConfig;
@@ -134,6 +136,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition
                              long timeSeriesCompactionGoalSizeMbytes,
                              long timeSeriesCompactionFileCountThreshold,
                              long timeSeriesCompactionTimeThresholdSeconds,
+                             long timeSeriesCompactionEmptyRowsetsThreshold,
                              boolean storeRowColumn,
                              BinlogConfig binlogConfig) {
         super(null, backendId, TTaskType.CREATE, dbId, tableId, partitionId, indexId, tabletId);
@@ -175,6 +178,7 @@ public CreateReplicaTask(long backendId, long dbId, long tableId, long partition
         this.timeSeriesCompactionGoalSizeMbytes = timeSeriesCompactionGoalSizeMbytes;
         this.timeSeriesCompactionFileCountThreshold = timeSeriesCompactionFileCountThreshold;
         this.timeSeriesCompactionTimeThresholdSeconds = timeSeriesCompactionTimeThresholdSeconds;
+        this.timeSeriesCompactionEmptyRowsetsThreshold = timeSeriesCompactionEmptyRowsetsThreshold;
         this.storeRowColumn = storeRowColumn;
         this.binlogConfig = binlogConfig;
     }
@@ -322,6 +326,7 @@ public TCreateTabletReq toThrift() {
         createTabletReq.setTimeSeriesCompactionGoalSizeMbytes(timeSeriesCompactionGoalSizeMbytes);
         createTabletReq.setTimeSeriesCompactionFileCountThreshold(timeSeriesCompactionFileCountThreshold);
         createTabletReq.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionTimeThresholdSeconds);
+        createTabletReq.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionEmptyRowsetsThreshold);
 
         if (binlogConfig != null) {
             createTabletReq.setBinlogConfig(binlogConfig.toThrift());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
index af5e45a8256c2c..65158cd31a1539 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/task/UpdateTabletMetaInfoTask.java
@@ -155,6 +155,11 @@ public TUpdateTabletMetaInfoReq toThrift() {
                         metaInfo.setTimeSeriesCompactionTimeThresholdSeconds(timeSeriesCompactionConfig
                                     .get(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_TIME_THRESHOLD_SECONDS));
                     }
+                    if (timeSeriesCompactionConfig
+                            .containsKey(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD)) {
+                        metaInfo.setTimeSeriesCompactionEmptyRowsetsThreshold(timeSeriesCompactionConfig
+                                    .get(PropertyAnalyzer.PROPERTIES_TIME_SERIES_COMPACTION_EMPTY_ROWSETS_THRESHOLD));
+                    }
                 }
                 if (enableSingleReplicaCompaction >= 0) {
                     metaInfo.setEnableSingleReplicaCompaction(enableSingleReplicaCompaction > 0);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
index 8ef71047377025..60ec442bedf8b2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/task/AgentTaskTest.java
@@ -107,7 +107,7 @@ public void setUp() throws AnalysisException {
         createReplicaTask = new CreateReplicaTask(backendId1, dbId, tableId, partitionId,
                 indexId1, tabletId1, replicaId1, shortKeyNum, schemaHash1, version, KeysType.AGG_KEYS, storageType,
                 TStorageMedium.SSD, columns, null, 0, latch, null, false, TTabletType.TABLET_TYPE_DISK, null,
-                TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, false, null);
+                TCompressionType.LZ4F, false, "", false, false, false, "", 0, 0, 0, 0, false, null);
 
         // drop
         dropTask = new DropReplicaTask(backendId1, tabletId1, replicaId1, schemaHash1, false);
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 0f24f6b8d2ca5e..2bf8470719c44f 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -335,6 +335,7 @@ message TabletMetaPB {
     optional int64 time_series_compaction_goal_size_mbytes = 29 [default = 1024];
     optional int64 time_series_compaction_file_count_threshold = 30 [default = 2000];
     optional int64 time_series_compaction_time_threshold_seconds = 31 [default = 3600];
+    optional int64 time_series_compaction_empty_rowsets_threshold = 32 [default = 5];
 }
 
 message OLAPRawDeltaHeaderMessage {
diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift
index 6f777675f3ee0a..a1c4020b1486e0 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -150,6 +150,7 @@ struct TCreateTabletReq {
     23: optional i64 time_series_compaction_goal_size_mbytes = 1024
     24: optional i64 time_series_compaction_file_count_threshold = 2000
     25: optional i64 time_series_compaction_time_threshold_seconds = 3600
+    26: optional i64 time_series_compaction_empty_rowsets_threshold = 5
 }
 
 struct TDropTabletReq {
@@ -419,6 +420,7 @@ struct TTabletMetaInfo {
     14: optional bool enable_single_replica_compaction
     15: optional bool skip_write_index_on_load
     16: optional bool disable_auto_compaction
+    17: optional i64 time_series_compaction_empty_rowsets_threshold
 }
 
 struct TUpdateTabletMetaInfoReq {

From 03ece437f0690e0902f5fcd24bd5129aa2fb0fb1 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@apache.org>
Date: Fri, 29 Dec 2023 23:04:20 +0800
Subject: [PATCH 182/299] Fix topicPublisher thread may NPE when no topic
 exists (#29306)

---
 .../apache/doris/common/publish/TopicPublisherThread.java    | 5 ++---
 .../apache/doris/common/publish/WorkloadGroupPublisher.java  | 4 +---
 .../workloadschedpolicy/WorkloadSchedPolicyPublisher.java    | 4 +---
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java b/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java
index db5158e24f5a01..0a6bed76d00a22 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/publish/TopicPublisherThread.java
@@ -66,9 +66,8 @@ protected void runAfterCatalogReady() {
             topicPublisher.getTopicInfo(request);
         }
 
-        if (request.getTopicMap().size() == 0) {
-            return;
-        }
+        // even request contains no group and schedule policy, we still need to send an empty rpc.
+        // because it may means workload group/policy is dropped
 
         // step 2: publish topic info to all be
         Collection<Backend> nodesToPublish = clusterInfoService.getIdToBackend().values();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java b/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java
index e81ccb06d3e564..45b3664631e450 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/publish/WorkloadGroupPublisher.java
@@ -35,8 +35,6 @@ public WorkloadGroupPublisher(Env env) {
     @Override
     public void getTopicInfo(TPublishTopicRequest req) {
         List<TopicInfo> list = env.getWorkloadGroupMgr().getPublishTopicInfo();
-        if (list.size() > 0) {
-            req.putToTopicMap(TTopicInfoType.WORKLOAD_GROUP, list);
-        }
+        req.putToTopicMap(TTopicInfoType.WORKLOAD_GROUP, list);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java
index 5083d183e6c957..e3b6e466a3d201 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadSchedPolicyPublisher.java
@@ -36,9 +36,7 @@ public WorkloadSchedPolicyPublisher(Env env) {
     @Override
     public void getTopicInfo(TPublishTopicRequest req) {
         List<TopicInfo> list = env.getWorkloadSchedPolicyMgr().getPublishTopicInfoList();
-        if (list.size() > 0) {
-            req.putToTopicMap(TTopicInfoType.WORKLOAD_SCHED_POLICY, list);
-        }
+        req.putToTopicMap(TTopicInfoType.WORKLOAD_SCHED_POLICY, list);
     }
 
 }

From 989d20e0ace73934f9dec991b541cce0065ef2d8 Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Fri, 29 Dec 2023 23:22:32 +0800
Subject: [PATCH 183/299] [opt](Nereids) remove decimalv2 signature from min,
 max, sum, nvl and case when (#29282)

---
 .../trees/expressions/functions/agg/Max.java      |  5 +++++
 .../trees/expressions/functions/agg/Min.java      |  5 +++++
 .../trees/expressions/functions/agg/Sum.java      | 15 +++++++++++----
 .../trees/expressions/functions/scalar/Nvl.java   |  6 +-----
 .../doris/nereids/util/TypeCoercionUtils.java     | 14 ++++++++------
 .../trees/expressions/GetDataTypeTest.java        |  4 ++--
 6 files changed, 32 insertions(+), 17 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java
index a8530bbc18092e..a879fe470b44ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Max.java
@@ -27,6 +27,8 @@
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.DecimalV2Type;
+import org.apache.doris.nereids.types.DecimalV3Type;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -58,6 +60,9 @@ public void checkLegalityBeforeTypeCoercion() {
     @Override
     public FunctionSignature customSignature() {
         DataType dataType = getArgument(0).getDataType();
+        if (dataType instanceof DecimalV2Type) {
+            dataType = DecimalV3Type.forType(dataType);
+        }
         return FunctionSignature.ret(dataType).args(dataType);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java
index d40c32844e180e..8e8d0c7e874176 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Min.java
@@ -27,6 +27,8 @@
 import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.DecimalV2Type;
+import org.apache.doris.nereids.types.DecimalV3Type;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -59,6 +61,9 @@ public void checkLegalityBeforeTypeCoercion() {
     @Override
     public FunctionSignature customSignature() {
         DataType dataType = getArgument(0).getDataType();
+        if (dataType instanceof DecimalV2Type) {
+            dataType = DecimalV3Type.forType(dataType);
+        }
         return FunctionSignature.ret(dataType).args(dataType);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java
index 34681a2611623f..87992039361095 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/Sum.java
@@ -29,9 +29,9 @@
 import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.types.BooleanType;
 import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.nereids.types.DecimalV2Type;
 import org.apache.doris.nereids.types.DecimalV3Type;
 import org.apache.doris.nereids.types.DoubleType;
+import org.apache.doris.nereids.types.FloatType;
 import org.apache.doris.nereids.types.IntegerType;
 import org.apache.doris.nereids.types.LargeIntType;
 import org.apache.doris.nereids.types.SmallIntType;
@@ -56,9 +56,8 @@ public class Sum extends NullableAggregateFunction
             FunctionSignature.ret(BigIntType.INSTANCE).args(IntegerType.INSTANCE),
             FunctionSignature.ret(BigIntType.INSTANCE).args(BigIntType.INSTANCE),
             FunctionSignature.ret(LargeIntType.INSTANCE).args(LargeIntType.INSTANCE),
-            FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE),
-            FunctionSignature.ret(DecimalV2Type.SYSTEM_DEFAULT).args(DecimalV2Type.SYSTEM_DEFAULT),
-            FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD)
+            FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD),
+            FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE)
     );
 
     /**
@@ -112,6 +111,14 @@ public List<FunctionSignature> getSignatures() {
         return SIGNATURES;
     }
 
+    @Override
+    public FunctionSignature searchSignature(List<FunctionSignature> signatures) {
+        if (getArgument(0).getDataType() instanceof FloatType) {
+            return FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE);
+        }
+        return ExplicitlyCastableSignature.super.searchSignature(signatures);
+    }
+
     @Override
     public Function constructRollUp(Expression param, Expression... varParams) {
         return new Sum(this.distinct, param);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java
index 9b8e51d238d061..e3d0335cbc3345 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Nvl.java
@@ -29,7 +29,6 @@
 import org.apache.doris.nereids.types.DateTimeV2Type;
 import org.apache.doris.nereids.types.DateType;
 import org.apache.doris.nereids.types.DateV2Type;
-import org.apache.doris.nereids.types.DecimalV2Type;
 import org.apache.doris.nereids.types.DecimalV3Type;
 import org.apache.doris.nereids.types.DoubleType;
 import org.apache.doris.nereids.types.FloatType;
@@ -58,6 +57,7 @@ public class Nvl extends ScalarFunction
             FunctionSignature.ret(IntegerType.INSTANCE).args(IntegerType.INSTANCE, IntegerType.INSTANCE),
             FunctionSignature.ret(BigIntType.INSTANCE).args(BigIntType.INSTANCE, BigIntType.INSTANCE),
             FunctionSignature.ret(LargeIntType.INSTANCE).args(LargeIntType.INSTANCE, LargeIntType.INSTANCE),
+            FunctionSignature.ret(DecimalV3Type.WILDCARD).args(DecimalV3Type.WILDCARD, DecimalV3Type.WILDCARD),
             FunctionSignature.ret(DoubleType.INSTANCE).args(DoubleType.INSTANCE, DoubleType.INSTANCE),
             FunctionSignature.ret(FloatType.INSTANCE).args(FloatType.INSTANCE, FloatType.INSTANCE),
             FunctionSignature.ret(DateType.INSTANCE).args(DateType.INSTANCE, DateType.INSTANCE),
@@ -66,10 +66,6 @@ public class Nvl extends ScalarFunction
                     .args(DateTimeV2Type.SYSTEM_DEFAULT, DateTimeV2Type.SYSTEM_DEFAULT),
             FunctionSignature.ret(DateV2Type.INSTANCE)
                     .args(DateV2Type.INSTANCE, DateV2Type.INSTANCE),
-            FunctionSignature.ret(DecimalV3Type.WILDCARD)
-                    .args(DecimalV3Type.WILDCARD, DecimalV3Type.WILDCARD),
-            FunctionSignature.ret(DecimalV2Type.SYSTEM_DEFAULT)
-                    .args(DecimalV2Type.SYSTEM_DEFAULT, DecimalV2Type.SYSTEM_DEFAULT),
             FunctionSignature.ret(BitmapType.INSTANCE).args(BitmapType.INSTANCE, BitmapType.INSTANCE),
             FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT)
                     .args(VarcharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
index 273ba4b643663a..168ee50c55cb8b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java
@@ -986,25 +986,27 @@ public static Expression processCaseWhen(CaseWhen caseWhen) {
         Optional<DataType> optionalCommonType = TypeCoercionUtils.findWiderCommonTypeForCaseWhen(dataTypesForCoercion);
         return optionalCommonType
                 .map(commonType -> {
+                    DataType realCommonType = commonType instanceof DecimalV2Type
+                            ? DecimalV3Type.forType(commonType) : commonType;
                     List<Expression> newChildren
                             = caseWhen.getWhenClauses().stream()
                             .map(wc -> {
                                 Expression valueExpr = TypeCoercionUtils.castIfNotSameType(
-                                        wc.getResult(), commonType);
+                                        wc.getResult(), realCommonType);
                                 // we must cast every child to the common type, and then
                                 // FoldConstantRuleOnFe can eliminate some branches and direct
                                 // return a branch value
-                                if (!valueExpr.getDataType().equals(commonType)) {
-                                    valueExpr = new Cast(valueExpr, commonType);
+                                if (!valueExpr.getDataType().equals(realCommonType)) {
+                                    valueExpr = new Cast(valueExpr, realCommonType);
                                 }
                                 return wc.withChildren(wc.getOperand(), valueExpr);
                             })
                             .collect(Collectors.toList());
                     caseWhen.getDefaultValue()
                             .map(dv -> {
-                                Expression defaultExpr = TypeCoercionUtils.castIfNotSameType(dv, commonType);
-                                if (!defaultExpr.getDataType().equals(commonType)) {
-                                    defaultExpr = new Cast(defaultExpr, commonType);
+                                Expression defaultExpr = TypeCoercionUtils.castIfNotSameType(dv, realCommonType);
+                                if (!defaultExpr.getDataType().equals(realCommonType)) {
+                                    defaultExpr = new Cast(defaultExpr, realCommonType);
                                 }
                                 return defaultExpr;
                             })
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java
index 62019d8bacaf96..05824d32802071 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/GetDataTypeTest.java
@@ -35,7 +35,7 @@
 import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
 import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.nereids.types.DecimalV2Type;
+import org.apache.doris.nereids.types.DecimalV3Type;
 import org.apache.doris.nereids.types.DoubleType;
 import org.apache.doris.nereids.types.LargeIntType;
 
@@ -74,7 +74,7 @@ public void testSum() {
         Assertions.assertEquals(LargeIntType.INSTANCE, checkAndGetDataType(new Sum(largeIntLiteral)));
         Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum(floatLiteral)));
         Assertions.assertEquals(DoubleType.INSTANCE, checkAndGetDataType(new Sum(doubleLiteral)));
-        Assertions.assertEquals(DecimalV2Type.createDecimalV2Type(27, 9), checkAndGetDataType(new Sum(decimalLiteral)));
+        Assertions.assertEquals(DecimalV3Type.createDecimalV3Type(38, 0), checkAndGetDataType(new Sum(decimalLiteral)));
         Assertions.assertEquals(BigIntType.INSTANCE, checkAndGetDataType(new Sum(bigIntLiteral)));
         Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum(charLiteral)));
         Assertions.assertThrows(RuntimeException.class, () -> checkAndGetDataType(new Sum(varcharLiteral)));

From 2518ed64ea12dfbe93c1d8abc18cca0fbe260389 Mon Sep 17 00:00:00 2001
From: Siyang Tang <82279870+TangSiyang2001@users.noreply.github.com>
Date: Fri, 29 Dec 2023 23:44:43 +0800
Subject: [PATCH 184/299] [enhancement](bulk-load) strict second level
 granularity for timeout between (0,1) (#29028)

---
 .../java/org/apache/doris/load/loadv2/LoadLoadingTask.java     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
index cbe293a8ab4145..a56ba1acd14f07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
@@ -160,7 +160,8 @@ private void executeOnce() throws Exception {
         if (leftTimeMs <= 0) {
             throw new LoadException("failed to execute loading task when timeout");
         }
-        int timeoutS = (int) (leftTimeMs / 1000);
+        // 1 second is the minimum granularity of actual execution
+        int timeoutS = Math.max((int) (leftTimeMs / 1000), 1);
         curCoordinator.setTimeout(timeoutS);
 
         try {

From 7f105facc4c64069619161bc461e8c7734bc51a8 Mon Sep 17 00:00:00 2001
From: Mryange <59914473+Mryange@users.noreply.github.com>
Date: Fri, 29 Dec 2023 23:48:46 +0800
Subject: [PATCH 185/299] [fix](pipelineX) fix unable to merge profiles in
 multi be (#29204)

---
 .../common/profile/ExecutionProfile.java      | 48 ++++++++++++++-----
 .../java/org/apache/doris/qe/Coordinator.java | 14 ++++--
 2 files changed, 46 insertions(+), 16 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
index ecee299f1046a5..9584a6f1fa154f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/profile/ExecutionProfile.java
@@ -23,6 +23,7 @@
 import org.apache.doris.common.util.DebugUtil;
 import org.apache.doris.common.util.RuntimeProfile;
 import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.thrift.TNetworkAddress;
 import org.apache.doris.thrift.TUniqueId;
 import org.apache.doris.thrift.TUnit;
 
@@ -35,9 +36,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 
-
 /**
  * ExecutionProfile is used to collect profile of a complete query plan(including query or load).
  * Need to call addToProfileAsChild() to add it to the root profile.
@@ -71,39 +72,62 @@ public class ExecutionProfile {
     // fragmentId -> dummy value
     private MarkedCountDownLatch<Integer, Long> profileFragmentDoneSignal;
 
+    // use to merge profile from multi be
+    private List<Map<TNetworkAddress, List<RuntimeProfile>>> multiBeProfile = null;
+
     public ExecutionProfile(TUniqueId queryId, int fragmentNum) {
         executionProfile = new RuntimeProfile("Execution Profile " + DebugUtil.printId(queryId));
         RuntimeProfile fragmentsProfile = new RuntimeProfile("Fragments");
         executionProfile.addChild(fragmentsProfile);
         fragmentProfiles = Lists.newArrayList();
+        multiBeProfile = Lists.newArrayList();
         for (int i = 0; i < fragmentNum; i++) {
             fragmentProfiles.add(new RuntimeProfile("Fragment " + i));
             fragmentsProfile.addChild(fragmentProfiles.get(i));
+            multiBeProfile.add(new ConcurrentHashMap<TNetworkAddress, List<RuntimeProfile>>());
         }
         loadChannelProfile = new RuntimeProfile("LoadChannels");
         executionProfile.addChild(loadChannelProfile);
     }
 
+    public void addMultiBeProfileByPipelineX(int profileFragmentId, TNetworkAddress address,
+            List<RuntimeProfile> taskProfile) {
+        multiBeProfile.get(profileFragmentId).put(address, taskProfile);
+    }
+
+    private List<List<RuntimeProfile>> getMultiBeProfile(int profileFragmentId) {
+        Map<TNetworkAddress, List<RuntimeProfile>> multiPipeline = multiBeProfile.get(profileFragmentId);
+        List<List<RuntimeProfile>> allPipelines = Lists.newArrayList();
+        int pipelineSize = 0;
+        for (List<RuntimeProfile> profiles : multiPipeline.values()) {
+            pipelineSize = profiles.size();
+            break;
+        }
+        for (int pipelineIdx = 0; pipelineIdx < pipelineSize; pipelineIdx++) {
+            List<RuntimeProfile> allPipelineTask = new ArrayList<RuntimeProfile>();
+            for (List<RuntimeProfile> pipelines : multiPipeline.values()) {
+                RuntimeProfile pipeline = pipelines.get(pipelineIdx);
+                for (Pair<RuntimeProfile, Boolean> runtimeProfile : pipeline.getChildList()) {
+                    allPipelineTask.add(runtimeProfile.first);
+                }
+            }
+            allPipelines.add(allPipelineTask);
+        }
+        return allPipelines;
+    }
+
     private RuntimeProfile getPipelineXAggregatedProfile(Map<Integer, String> planNodeMap) {
         RuntimeProfile fragmentsProfile = new RuntimeProfile("Fragments");
         for (int i = 0; i < fragmentProfiles.size(); ++i) {
-            RuntimeProfile oldFragmentProfile = fragmentProfiles.get(i);
             RuntimeProfile newFragmentProfile = new RuntimeProfile("Fragment " + i);
             fragmentsProfile.addChild(newFragmentProfile);
-            List<RuntimeProfile> allPipelines = new ArrayList<RuntimeProfile>();
-            for (Pair<RuntimeProfile, Boolean> runtimeProfile : oldFragmentProfile.getChildList()) {
-                allPipelines.add(runtimeProfile.first);
-            }
+            List<List<RuntimeProfile>> allPipelines = getMultiBeProfile(i);
             int pipelineIdx = 0;
-            for (RuntimeProfile pipeline : allPipelines) {
-                List<RuntimeProfile> allPipelineTask = new ArrayList<RuntimeProfile>();
-                for (Pair<RuntimeProfile, Boolean> runtimeProfile : pipeline.getChildList()) {
-                    allPipelineTask.add(runtimeProfile.first);
-                }
+            for (List<RuntimeProfile> allPipelineTask : allPipelines) {
                 RuntimeProfile mergedpipelineProfile = new RuntimeProfile(
                         "Pipeline : " + pipelineIdx + "(instance_num="
                                 + allPipelineTask.size() + ")",
-                        allPipelines.get(0).nodeId());
+                        allPipelineTask.get(0).nodeId());
                 RuntimeProfile.mergeProfiles(allPipelineTask, mergedpipelineProfile, planNodeMap);
                 newFragmentProfile.addChild(mergedpipelineProfile);
                 pipelineIdx++;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 92bb329b98c970..421fa1ac5238e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -3174,9 +3174,11 @@ public Stream<RuntimeProfile> profileStream() {
             return fragmentInstancesMap.values().stream();
         }
 
-        private void attachInstanceProfileToFragmentProfile() {
+        public void attachPipelineProfileToFragmentProfile() {
             profileStream()
                     .forEach(p -> executionProfile.addInstanceProfile(this.profileFragmentId, p));
+            executionProfile.addMultiBeProfileByPipelineX(profileFragmentId, address,
+                    taskProfile);
         }
 
         /**
@@ -3218,7 +3220,7 @@ public synchronized boolean updateProfile(TReportExecStatusParams params) {
                     loadChannelProfile.update(params.loadChannelProfile);
                 }
                 this.done = params.done;
-                attachInstanceProfileToFragmentProfile();
+                attachPipelineProfileToFragmentProfile();
                 return this.done;
             } else {
                 RuntimeProfile profile = fragmentInstancesMap.get(params.fragment_instance_id);
@@ -3940,8 +3942,12 @@ public List<QueryStatisticsItem.FragmentInstanceInfo> getFragmentInstanceInfos()
     private void attachInstanceProfileToFragmentProfile() {
         if (enablePipelineEngine) {
             for (PipelineExecContext ctx : pipelineExecContexts.values()) {
-                ctx.profileStream()
-                        .forEach(p -> executionProfile.addInstanceProfile(ctx.profileFragmentId, p));
+                if (enablePipelineXEngine) {
+                    ctx.attachPipelineProfileToFragmentProfile();
+                } else {
+                    ctx.profileStream()
+                            .forEach(p -> executionProfile.addInstanceProfile(ctx.profileFragmentId, p));
+                }
             }
         } else {
             for (BackendExecState backendExecState : backendExecStates) {

From b8cdbf4ffeb4614d61fae1bf630e10056ce17c92 Mon Sep 17 00:00:00 2001
From: Luwei <814383175@qq.com>
Date: Sat, 30 Dec 2023 00:02:01 +0800
Subject: [PATCH 186/299] Fix](schema change) fix metadata inconsistency bug
 caused by enable light schema change (#28255) (#29060)

---
 be/src/service/internal_service.cpp           |  31 +++-
 .../test_enable_light_schema_change.groovy    | 156 ++++++++++++++++++
 2 files changed, 186 insertions(+), 1 deletion(-)
 create mode 100644 regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy

diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index bca48737bf508a..ba2686c0fd76c3 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -807,6 +807,7 @@ void PInternalServiceImpl::_get_column_ids_by_tablet_ids(
         int64_t index_id = param.indexid();
         auto tablet_ids = param.tablet_ids();
         std::set<std::set<int32_t>> filter_set;
+        std::map<int32_t, const TabletColumn*> id_to_column;
         for (const int64_t tablet_id : tablet_ids) {
             TabletSharedPtr tablet = tablet_mgr->get_tablet(tablet_id);
             if (tablet == nullptr) {
@@ -819,17 +820,45 @@ void PInternalServiceImpl::_get_column_ids_by_tablet_ids(
             }
             // check schema consistency, column ids should be the same
             const auto& columns = tablet->tablet_schema()->columns();
+
             std::set<int32_t> column_ids;
             for (const auto& col : columns) {
                 column_ids.insert(col.unique_id());
             }
             filter_set.insert(column_ids);
+
+            if (id_to_column.empty()) {
+                for (const auto& col : columns) {
+                    id_to_column.insert(std::pair {col.unique_id(), &col});
+                }
+            } else {
+                for (const auto& col : columns) {
+                    auto it = id_to_column.find(col.unique_id());
+                    if (it == id_to_column.end() || *(it->second) != col) {
+                        ColumnPB prev_col_pb;
+                        ColumnPB curr_col_pb;
+                        if (it != id_to_column.end()) {
+                            it->second->to_schema_pb(&prev_col_pb);
+                        }
+                        col.to_schema_pb(&curr_col_pb);
+                        std::stringstream ss;
+                        ss << "consistency check failed: index{ " << index_id << " }"
+                           << " got inconsistent schema, prev column: " << prev_col_pb.DebugString()
+                           << " current column: " << curr_col_pb.DebugString();
+                        LOG(WARNING) << ss.str();
+                        response->mutable_status()->set_status_code(TStatusCode::ILLEGAL_STATE);
+                        response->mutable_status()->add_error_msgs(ss.str());
+                        return;
+                    }
+                }
+            }
         }
+
         if (filter_set.size() > 1) {
             // consistecy check failed
             std::stringstream ss;
             ss << "consistency check failed: index{" << index_id << "}"
-               << "got inconsistent shema";
+               << "got inconsistent schema";
             LOG(WARNING) << ss.str();
             response->mutable_status()->set_status_code(TStatusCode::ILLEGAL_STATE);
             response->mutable_status()->add_error_msgs(ss.str());
diff --git a/regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy b/regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy
new file mode 100644
index 00000000000000..0c09af5e5de116
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/test_enable_light_schema_change.groovy
@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.util.Date
+import java.text.SimpleDateFormat
+import org.apache.http.HttpResponse
+import org.apache.http.client.methods.HttpPut
+import org.apache.http.impl.client.CloseableHttpClient
+import org.apache.http.impl.client.HttpClients
+import org.apache.http.entity.ContentType
+import org.apache.http.entity.StringEntity
+import org.apache.http.client.config.RequestConfig
+import org.apache.http.client.RedirectStrategy
+import org.apache.http.protocol.HttpContext
+import org.apache.http.HttpRequest
+import org.apache.http.impl.client.LaxRedirectStrategy
+import org.apache.http.client.methods.RequestBuilder
+import org.apache.http.entity.StringEntity
+import org.apache.http.client.methods.CloseableHttpResponse
+import org.apache.http.util.EntityUtils
+
+suite("test_enable_light_schema_change", "p0") {
+    def tableName1 = "test_enable_lsc"
+    def tableName2 = "test_enable_lsc_normal"
+
+    def getJobState = { tableName ->
+        def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+        return jobStateResult[0][9]
+    }
+
+    def getCreateViewState = { tableName ->
+        def createViewStateResult = sql """ SHOW ALTER TABLE MATERIALIZED VIEW WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+        return createViewStateResult[0][8]
+    }
+
+    sql """ DROP TABLE IF EXISTS ${tableName1} """
+
+    sql """
+        CREATE TABLE ${tableName1}
+        (
+            k1 DATE,
+            k2 int NOT NULL DEFAULT "1",
+            k3 CHAR(10) COMMENT "string column",
+            k4 INT NOT NULL DEFAULT "1" COMMENT "int column"
+        )
+        unique KEY(k1, k2)
+        COMMENT "my first table"
+        PARTITION BY RANGE(k1)
+        (
+            PARTITION p1 VALUES LESS THAN ("2020-02-01"),
+            PARTITION p2 VALUES LESS THAN ("2020-03-01"),
+            PARTITION p3 VALUES LESS THAN ("2020-04-01")
+        )
+        DISTRIBUTED BY HASH(k1) BUCKETS 1
+        PROPERTIES (
+            "replication_num" = "1",
+            "light_schema_change" = "false"
+        );
+    """
+
+    sql """ insert into ${tableName1} values ('2020-02-10', 2, 'a', 4) """
+
+    sql """ alter table ${tableName1} order by (k1, k2, k4, k3) """
+
+    max_try_num = 60
+    while (max_try_num--) {
+        String res = getJobState(tableName1)
+        if (res == "FINISHED" || res == "CANCELLED") {
+            assertEquals("FINISHED", res)
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_num < 1) {
+                println "test timeout," + "state:" + res
+                assertEquals("FINISHED",res)
+            }
+        }
+    }
+
+    sql """ alter table ${tableName1} ADD PARTITION p4 VALUES LESS THAN ("2020-05-01") """
+    sql """ insert into ${tableName1} values ('2020-04-10', 2, 5, 'b') """
+
+    test {
+        sql """ alter table ${tableName1} set ("light_schema_change"="true") """
+        exception "errCode = 2, detailMessage = failed to enable light schema change for table"
+    }
+
+    sql """ select * from ${tableName1} """
+
+    sql """ DROP TABLE IF EXISTS ${tableName2} """
+    sql """
+        CREATE TABLE ${tableName2}
+        (
+            k1 DATE,
+            k2 int NOT NULL DEFAULT "1",
+            k3 CHAR(10) COMMENT "string column",
+            k4 INT NOT NULL DEFAULT "1" COMMENT "int column"
+        )
+        duplicate KEY(k1, k2)
+        COMMENT "my first table"
+        PARTITION BY RANGE(k1)
+        (
+            PARTITION p1 VALUES LESS THAN ("2020-02-01"),
+            PARTITION p2 VALUES LESS THAN ("2020-03-01"),
+            PARTITION p3 VALUES LESS THAN ("2020-04-01")
+        )
+        DISTRIBUTED BY HASH(k1) BUCKETS 1
+        PROPERTIES (
+            "replication_num" = "1",
+            "light_schema_change" = "false"
+        );
+    """
+
+    sql """ insert into ${tableName2} values ('2020-02-10', 2, 'a', 4) """
+
+    sql """ alter table ${tableName2} ADD COLUMN k5 string after k4"""
+
+    max_try_num = 60
+    while (max_try_num--) {
+        String res = getJobState(tableName2)
+        if (res == "FINISHED" || res == "CANCELLED") {
+            assertEquals("FINISHED", res)
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_num < 1) {
+                println "test timeout," + "state:" + res
+                assertEquals("FINISHED",res)
+            }
+        }
+    }
+
+    sql """ alter table ${tableName2} ADD PARTITION p4 VALUES LESS THAN ("2020-05-01") """
+    sql """ insert into ${tableName2} values ('2020-04-10', 2, 'b', 5, 'test') """
+
+    sql """ alter table ${tableName2} set ("light_schema_change"="true") """
+
+    sql """ select * from ${tableName2} """
+}
+

From 4e56adee1518098f42ff2ca928ed9cfa636590df Mon Sep 17 00:00:00 2001
From: Petrichor <31833513+vinlee19@users.noreply.github.com>
Date: Sat, 30 Dec 2023 00:02:43 +0800
Subject: [PATCH 187/299] [improvement](docs) add Json load case doc (#29001)

---
 .../import/import-way/load-json-format.md     | 84 +++++++++++++++----
 .../import/import-way/load-json-format.md     | 55 ++++++++++++
 2 files changed, 125 insertions(+), 14 deletions(-)

diff --git a/docs/en/docs/data-operate/import/import-way/load-json-format.md b/docs/en/docs/data-operate/import/import-way/load-json-format.md
index d66845cb592057..15968f26436562 100644
--- a/docs/en/docs/data-operate/import/import-way/load-json-format.md
+++ b/docs/en/docs/data-operate/import/import-way/load-json-format.md
@@ -243,9 +243,9 @@ In import statement 1, only JSON Path is specified, and Columns is not specified
 
 ````text
 +------+------+
-| k1 | k2 |
+| k1   | k2   |
 +------+------+
-| 2 | 1 |
+| 2    | 1    |
 +------+------+
 ````
 
@@ -261,9 +261,9 @@ Compared with the import statement 1, the Columns field is added here to describ
 
 ````text
 +------+------+
-| k1 | k2 |
+| k1   |  k2  |
 +------+------+
-| 1 | 2 |
+| 1    | 2    |
 +------+------+
 ````
 
@@ -277,12 +277,68 @@ The above example will import the value of k1 multiplied by 100. The final impor
 
 ````text
 +------+------+
-| k1 | k2 |
+| k1   | k2   |
 +------+------+
-| 100 | 2 |
+| 100  | 2    |
 +------+------+
 ````
 
+Import statement 3：
+
+Compared with the  import statement 1 and import statement 2, the columns field `k1_copy` is added here.
+Table Structure：
+
+```
+k2 int, k1 int, k1_copy int
+```
+
+If you want to assign a column field in JSON to several column fields in the table multiple times, you can specify the column multiple times in jsonPaths and specify the mapping order in sequence. An example is as follows:
+
+```bash
+curl -v --location-trusted -u root: -H "format: json" -H "jsonpaths: [\"$.k2\", \"$.k1\", \"$.k1\"]" -H "columns: k2,k1,k1_copy" -T example.json http://127.0.0.1:8030/api/db1/tbl1/_stream_load
+```
+
+The above example will extract the fields in the order specified by the JSON Path. It designates the first column as the value for the `k2` column in the table, the second column as the value for the `k1` column, and the third column as the value for the `k1_copy` column. The final imported data result is as follows:
+
+```text
++------+------+---------+
+| k2   | k1   | k2_copy |
++------+------+---------+
+|    2 |    1 |       2 |
++------+------+---------+
+```
+
+Import statement 4：
+
+Data content：
+
+```json
+{"k1" : 1, "k2": 2, "k3": {"k1" : 31, "k1_nested" : {"k1" : 32} } }
+```
+
+Compared with the  import statement 1 and import statement 2, the columns field `k1_nested1` and `k1_nested2` are added here.
+
+Table Structure:
+
+```
+k2 int, k1 int, k1_nested1 int, k1_nested2 int
+```
+If you want to assign multi-level fields with the same name nested in json to different columns in the table, you can specify the column in jsonPaths and specify the mapping order in turn. An example are as follows:
+
+```bash
+curl -v --location-trusted -u root: -H "format: json" -H "jsonpaths: [\"$.k2\", \"$.k1\",\"$.k3.k1\",\"$.k3.k1_nested.k1\" -H "columns: k2,k1,k1_nested1,k1_nested2" -T example.json http://127.0.0.1:8030/api/db1/tbl1/_stream_load
+```
+
+The above example will extract the fields in the order of the JSON Path, specifying that the first column is the value of the `k2` column in the table, the second column is the value of the `k1` column in the table, and the third column is the `k1` column in the nested type. The value of the `k1_nested1` column, from which we know that the `k3.k1_nested.k1` column is the value of the `k1_nested2` column in the table. The final imported data results are as follows:
+
+```text
++------+------+------------+------------+
+| k2   | k1   | k1_nested1 | k1_nested2 |
++------+------+------------+------------+
+|    2 |    1 |         31 |         32 |
++------+------+------------+------------+
+```
+
 ## JSON root
 
 Doris supports extracting data specified in JSON through JSON root.
@@ -340,13 +396,13 @@ The import results that users may expect are as follows, that is, for missing co
 
 ````text
 +------+------+
-| k1 | k2 |
+| k1 | k2     |
 +------+------+
-| 1 | a |
+| 1    | a    |
 +------+------+
-| 2 | x |
+| 2    | x    |
 +------+------+
-|3|c|
+|3     |c     |
 +------+------+
 ````
 
@@ -354,13 +410,13 @@ But the actual import result is as follows, that is, for the missing column, NUL
 
 ````text
 +------+------+
-| k1 | k2 |
+| k1 | k2     |
 +------+------+
-| 1 | a |
+| 1    | a    |
 +------+------+
-| 2 | NULL |
+| 2    | NULL |
 +------+------+
-|3|c|
+|3     |c     |
 +------+------+
 ````
 
diff --git a/docs/zh-CN/docs/data-operate/import/import-way/load-json-format.md b/docs/zh-CN/docs/data-operate/import/import-way/load-json-format.md
index f0e35f7340abcc..82bb9a895b90cc 100644
--- a/docs/zh-CN/docs/data-operate/import/import-way/load-json-format.md
+++ b/docs/zh-CN/docs/data-operate/import/import-way/load-json-format.md
@@ -284,6 +284,61 @@ curl -v --location-trusted -u root: -H "format: json" -H "jsonpaths: [\"$.k2\",
 +------+------+
 ```
 
+导入语句3：
+
+相比于导入语句1和导入语句2的表结构，这里增加`k1_copy`列。
+表结构：
+
+```
+k2 int, k1 int, k1_copy int
+```
+如果你想将json中的某一字段多次赋予给表中几列，那么可以在jsonPaths中多次指定该列，并且依次指定映射顺序。示例如下：
+
+```bash
+curl -v --location-trusted -u root: -H "format: json" -H "jsonpaths: [\"$.k2\", \"$.k1\", \"$.k1\"]" -H "columns: k2,k1,k1_copy" -T example.json http://127.0.0.1:8030/api/db1/tbl1/_stream_load
+```
+
+上述示例会按 JSON Path 中字段的顺序抽取后，指定第一列为表中 k2 列的值，而第二列为表中 k1 列的值，第二列为表中 k1_copy 列的值。最终导入的数据结果如下：
+
+```text
++------+------+---------+
+| k2   | k1   | k2_copy |
++------+------+---------+
+|    2 |    1 |       2 |
++------+------+---------+
+```
+
+导入语句4：
+
+数据内容：
+
+```json
+{"k1" : 1, "k2": 2, "k3": {"k1" : 31, "k1_nested" : {"k1" : 32} } }
+```
+
+相比于导入语句1和导入语句2的表结构，这里增加`k1_nested1`,`k1_nested2`列。
+表结构：
+
+```
+k2 int, k1 int, k1_nested1 int, k1_nested2 int
+```
+如果你想将json中嵌套的多级同名字段赋予给表中不同的列，那么可以在jsonPaths中指定该列，并且依次指定映射顺序。示例如下：
+
+```bash
+curl -v --location-trusted -u root: -H "format: json" -H "jsonpaths: [\"$.k2\", \"$.k1\",\"$.k3.k1\",\"$.k3.k1_nested.k1\" -H "columns: k2,k1,k1_nested1,k1_nested2" -T example.json http://127.0.0.1:8030/api/db1/tbl1/_stream_load
+```
+
+上述示例会按 JSON Path 中字段的顺序抽取后，指定第一列为表中 k2 列的值，而第二列为表中 k1 列的值，第三列嵌套类型中的 k1 列为表中 k1_nested1 列的值，由此可知 k3.k1_nested.k1 列为表中 k1_nested2列的值。 最终导入的数据结果如下：
+
+```text
++------+------+------------+------------+
+| k2   | k1   | k1_nested1 | k1_nested2 |
++------+------+------------+------------+
+|    2 |    1 |         31 |         32 |
++------+------+------------+------------+
+```
+
+
 ## JSON root
 
 Doris 支持通过 JSON root 抽取 JSON 中指定的数据。

From 0bf0f945a2dd341a207d8c1d79c99d90383be6c8 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Fri, 29 Dec 2023 23:04:01 +0700
Subject: [PATCH 188/299] [test](ut) added UT cases for show broker stmt
 (#28629)

---
 .../doris/analysis/ShowBrokerStmtTest.java    | 87 +++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBrokerStmtTest.java

diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBrokerStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBrokerStmtTest.java
new file mode 100644
index 00000000000000..bb361f060616fe
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBrokerStmtTest.java
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.ShowResultSetMetaData;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ShowBrokerStmtTest {
+    private Analyzer analyzer;
+    private ConnectContext ctx = new ConnectContext();
+
+    @Before
+    public void setUp() {
+        analyzer = AccessTestUtil.fetchAdminAnalyzer(true);
+        ctx.setSessionVariable(new SessionVariable());
+        ctx.setThreadLocalInfo();
+    }
+
+    @After
+    public void tearDown() {
+        ConnectContext.remove();
+    }
+
+    @Test
+    public void testAnalyze() throws AnalysisException {
+        AtomicBoolean privilege = new AtomicBoolean(false);
+        new MockUp<AccessControllerManager>() {
+            @Mock
+            public boolean checkGlobalPriv(ConnectContext ctx, PrivPredicate wanted) {
+                return privilege.get();
+            }
+        };
+
+        ShowBrokerStmt stmt = new ShowBrokerStmt();
+        Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+
+        privilege.set(true);
+        stmt.analyze(analyzer);
+    }
+
+    @Test
+    public void getMetaData() {
+        ShowBrokerStmt stmt = new ShowBrokerStmt();
+        ShowResultSetMetaData result = stmt.getMetaData();
+        Assertions.assertEquals(result.getColumnCount(), 7);
+        result.getColumns().forEach(col -> Assertions.assertEquals(col.getType(), ScalarType.createVarchar(30)));
+    }
+
+
+    @Test
+    public void getRedirectStatus() {
+        ShowBrokerStmt stmt = new ShowBrokerStmt();
+        Assertions.assertEquals(RedirectStatus.FORWARD_NO_SYNC, stmt.getRedirectStatus());
+
+        ctx.getSessionVariable().forwardToMaster = false;
+        Assertions.assertEquals(RedirectStatus.NO_FORWARD, stmt.getRedirectStatus());
+    }
+}

From d7001ce6e1c2a4bee71a02c20f1738e003813f0c Mon Sep 17 00:00:00 2001
From: deardeng <565620795@qq.com>
Date: Sat, 30 Dec 2023 00:07:43 +0800
Subject: [PATCH 189/299] =?UTF-8?q?[fix](partition)=20Comment=20out=20some?=
 =?UTF-8?q?=20exceptions=20with=20partition=20equal=20to=20=E2=80=A6=20(#2?=
 =?UTF-8?q?8702)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* [fix](partition) Comment out some exceptions with partition equal to 0 and return

In some scenarios, the partition id in the table meta is 0, but the rowset meta partition is not 0, (#28055) PR check partition id will caused be failed to start when upgrade

meta like this

```
{
    "table_id": 1266302,
    "partition_id": 0,
    "tablet_id": 1289222,
    "schema_hash": 816078096,
    "shard_id": 949,
    "creation_time": 1684805433,
    "cumulative_layer_point": -1,
    "tablet_state": "PB_RUNNING",
    "schema": {
        "keys_type": "DUP_KEYS",
        "column": [
	    ...
        ],
        "num_short_key_columns": 1,
        "num_rows_per_row_block": 1024,
	...
        "skip_write_index_on_load": false
    },
    "rs_metas": [
        {
            "rowset_id": 0,
            "partition_id": 1289193,
            "tablet_id": 1289222,
            "tablet_schema_hash": 816078096,
            "newest_write_timestamp": 1684805433,
	    ...
            "segments_overlap_pb": "NONOVERLAPPING"
        }
    ],
    "in_restore_mode": false,
    "tablet_uid": {
        "hi": -4301918259617109775,
        "lo": 2820124814688031668
    },
    "preferred_rowset_type": "BETA_ROWSET",
    "tablet_type": "TABLET_TYPE_DISK",
    "replica_id": 1812028,
    "enable_unique_key_merge_on_write": false,
    "binlog_config": {
        "enable": false,
        "ttl_seconds": 9223372036854775807,
        "max_bytes": 9223372036854775807,
        "max_history_nums": 9223372036854775807
    },
    "compaction_policy": "size_based",
    "time_series_compaction_goal_size_mbytes": 1024,
    "time_series_compaction_file_count_threshold": 2000,
    "time_series_compaction_time_threshold_seconds": 3600
}
```

* fix
---
 be/src/olap/rowset/rowset_meta_manager.cpp | 5 +++--
 be/src/olap/tablet_manager.cpp             | 8 ++++++++
 be/src/olap/tablet_meta_manager.cpp        | 5 +++--
 3 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp
index 01384a560575d5..be49e66ee1114d 100644
--- a/be/src/olap/rowset/rowset_meta_manager.cpp
+++ b/be/src/olap/rowset/rowset_meta_manager.cpp
@@ -91,8 +91,9 @@ Status RowsetMetaManager::save(OlapMeta* meta, TabletUid tablet_uid, const Rowse
     if (rowset_meta_pb.partition_id() <= 0) {
         LOG(WARNING) << "invalid partition id " << rowset_meta_pb.partition_id() << " tablet "
                      << rowset_meta_pb.tablet_id();
-        return Status::InternalError("invalid partition id {}, tablet {}",
-                                     rowset_meta_pb.partition_id(), rowset_meta_pb.tablet_id());
+        // TODO(dx): after fix partition id eq 0 bug, fix it
+        // return Status::InternalError("invaid partition id {} tablet {}",
+        //  rowset_meta_pb.partition_id(), rowset_meta_pb.tablet_id());
     }
     if (enable_binlog) {
         return _save_with_binlog(meta, tablet_uid, rowset_id, rowset_meta_pb);
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 6010121424d5df..45c12ce582f7f9 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -501,6 +501,10 @@ TabletSharedPtr TabletManager::_create_tablet_meta_and_dir_unlocked(
             }
         }
 
+        if (tablet_meta->partition_id() <= 0) {
+            LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << ", tablet "
+                         << tablet_meta->tablet_id();
+        }
         TabletSharedPtr new_tablet = std::make_shared<Tablet>(std::move(tablet_meta), data_dir);
         COUNTER_UPDATE(ADD_CHILD_TIMER(profile, "CreateTabletFromMeta", parent_timer_name),
                        static_cast<int64_t>(watch.reset()));
@@ -826,6 +830,10 @@ Status TabletManager::load_tablet_from_meta(DataDir* data_dir, TTabletId tablet_
         tablet_meta->set_tablet_state(TABLET_RUNNING);
     }
 
+    if (tablet_meta->partition_id() <= 0) {
+        LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << ", tablet "
+                     << tablet_meta->tablet_id();
+    }
     TabletSharedPtr tablet = std::make_shared<Tablet>(std::move(tablet_meta), data_dir);
 
     // NOTE: method load_tablet_from_meta could be called by two cases as below
diff --git a/be/src/olap/tablet_meta_manager.cpp b/be/src/olap/tablet_meta_manager.cpp
index 17d5d2e9f74b09..288da455e6a888 100644
--- a/be/src/olap/tablet_meta_manager.cpp
+++ b/be/src/olap/tablet_meta_manager.cpp
@@ -96,8 +96,9 @@ Status TabletMetaManager::save(DataDir* store, TTabletId tablet_id, TSchemaHash
     if (tablet_meta->partition_id() <= 0) {
         LOG(WARNING) << "invalid partition id " << tablet_meta->partition_id() << " tablet "
                      << tablet_meta->tablet_id();
-        return Status::InternalError("invaid partition id {} tablet {}",
-                                     tablet_meta->partition_id(), tablet_meta->tablet_id());
+        // TODO(dx): after fix partition id eq 0 bug, fix it
+        // return Status::InternalError("invaid partition id {} tablet {}",
+        //  tablet_meta->partition_id(), tablet_meta->tablet_id());
     }
     OlapMeta* meta = store->get_meta();
     VLOG_NOTICE << "save tablet meta"

From 445f72b39514569a0d3dae4bee1539832046f3d6 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Fri, 29 Dec 2023 23:09:50 +0700
Subject: [PATCH 190/299] [ut](stats) Added tests for HMS analysis tasks
 (#28583)

---
 .../doris/statistics/HMSAnalysisTask.java     |   2 +-
 .../doris/statistics/HMSAnalysisTaskTest.java | 151 ++++++++++++++++++
 2 files changed, 152 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
index efd99d1eca953d..5053fc62a2d15f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/HMSAnalysisTask.java
@@ -88,7 +88,7 @@ private void getTableStats() throws Exception {
     /**
      * Get column statistics and insert the result to __internal_schema.column_statistics
      */
-    private void getTableColumnStats() throws Exception {
+    protected void getTableColumnStats() throws Exception {
         if (!info.usingSqlForPartitionColumn && isPartitionColumn()) {
             try {
                 getPartitionColumnStats();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
index 12a1a9c046b4d0..fb0a3b3c2ca91f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/HMSAnalysisTaskTest.java
@@ -19,12 +19,17 @@
 
 import org.apache.doris.analysis.TableSample;
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.DatabaseIf;
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.catalog.external.HMSExternalTable;
 import org.apache.doris.common.Pair;
+import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.statistics.util.StatisticsUtil;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
+import mockit.Expectations;
 import mockit.Mock;
 import mockit.MockUp;
 import mockit.Mocked;
@@ -32,7 +37,9 @@
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import java.util.Set;
 
 public class HMSAnalysisTaskTest {
 
@@ -160,4 +167,148 @@ public List<Long> getChunkSizes() {
         Assertions.assertEquals(0, info2.second);
     }
 
+    @Test
+    public void testGetSampleInfoPercent(@Mocked HMSExternalTable tableIf)
+            throws Exception {
+        new MockUp<HMSExternalTable>() {
+            @Mock
+            public List<Long> getChunkSizes() {
+                return Arrays.asList(1024L, 2048L);
+            }
+        };
+        HMSAnalysisTask task = new HMSAnalysisTask();
+        task.setTable(tableIf);
+        AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
+        analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
+        analysisInfoBuilder.setAnalysisMethod(AnalysisInfo.AnalysisMethod.SAMPLE);
+        analysisInfoBuilder.setSamplePercent(10);
+        task.info = analysisInfoBuilder.build();
+
+        task.tableSample = new TableSample(true, 10L);
+        Pair<Double, Long> info = task.getSampleInfo();
+        Assertions.assertEquals(1.5, info.first);
+        Assertions.assertEquals(2048, info.second);
+    }
+
+    @Test
+    public void testOrdinaryStats(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked HMSExternalTable tableIf)
+            throws Exception {
+
+        new Expectations() {
+            {
+                tableIf.getId();
+                result = 30001;
+                tableIf.getName();
+                result = "test";
+                catalogIf.getId();
+                result = 10001;
+                catalogIf.getName();
+                result = "hms";
+                databaseIf.getId();
+                result = 20001;
+                databaseIf.getFullName();
+                result = "default";
+            }
+        };
+
+        new MockUp<HMSExternalTable>() {
+            @Mock
+            public Set<String> getPartitionNames() {
+                return ImmutableSet.of("date=20230101/hour=12");
+            }
+        };
+
+        new MockUp<HMSAnalysisTask>() {
+            @Mock
+            public void runQuery(String sql) {
+                Assertions.assertEquals("SELECT CONCAT(30001, '-', -1, '-', 'hour') AS `id`,"
+                        + "          10001 AS `catalog_id`,"
+                        + "          20001 AS `db_id`,"
+                        + "          30001 AS `tbl_id`,"
+                        + "          -1 AS `idx_id`,"
+                        + "          'hour' AS `col_id`,"
+                        + "          NULL AS `part_id`,"
+                        + "          COUNT(1) AS `row_count`,"
+                        + "          NDV(`hour`) AS `ndv`,"
+                        + "          COUNT(1) - COUNT(`hour`) AS `null_count`,"
+                        + "          SUBSTRING(CAST(MIN(`hour`) AS STRING), 1, 1024) AS `min`,"
+                        + "          SUBSTRING(CAST(MAX(`hour`) AS STRING), 1, 1024) AS `max`,"
+                        + "          COUNT(1) * 4 AS `data_size`,"
+                        + "          NOW() AS `update_time`"
+                        + "  FROM `hms`.`default`.`test`", sql);
+            }
+        };
+
+        HMSAnalysisTask task = new HMSAnalysisTask();
+        task.col = new Column("hour", PrimitiveType.INT);
+        task.tbl = tableIf;
+        task.catalog = catalogIf;
+        task.db = databaseIf;
+        task.setTable(tableIf);
+
+        AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
+        analysisInfoBuilder.setColName("hour");
+        analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
+        analysisInfoBuilder.setUsingSqlForPartitionColumn(true);
+        task.info = analysisInfoBuilder.build();
+
+        task.getTableColumnStats();
+    }
+
+
+    @Test
+    public void testPartitionHMSStats(@Mocked CatalogIf catalogIf, @Mocked DatabaseIf databaseIf, @Mocked HMSExternalTable tableIf)
+            throws Exception {
+
+        new Expectations() {
+            {
+                tableIf.getId();
+                result = 30001;
+                catalogIf.getId();
+                result = 10001;
+                catalogIf.getName();
+                result = "hms";
+                databaseIf.getId();
+                result = 20001;
+            }
+        };
+
+        new MockUp<HMSExternalTable>() {
+            @Mock
+            public Set<String> getPartitionNames() {
+                return ImmutableSet.of("date=20230101/hour=12");
+            }
+
+            @Mock
+            public List<Column> getPartitionColumns() {
+                return ImmutableList.of(new Column("hour", PrimitiveType.INT));
+            }
+        };
+
+        new MockUp<HMSAnalysisTask>() {
+            @Mock
+            public void runQuery(String sql) {
+                Assertions.assertEquals(" SELECT CONCAT(30001, '-', -1, '-', 'hour') AS `id`, "
+                        + "10001 AS `catalog_id`, 20001 AS `db_id`, 30001 AS `tbl_id`, -1 AS `idx_id`, "
+                        + "'hour' AS `col_id`, NULL AS `part_id`, 0 AS `row_count`, 1 AS `ndv`, "
+                        + "0 AS `null_count`, SUBSTRING(CAST('12' AS STRING), 1, 1024) AS `min`, "
+                        + "SUBSTRING(CAST('12' AS STRING), 1, 1024) AS `max`, 0 AS `data_size`, NOW() ", sql);
+            }
+        };
+
+        HMSAnalysisTask task = new HMSAnalysisTask();
+        task.col = new Column("hour", PrimitiveType.INT);
+        task.tbl = tableIf;
+        task.catalog = catalogIf;
+        task.db = databaseIf;
+        task.setTable(tableIf);
+
+        AnalysisInfoBuilder analysisInfoBuilder = new AnalysisInfoBuilder();
+        analysisInfoBuilder.setColName("hour");
+        analysisInfoBuilder.setJobType(AnalysisInfo.JobType.MANUAL);
+        analysisInfoBuilder.setUsingSqlForPartitionColumn(false);
+        task.info = analysisInfoBuilder.build();
+
+        task.getTableColumnStats();
+    }
 }

From 8407490053cdc4cea94cef661739b7852b2d550d Mon Sep 17 00:00:00 2001
From: Xiangyu Wang <dut.xiangyu@gmail.com>
Date: Sat, 30 Dec 2023 00:10:35 +0800
Subject: [PATCH 191/299] [feature-wip](nereids) Support some spark-sql
 built-in functions when set dialect=spark_sql (#28531)

---
 .../analyzer/PlaceholderExpression.java       | 31 +++++--
 .../parser/AbstractFnCallTransformers.java    |  6 +-
 .../parser/CommonFnCallTransformer.java       | 91 ++++++++++++-------
 ...mer.java => ComplexFnCallTransformer.java} |  6 +-
 .../spark/DateTruncFnCallTransformer.java     | 81 +++++++++++++++++
 .../spark/SparkSql3FnCallTransformers.java    | 56 ++++++++----
 .../trino/DateDiffFnCallTransformer.java      |  8 +-
 .../parser/trino/TrinoFnCallTransformers.java |  7 +-
 .../nereids/parser/spark/FnTransformTest.java | 72 +++++++++++----
 9 files changed, 263 insertions(+), 95 deletions(-)
 rename fe/fe-core/src/main/java/org/apache/doris/nereids/parser/{trino/ComplexTrinoFnCallTransformer.java => ComplexFnCallTransformer.java} (81%)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/DateTruncFnCallTransformer.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java
index b2acdddd64fec5..9b2dcde49bd1f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/PlaceholderExpression.java
@@ -23,9 +23,11 @@
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 
 /**
  * Expression placeHolder, the expression in PlaceHolderExpression will be collected by
@@ -33,15 +35,25 @@
  * @see PlaceholderCollector
  */
 public class PlaceholderExpression extends Expression implements AlwaysNotNullable {
-    private final Class<? extends Expression> delegateClazz;
+
+    private final ImmutableSet<Class<? extends Expression>> delegateClazzSet;
     /**
-     * 1 based
+     * start from 1, set the index of this placeholderExpression in sourceFnTransformedArguments
+     * this placeholderExpression will be replaced later
      */
     private final int position;
 
     public PlaceholderExpression(List<Expression> children, Class<? extends Expression> delegateClazz, int position) {
         super(children);
-        this.delegateClazz = Objects.requireNonNull(delegateClazz, "delegateClazz should not be null");
+        this.delegateClazzSet = ImmutableSet.of(
+                Objects.requireNonNull(delegateClazz, "delegateClazz should not be null"));
+        this.position = position;
+    }
+
+    public PlaceholderExpression(List<Expression> children,
+                                 Set<Class<? extends Expression>> delegateClazzSet, int position) {
+        super(children);
+        this.delegateClazzSet = ImmutableSet.copyOf(delegateClazzSet);
         this.position = position;
     }
 
@@ -49,13 +61,18 @@ public static PlaceholderExpression of(Class<? extends Expression> delegateClazz
         return new PlaceholderExpression(ImmutableList.of(), delegateClazz, position);
     }
 
+    public static PlaceholderExpression of(Set<Class<? extends Expression>> delegateClazzSet, int position) {
+        return new PlaceholderExpression(ImmutableList.of(), delegateClazzSet, position);
+    }
+
     @Override
     public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+        visitor.visitPlaceholderExpression(this, context);
         return visitor.visit(this, context);
     }
 
-    public Class<? extends Expression> getDelegateClazz() {
-        return delegateClazz;
+    public Set<Class<? extends Expression>> getDelegateClazzSet() {
+        return delegateClazzSet;
     }
 
     public int getPosition() {
@@ -74,11 +91,11 @@ public boolean equals(Object o) {
             return false;
         }
         PlaceholderExpression that = (PlaceholderExpression) o;
-        return position == that.position && Objects.equals(delegateClazz, that.delegateClazz);
+        return position == that.position && Objects.equals(delegateClazzSet, that.delegateClazzSet);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), delegateClazz, position);
+        return Objects.hash(super.hashCode(), delegateClazzSet, position);
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/AbstractFnCallTransformers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/AbstractFnCallTransformers.java
index 75b5f87263c968..4386123c4286de 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/AbstractFnCallTransformers.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/AbstractFnCallTransformers.java
@@ -77,17 +77,15 @@ private Function doTransform(List<AbstractFnCallTransformer> transformers,
 
     protected void doRegister(
             String sourceFnNme,
-            int sourceFnArgumentsNum,
             String targetFnName,
-            List<? extends Expression> targetFnArguments,
-            boolean variableArgument) {
+            List<? extends Expression> targetFnArguments) {
 
         List<Expression> castedTargetFnArguments = targetFnArguments
                 .stream()
                 .map(each -> (Expression) each)
                 .collect(Collectors.toList());
         transformerBuilder.put(sourceFnNme, new CommonFnCallTransformer(new UnboundFunction(
-                targetFnName, castedTargetFnArguments), variableArgument, sourceFnArgumentsNum));
+                targetFnName, castedTargetFnArguments)));
     }
 
     protected void doRegister(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/CommonFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/CommonFnCallTransformer.java
index 872c21a71e5964..fa05416067410e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/CommonFnCallTransformer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/CommonFnCallTransformer.java
@@ -23,56 +23,76 @@
 import org.apache.doris.nereids.trees.expressions.functions.Function;
 import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
 
+import com.google.common.collect.Lists;
+
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 
 /**
- * Trino function transformer
+ * Common function transformer,
+ * can transform functions which the size and type of target arguments are both the same with the source function,
+ * or source function is a variable-arguments function.
  */
 public class CommonFnCallTransformer extends AbstractFnCallTransformer {
     private final UnboundFunction targetFunction;
     private final List<PlaceholderExpression> targetArguments;
-    private final boolean variableArgument;
-    private final int sourceArgumentsNum;
+
+    // true means the arguments of this function is dynamic, for example:
+    // - named_struct('f1', 1, 'f2', 'a', 'f3', "abc")
+    // - struct(1, 'a', 'abc');
+    private final boolean variableArguments;
 
     /**
-     * Trino function transformer, mostly this handle common function.
+     * Common function transformer, mostly this handle common function.
      */
-    public CommonFnCallTransformer(UnboundFunction targetFunction,
-                                   boolean variableArgument,
-                                   int sourceArgumentsNum) {
+    public CommonFnCallTransformer(UnboundFunction targetFunction, boolean variableArguments) {
+        this.targetFunction = targetFunction;
+        PlaceholderCollector placeHolderCollector = new PlaceholderCollector();
+        placeHolderCollector.visit(targetFunction, null);
+        this.targetArguments = placeHolderCollector.getPlaceholderExpressions();
+        this.variableArguments = variableArguments;
+    }
+
+    public CommonFnCallTransformer(UnboundFunction targetFunction) {
         this.targetFunction = targetFunction;
-        this.variableArgument = variableArgument;
-        this.sourceArgumentsNum = sourceArgumentsNum;
-        PlaceholderCollector placeHolderCollector = new PlaceholderCollector(variableArgument);
+        PlaceholderCollector placeHolderCollector = new PlaceholderCollector();
         placeHolderCollector.visit(targetFunction, null);
         this.targetArguments = placeHolderCollector.getPlaceholderExpressions();
+        this.variableArguments = false;
     }
 
     @Override
     protected boolean check(String sourceFnName,
             List<Expression> sourceFnTransformedArguments,
             ParserContext context) {
+        // if variableArguments=true, we can not recognize if the type of all arguments is valid or not,
+        // because:
+        //     1. the argument size is not sure
+        //     2. there are some functions which can accept different types of arguments,
+        //        for example: struct(1, 'a', 'abc')
+        // so just return true here.
+        if (variableArguments) {
+            return true;
+        }
         List<Class<? extends Expression>> sourceFnTransformedArgClazz = sourceFnTransformedArguments.stream()
                 .map(Expression::getClass)
                 .collect(Collectors.toList());
-        if (variableArgument) {
-            if (targetArguments.isEmpty()) {
-                return false;
-            }
-            Class<? extends Expression> targetArgumentClazz = targetArguments.get(0).getDelegateClazz();
-            for (Expression argument : sourceFnTransformedArguments) {
-                if (!targetArgumentClazz.isAssignableFrom(argument.getClass())) {
-                    return false;
-                }
-            }
-        }
-        if (sourceFnTransformedArguments.size() != sourceArgumentsNum) {
+        if (sourceFnTransformedArguments.size() != targetArguments.size()) {
             return false;
         }
-        for (int i = 0; i < targetArguments.size(); i++) {
-            if (!targetArguments.get(i).getDelegateClazz().isAssignableFrom(sourceFnTransformedArgClazz.get(i))) {
+        for (PlaceholderExpression targetArgument : targetArguments) {
+            // replace the arguments of target function by the position of target argument
+            int position = targetArgument.getPosition();
+            Class<? extends Expression> sourceArgClazz = sourceFnTransformedArgClazz.get(position - 1);
+            boolean valid = false;
+            for (Class<? extends Expression> targetArgClazz : targetArgument.getDelegateClazzSet()) {
+                if (targetArgClazz.isAssignableFrom(sourceArgClazz)) {
+                    valid = true;
+                    break;
+                }
+            }
+            if (!valid) {
                 return false;
             }
         }
@@ -83,7 +103,16 @@ protected boolean check(String sourceFnName,
     protected Function transform(String sourceFnName,
             List<Expression> sourceFnTransformedArguments,
             ParserContext context) {
-        return targetFunction.withChildren(sourceFnTransformedArguments);
+        if (variableArguments) {
+            // not support adjust the order of arguments when variableArguments=true
+            return targetFunction.withChildren(sourceFnTransformedArguments);
+        }
+        List<Expression> sourceFnTransformedArgumentsInorder = Lists.newArrayList();
+        for (PlaceholderExpression placeholderExpression : targetArguments) {
+            Expression expression = sourceFnTransformedArguments.get(placeholderExpression.getPosition() - 1);
+            sourceFnTransformedArgumentsInorder.add(expression);
+        }
+        return targetFunction.withChildren(sourceFnTransformedArgumentsInorder);
     }
 
     /**
@@ -93,20 +122,12 @@ protected Function transform(String sourceFnName,
     public static final class PlaceholderCollector extends DefaultExpressionVisitor<Void, Void> {
 
         private final List<PlaceholderExpression> placeholderExpressions = new ArrayList<>();
-        private final boolean variableArgument;
 
-        public PlaceholderCollector(boolean variableArgument) {
-            this.variableArgument = variableArgument;
-        }
+        public PlaceholderCollector() {}
 
         @Override
         public Void visitPlaceholderExpression(PlaceholderExpression placeholderExpression, Void context) {
-
-            if (variableArgument) {
-                placeholderExpressions.add(placeholderExpression);
-                return null;
-            }
-            placeholderExpressions.set(placeholderExpression.getPosition() - 1, placeholderExpression);
+            placeholderExpressions.add(placeholderExpression);
             return null;
         }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ComplexFnCallTransformer.java
similarity index 81%
rename from fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java
rename to fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ComplexFnCallTransformer.java
index d3a687a289feed..2583320b534646 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/ComplexTrinoFnCallTransformer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/ComplexFnCallTransformer.java
@@ -15,14 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package org.apache.doris.nereids.parser.trino;
-
-import org.apache.doris.nereids.parser.AbstractFnCallTransformer;
+package org.apache.doris.nereids.parser;
 
 /**
  * Trino complex function transformer
  */
-public abstract class ComplexTrinoFnCallTransformer extends AbstractFnCallTransformer {
+public abstract class ComplexFnCallTransformer extends AbstractFnCallTransformer {
 
     protected abstract String getSourceFnName();
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/DateTruncFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/DateTruncFnCallTransformer.java
new file mode 100644
index 00000000000000..1503c6db22c61f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/DateTruncFnCallTransformer.java
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.parser.spark;
+
+import org.apache.doris.nereids.analyzer.UnboundFunction;
+import org.apache.doris.nereids.parser.ComplexFnCallTransformer;
+import org.apache.doris.nereids.parser.ParserContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.functions.Function;
+import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+
+/**
+ * DateTrunc complex function transformer
+ */
+public class DateTruncFnCallTransformer extends ComplexFnCallTransformer {
+
+    // reference: https://spark.apache.org/docs/latest/api/sql/index.html#trunc
+    // spark-sql support YEAR/YYYY/YY for year, support MONTH/MON/MM for month
+    private static final ImmutableSet<String> YEAR = ImmutableSet.<String>builder()
+            .add("YEAR")
+            .add("YYYY")
+            .add("YY")
+            .build();
+
+    private static final ImmutableSet<String> MONTH = ImmutableSet.<String>builder()
+            .add("MONTH")
+            .add("MON")
+            .add("MM")
+            .build();
+
+    @Override
+    public String getSourceFnName() {
+        return "trunc";
+    }
+
+    @Override
+    protected boolean check(String sourceFnName, List<Expression> sourceFnTransformedArguments,
+            ParserContext context) {
+        return getSourceFnName().equalsIgnoreCase(sourceFnName) && (sourceFnTransformedArguments.size() == 2);
+    }
+
+    @Override
+    protected Function transform(String sourceFnName, List<Expression> sourceFnTransformedArguments,
+            ParserContext context) {
+        VarcharLiteral fmtLiteral = (VarcharLiteral) sourceFnTransformedArguments.get(1);
+        if (YEAR.contains(fmtLiteral.getValue().toUpperCase())) {
+            return new UnboundFunction(
+                    "date_trunc",
+                    ImmutableList.of(sourceFnTransformedArguments.get(0), new VarcharLiteral("YEAR")));
+        }
+        if (MONTH.contains(fmtLiteral.getValue().toUpperCase())) {
+            return new UnboundFunction(
+                    "date_trunc",
+                    ImmutableList.of(sourceFnTransformedArguments.get(0), new VarcharLiteral("MONTH")));
+        }
+
+        return new UnboundFunction(
+                "date_trunc",
+                ImmutableList.of(sourceFnTransformedArguments.get(0), sourceFnTransformedArguments.get(1)));
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java
index 5a6ec21fc9a1c5..341a3d1951dd0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/spark/SparkSql3FnCallTransformers.java
@@ -18,15 +18,13 @@
 package org.apache.doris.nereids.parser.spark;
 
 import org.apache.doris.nereids.analyzer.PlaceholderExpression;
-import org.apache.doris.nereids.parser.AbstractFnCallTransformer;
 import org.apache.doris.nereids.parser.AbstractFnCallTransformers;
 import org.apache.doris.nereids.trees.expressions.Expression;
 
 import com.google.common.collect.Lists;
 
 /**
- * The builder and factory for spark-sql 3.x {@link AbstractFnCallTransformer},
- * and supply transform facade ability.
+ * The builder and factory for spark-sql 3.x FnCallTransformers, supply transform facade ability.
  */
 public class SparkSql3FnCallTransformers extends AbstractFnCallTransformers {
 
@@ -35,30 +33,54 @@ private SparkSql3FnCallTransformers() {
 
     @Override
     protected void registerTransformers() {
-        doRegister("get_json_object", 2, "json_extract",
-                Lists.newArrayList(
-                        PlaceholderExpression.of(Expression.class, 1),
-                        PlaceholderExpression.of(Expression.class, 2)), true);
+        // register json functions
+        registerJsonFunctionTransformers();
+        // register string functions
+        registerStringFunctionTransformers();
+        // register date functions
+        registerDateFunctionTransformers();
+        // register numeric functions
+        registerNumericFunctionTransformers();
+        // TODO: add other function transformer
+    }
+
+    @Override
+    protected void registerComplexTransformers() {
+        DateTruncFnCallTransformer dateTruncFnCallTransformer = new DateTruncFnCallTransformer();
+        doRegister(dateTruncFnCallTransformer.getSourceFnName(), dateTruncFnCallTransformer);
+        // TODO: add other complex function transformer
+    }
 
-        doRegister("get_json_object", 2, "json_extract",
+    private void registerJsonFunctionTransformers() {
+        doRegister("get_json_object", "json_extract",
                 Lists.newArrayList(
                         PlaceholderExpression.of(Expression.class, 1),
-                        PlaceholderExpression.of(Expression.class, 2)), false);
+                        PlaceholderExpression.of(Expression.class, 2)));
+    }
 
-        doRegister("split", 2, "split_by_string",
+    private void registerStringFunctionTransformers() {
+        doRegister("split", "split_by_string",
                 Lists.newArrayList(
                         PlaceholderExpression.of(Expression.class, 1),
-                        PlaceholderExpression.of(Expression.class, 2)), true);
-        doRegister("split", 2, "split_by_string",
+                        PlaceholderExpression.of(Expression.class, 2)));
+    }
+
+    private void registerDateFunctionTransformers() {
+        // spark-sql support to_date(date_str, fmt) function but doris only support to_date(date_str)
+        // here try to compat with this situation by using str_to_date(str, fmt),
+        // this function support the following three formats which can handle the mainly situations:
+        //  1. yyyyMMdd
+        //  2. yyyy-MM-dd
+        //  3. yyyy-MM-dd HH:mm:ss
+        doRegister("to_date", "str_to_date",
                 Lists.newArrayList(
                         PlaceholderExpression.of(Expression.class, 1),
-                        PlaceholderExpression.of(Expression.class, 2)), false);
-        // TODO: add other function transformer
+                        PlaceholderExpression.of(Expression.class, 2)));
     }
 
-    @Override
-    protected void registerComplexTransformers() {
-        // TODO: add other complex function transformer
+    private void registerNumericFunctionTransformers() {
+        doRegister("mean", "avg",
+                Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)));
     }
 
     static class SingletonHolder {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java
index b59a9327bdedc0..986f9996f90a9b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/DateDiffFnCallTransformer.java
@@ -18,6 +18,7 @@
 package org.apache.doris.nereids.parser.trino;
 
 import org.apache.doris.nereids.analyzer.UnboundFunction;
+import org.apache.doris.nereids.parser.ComplexFnCallTransformer;
 import org.apache.doris.nereids.parser.ParserContext;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.functions.Function;
@@ -30,7 +31,7 @@
 /**
  * DateDiff complex function transformer
  */
-public class DateDiffFnCallTransformer extends ComplexTrinoFnCallTransformer {
+public class DateDiffFnCallTransformer extends ComplexFnCallTransformer {
 
     private static final String SECOND = "second";
     private static final String HOUR = "hour";
@@ -45,15 +46,12 @@ public String getSourceFnName() {
     @Override
     protected boolean check(String sourceFnName, List<Expression> sourceFnTransformedArguments,
             ParserContext context) {
-        return getSourceFnName().equalsIgnoreCase(sourceFnName);
+        return getSourceFnName().equalsIgnoreCase(sourceFnName) && (sourceFnTransformedArguments.size() == 3);
     }
 
     @Override
     protected Function transform(String sourceFnName, List<Expression> sourceFnTransformedArguments,
             ParserContext context) {
-        if (sourceFnTransformedArguments.size() != 3) {
-            return null;
-        }
         VarcharLiteral diffGranularity = (VarcharLiteral) sourceFnTransformedArguments.get(0);
         if (SECOND.equals(diffGranularity.getValue())) {
             return new UnboundFunction(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java
index 883cb1cd132e9c..662439cfc9d341 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/trino/TrinoFnCallTransformers.java
@@ -18,14 +18,13 @@
 package org.apache.doris.nereids.parser.trino;
 
 import org.apache.doris.nereids.analyzer.PlaceholderExpression;
-import org.apache.doris.nereids.parser.AbstractFnCallTransformer;
 import org.apache.doris.nereids.parser.AbstractFnCallTransformers;
 import org.apache.doris.nereids.trees.expressions.Expression;
 
 import com.google.common.collect.Lists;
 
 /**
- * The builder and factory for trino {@link AbstractFnCallTransformer},
+ * The builder and factory for trino function call transformers,
  * and supply transform facade ability.
  */
 public class TrinoFnCallTransformers extends AbstractFnCallTransformers {
@@ -47,8 +46,8 @@ protected void registerComplexTransformers() {
     }
 
     protected void registerStringFunctionTransformer() {
-        doRegister("codepoint", 1, "ascii",
-                Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)), false);
+        doRegister("codepoint", "ascii",
+                Lists.newArrayList(PlaceholderExpression.of(Expression.class, 1)));
         // TODO: add other string function transformer
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/spark/FnTransformTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/spark/FnTransformTest.java
index f652e171280617..a4e972c577ae5a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/spark/FnTransformTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/spark/FnTransformTest.java
@@ -21,6 +21,7 @@
 import org.apache.doris.nereids.parser.ParserTestBase;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 
+import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
@@ -30,26 +31,59 @@
 public class FnTransformTest extends ParserTestBase {
 
     @Test
-    public void testCommonFnTransform() {
-        NereidsParser nereidsParser = new NereidsParser();
+    public void testCommonFnTransformers() {
+        // test json functions
+        testFunction("SELECT json_extract('{\"c1\": 1}', '$.c1') as b FROM t",
+                    "SELECT get_json_object('{\"c1\": 1}', '$.c1') as b FROM t",
+                    "json_extract('{\"c1\": 1}', '$.c1')");
 
-        String sql1 = "SELECT json_extract('{\"a\": 1}', '$.a') as b FROM t";
-        String dialectSql1 = "SELECT get_json_object('{\"a\": 1}', '$.a') as b FROM t";
-        LogicalPlan logicalPlan1 = nereidsParser.parseSingle(sql1);
-        LogicalPlan dialectLogicalPlan1 = nereidsParser.parseSingle(dialectSql1,
-                    new SparkSql3LogicalPlanBuilder());
-        Assertions.assertEquals(dialectLogicalPlan1, logicalPlan1);
-        Assertions.assertTrue(dialectLogicalPlan1.child(0).toString().toLowerCase()
-                    .contains("json_extract('{\"a\": 1}', '$.a')"));
-
-        String sql2 = "SELECT json_extract(a, '$.a') as b FROM t";
-        String dialectSql2 = "SELECT get_json_object(a, '$.a') as b FROM t";
-        LogicalPlan logicalPlan2 = nereidsParser.parseSingle(sql2);
-        LogicalPlan dialectLogicalPlan2 = nereidsParser.parseSingle(dialectSql2,
-                new SparkSql3LogicalPlanBuilder());
-        Assertions.assertEquals(dialectLogicalPlan2, logicalPlan2);
-        Assertions.assertTrue(dialectLogicalPlan2.child(0).toString().toLowerCase()
-                    .contains("json_extract('a, '$.a')"));
+        testFunction("SELECT json_extract(c1, '$.c1') as b FROM t",
+                    "SELECT get_json_object(c1, '$.c1') as b FROM t",
+                    "json_extract('c1, '$.c1')");
+
+        // test string functions
+        testFunction("SELECT str_to_date('2023-12-16', 'yyyy-MM-dd') as b FROM t",
+                    "SELECT to_date('2023-12-16', 'yyyy-MM-dd') as b FROM t",
+                    "str_to_date('2023-12-16', 'yyyy-MM-dd')");
+        testFunction("SELECT str_to_date(c1, 'yyyy-MM-dd') as b FROM t",
+                "SELECT to_date(c1, 'yyyy-MM-dd') as b FROM t",
+                "str_to_date('c1, 'yyyy-MM-dd')");
+
+        testFunction("SELECT date_trunc('2023-12-16', 'YEAR') as a FROM t",
+                    "SELECT trunc('2023-12-16', 'YEAR') as a FROM t",
+                    "date_trunc('2023-12-16', 'YEAR')");
+        testFunction("SELECT date_trunc(c1, 'YEAR') as a FROM t",
+                "SELECT trunc(c1, 'YEAR') as a FROM t",
+                "date_trunc('c1, 'YEAR')");
+
+        testFunction("SELECT date_trunc('2023-12-16', 'YEAR') as a FROM t",
+                    "SELECT trunc('2023-12-16', 'YY') as a FROM t",
+                    "date_trunc('2023-12-16', 'YEAR')");
+        testFunction("SELECT date_trunc(c1, 'YEAR') as a FROM t",
+                "SELECT trunc(c1, 'YY') as a FROM t",
+                "date_trunc('c1, 'YEAR')");
+
+        testFunction("SELECT date_trunc('2023-12-16', 'MONTH') as a FROM t",
+                    "SELECT trunc('2023-12-16', 'MON') as a FROM t",
+                    "date_trunc('2023-12-16', 'MONTH')");
+        testFunction("SELECT date_trunc(c1, 'MONTH') as a FROM t",
+                "SELECT trunc(c1, 'MON') as a FROM t",
+                "date_trunc('c1, 'MONTH')");
+
+        // test numeric functions
+        testFunction("SELECT avg(c1) as a from t",
+                    "SELECT mean(c1) as a from t",
+                    "avg('c1)");
     }
 
+    private void testFunction(String sql, String dialectSql, String expectLogicalPlanStr) {
+        NereidsParser nereidsParser = new NereidsParser();
+        LogicalPlan logicalPlan = nereidsParser.parseSingle(sql);
+        LogicalPlan dialectLogicalPlan = nereidsParser.parseSingle(dialectSql,
+                new SparkSql3LogicalPlanBuilder());
+        Assertions.assertEquals(dialectLogicalPlan, logicalPlan);
+        String dialectLogicalPlanStr = dialectLogicalPlan.child(0).toString().toLowerCase();
+        System.out.println("dialectLogicalPlanStr: " + dialectLogicalPlanStr);
+        Assertions.assertTrue(StringUtils.containsIgnoreCase(dialectLogicalPlanStr, expectLogicalPlanStr));
+    }
 }

From 41d907912a668f3de934b52593ec41463e59da3b Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Sat, 30 Dec 2023 01:29:28 +0800
Subject: [PATCH 192/299] [improve](move-memtable) add fault injection in load
 stream stub (#29105)

---
 be/src/io/fs/stream_sink_file_writer.cpp      |  20 ++++
 be/src/vec/sink/load_stream_stub.cpp          |   2 +
 be/src/vec/sink/load_stream_stub_pool.cpp     |   6 ++
 ..._load_stream_stub_failure_injection.groovy | 102 ++++++++++++++++++
 4 files changed, 130 insertions(+)
 create mode 100644 regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy

diff --git a/be/src/io/fs/stream_sink_file_writer.cpp b/be/src/io/fs/stream_sink_file_writer.cpp
index 2fce7c6baa9c64..74f34b44e6d560 100644
--- a/be/src/io/fs/stream_sink_file_writer.cpp
+++ b/be/src/io/fs/stream_sink_file_writer.cpp
@@ -51,10 +51,30 @@ Status StreamSinkFileWriter::appendv(const Slice* data, size_t data_cnt) {
                << ", data_length: " << bytes_req;
 
     std::span<const Slice> slices {data, data_cnt};
+    size_t stream_index = 0;
     bool ok = false;
     for (auto& stream : _streams) {
         auto st = stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id,
                                       _bytes_appended, slices);
+        DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", {
+            if (stream_index >= 2) {
+                st = Status::InternalError("stream sink file writer append data failed");
+            } else {
+                stream_index++;
+            }
+        });
+
+        DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", {
+            if (stream_index >= 1) {
+                st = Status::InternalError("stream sink file writer append data failed");
+            } else {
+                stream_index++;
+            }
+        });
+
+        DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", {
+            st = Status::InternalError("stream sink file writer append data failed");
+        });
         ok = ok || st.ok();
     }
     if (!ok) {
diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 7835876596701e..4b557cae93d06e 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -21,6 +21,7 @@
 
 #include "olap/rowset/rowset_writer.h"
 #include "util/brpc_client_cache.h"
+#include "util/debug_points.h"
 #include "util/network_util.h"
 #include "util/thrift_util.h"
 #include "util/uid_util.h"
@@ -330,6 +331,7 @@ Status LoadStreamStub::_send_with_retry(butil::IOBuf& buf) {
             SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->orphan_mem_tracker());
             ret = brpc::StreamWrite(_stream_id, buf);
         }
+        DBUG_EXECUTE_IF("LoadStreamStub._send_with_retry.stream_write_failed", { ret = EPIPE; });
         switch (ret) {
         case 0:
             return Status::OK();
diff --git a/be/src/vec/sink/load_stream_stub_pool.cpp b/be/src/vec/sink/load_stream_stub_pool.cpp
index bc19bad532c62e..f7eeb96bb04ec1 100644
--- a/be/src/vec/sink/load_stream_stub_pool.cpp
+++ b/be/src/vec/sink/load_stream_stub_pool.cpp
@@ -17,6 +17,7 @@
 
 #include "vec/sink/load_stream_stub_pool.h"
 
+#include "util/debug_points.h"
 #include "vec/sink/load_stream_stub.h"
 
 namespace doris {
@@ -29,16 +30,21 @@ LoadStreams::LoadStreams(UniqueId load_id, int64_t dst_id, int num_use, LoadStre
 
 void LoadStreams::release() {
     int num_use = --_use_cnt;
+    DBUG_EXECUTE_IF("LoadStreams.release.keeping_streams", { num_use = 1; });
     if (num_use == 0) {
         LOG(INFO) << "releasing streams, load_id=" << _load_id << ", dst_id=" << _dst_id;
         for (auto& stream : _streams) {
             auto st = stream->close_stream();
+            DBUG_EXECUTE_IF("LoadStreams.release.close_stream_failed",
+                            { st = Status::InternalError("stream close failed"); });
             if (!st.ok()) {
                 LOG(WARNING) << "close stream failed " << st;
             }
         }
         for (auto& stream : _streams) {
             auto st = stream->close_wait();
+            DBUG_EXECUTE_IF("LoadStreams.release.close_wait_failed",
+                            { st = Status::InternalError("stream close wait timeout"); });
             if (!st.ok()) {
                 LOG(WARNING) << "close wait failed " << st;
             }
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
new file mode 100644
index 00000000000000..ed02d3e7ac1134
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("test_stream_stub_fault_injection", "nonConcurrent") {
+    sql """ set enable_memtable_on_sink_node=true """
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """
+        CREATE TABLE IF NOT EXISTS `baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+    sql """
+        CREATE TABLE IF NOT EXISTS `test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    GetDebugPoint().clearDebugPointsForAllBEs()
+    streamLoad {
+        table "baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    def load_with_injection = { injection, error_msg->
+        try {
+            GetDebugPoint().enableDebugPointForAllBEs(injection)
+            sql "insert into test select * from baseall where k1 <= 3"
+        } catch(Exception e) {
+            logger.info(e.getMessage())
+            assertTrue(e.getMessage().contains(error_msg))
+        } finally {
+            GetDebugPoint().disableDebugPointForAllBEs(injection)
+        }
+    }
+
+    // StreamSinkFileWriter appendv write segment failed one replica
+    load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", "")
+    // StreamSinkFileWriter appendv write segment failed two replica
+    load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", "")
+    // StreamSinkFileWriter appendv write segment failed all replica
+    load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "stream sink file writer append data failed")
+    // LoadStreams stream wait failed
+    load_with_injection("LoadStreamStub._send_with_retry.stream_write_failed", "StreamWrite failed, err=32")
+    // LoadStreams keeping stream when release
+    load_with_injection("LoadStreams.release.keeping_streams", "")
+    // LoadStreams close stream failed
+    load_with_injection("LoadStreams.release.close_stream_failed", "")
+    // LoadStreams close wait failed
+    load_with_injection("LoadStreams.release.close_wait_failed", "")
+
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """ set enable_memtable_on_sink_node=false """
+}
\ No newline at end of file

From e62857e5a4bd92638642451b9f6a6a6f5f746d0d Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Sat, 30 Dec 2023 01:31:05 +0800
Subject: [PATCH 193/299] [improve](move-memtable) return error when back
 pressure cost too much time (#29327)

---
 be/src/common/config.cpp                      |   2 +
 be/src/common/config.h                        |   2 +
 be/src/runtime/load_stream.cpp                |  33 ++++-
 ...tream_back_pressure_fault_injection.groovy | 137 ++++++++++++++++++
 4 files changed, 172 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 120ca28be7a26a..785aab61c97494 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -784,6 +784,8 @@ DEFINE_Int32(load_stream_messages_in_batch, "128");
 DEFINE_Int32(load_stream_eagain_wait_seconds, "60");
 // max tasks per flush token in load stream
 DEFINE_Int32(load_stream_flush_token_max_tasks, "15");
+// max wait flush token time in load stream
+DEFINE_Int32(load_stream_max_wait_flush_token_time_ms, "600000");
 
 // max send batch parallelism for OlapTableSink
 // The value set by the user for send_batch_parallelism is not allowed to exceed max_send_batch_parallelism_per_job,
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 68c02941cdcb08..1cbfbffb4cbf87 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -837,6 +837,8 @@ DECLARE_Int32(load_stream_messages_in_batch);
 DECLARE_Int32(load_stream_eagain_wait_seconds);
 // max tasks per flush token in load stream
 DECLARE_Int32(load_stream_flush_token_max_tasks);
+// max wait flush token time in load stream
+DECLARE_Int32(load_stream_max_wait_flush_token_time_ms);
 
 // max send batch parallelism for OlapTableSink
 // The value set by the user for send_batch_parallelism is not allowed to exceed max_send_batch_parallelism_per_job,
diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index d2ea41f1c35762..8049f41893c9f8 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -147,9 +147,22 @@ Status TabletStream::append_data(const PStreamHeader& header, butil::IOBuf* data
         }
     };
     auto& flush_token = _flush_tokens[new_segid % _flush_tokens.size()];
+    auto load_stream_flush_token_max_tasks = config::load_stream_flush_token_max_tasks;
+    auto load_stream_max_wait_flush_token_time_ms =
+            config::load_stream_max_wait_flush_token_time_ms;
+    DBUG_EXECUTE_IF("TabletStream.append_data.long_wait", {
+        load_stream_flush_token_max_tasks = 0;
+        load_stream_max_wait_flush_token_time_ms = 1000;
+    });
     MonotonicStopWatch timer;
     timer.start();
-    while (flush_token->num_tasks() >= config::load_stream_flush_token_max_tasks) {
+    while (flush_token->num_tasks() >= load_stream_flush_token_max_tasks) {
+        if (timer.elapsed_time() / 1000 / 1000 >= load_stream_max_wait_flush_token_time_ms) {
+            return Status::Error<true>(
+                    "wait flush token back pressure time is more than "
+                    "load_stream_max_wait_flush_token_time {}",
+                    load_stream_max_wait_flush_token_time_ms);
+        }
         bthread_usleep(2 * 1000); // 2ms
     }
     timer.stop();
@@ -193,9 +206,25 @@ Status TabletStream::add_segment(const PStreamHeader& header, butil::IOBuf* data
         }
     };
     auto& flush_token = _flush_tokens[new_segid % _flush_tokens.size()];
-    while (flush_token->num_tasks() >= config::load_stream_flush_token_max_tasks) {
+    auto load_stream_flush_token_max_tasks = config::load_stream_flush_token_max_tasks;
+    auto load_stream_max_wait_flush_token_time_ms =
+            config::load_stream_max_wait_flush_token_time_ms;
+    DBUG_EXECUTE_IF("TabletStream.add_segment.long_wait", {
+        load_stream_flush_token_max_tasks = 0;
+        load_stream_max_wait_flush_token_time_ms = 1000;
+    });
+    MonotonicStopWatch timer;
+    timer.start();
+    while (flush_token->num_tasks() >= load_stream_flush_token_max_tasks) {
+        if (timer.elapsed_time() / 1000 / 1000 >= load_stream_max_wait_flush_token_time_ms) {
+            return Status::Error<true>(
+                    "wait flush token back pressure time is more than "
+                    "load_stream_max_wait_flush_token_time {}",
+                    load_stream_max_wait_flush_token_time_ms);
+        }
         bthread_usleep(10 * 1000); // 10ms
     }
+    timer.stop();
     return flush_token->submit_func(add_segment_func);
 }
 
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy
new file mode 100644
index 00000000000000..57825c80d17d0e
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_back_pressure_fault_injection.groovy
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("test_load_stream_back_pressure_fault_injection", "nonConcurrent") {
+    sql """ set enable_memtable_on_sink_node=true """
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """
+        CREATE TABLE IF NOT EXISTS `baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+    sql """
+        CREATE TABLE IF NOT EXISTS `test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    GetDebugPoint().clearDebugPointsForAllBEs()
+    streamLoad {
+        table "baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    try {
+        GetDebugPoint().enableDebugPointForAllBEs("TabletStream.append_data.long_wait")
+        def thread1 = new Thread({
+            try {
+                def res = sql "insert into test select * from baseall where k1 <= 3"
+                logger.info(res.toString())
+            } catch(Exception e) {
+                logger.info(e.getMessage())
+                assertTrue(e.getMessage().contains("Communications link failure"))
+            } finally {
+                GetDebugPoint().disableDebugPointForAllBEs("TabletStream.append_data.long_wait")
+            }
+        })
+        thread1.start()
+
+        sleep(1000)
+
+        def processList = sql "show processlist"
+        logger.info(processList.toString())
+        processList.each { item ->
+            logger.info(item[1].toString())
+            logger.info(item[11].toString())
+            if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
+                def res = sql "kill ${item[1]}"
+                logger.info(res.toString())
+            }
+        }
+    } catch(Exception e) {
+        logger.info(e.getMessage())
+    }
+
+    try {
+        GetDebugPoint().enableDebugPointForAllBEs("TabletStream.add_segment.long_wait")
+        def thread1 = new Thread({
+            try {
+                def res = sql "insert into test select * from baseall where k1 <= 3"
+                logger.info(res.toString())
+            } catch(Exception e) {
+                logger.info(e.getMessage())
+                assertTrue(e.getMessage().contains("Communications link failure"))
+            } finally {
+                GetDebugPoint().disableDebugPointForAllBEs("TabletStream.add_segment.long_wait")
+            }
+        })
+        thread1.start()
+
+        sleep(1000)
+
+        def processList = sql "show processlist"
+        logger.info(processList.toString())
+        processList.each { item ->
+            logger.info(item[1].toString())
+            logger.info(item[11].toString())
+            if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
+                def res = sql "kill ${item[1]}"
+                logger.info(res.toString())
+            }
+        }
+    } catch(Exception e) {
+        logger.info(e.getMessage())
+    }
+
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """ set enable_memtable_on_sink_node=false """
+}
\ No newline at end of file

From 5bc72bda78282eed4958435669dff23fc8b8bc52 Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Sat, 30 Dec 2023 01:32:26 +0800
Subject: [PATCH 194/299] [improve](move-memtable) add fault injection in
 writer v2 (#29177)

---
 be/src/olap/delta_writer_v2.cpp               |  2 +
 be/src/olap/memtable_flush_executor.cpp       |  2 +-
 be/src/vec/sink/writer/vtablet_writer_v2.cpp  | 14 +++
 .../pipeline/p0/conf/regression-conf.groovy   |  2 +-
 .../test_build_index_fault.groovy             |  2 +-
 ...st_mem_gc_when_load_fault_injection.groovy |  2 +-
 .../test_memtable_flush_fault.groovy          |  2 +-
 .../test_segcompaction_fault_injection.groovy | 37 ++++---
 .../test_writer_v2_fault_injection.groovy     | 98 +++++++++++++++++++
 9 files changed, 137 insertions(+), 24 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy

diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index 006e8a15a6ffc7..643f1837982a0f 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -53,6 +53,7 @@
 #include "runtime/exec_env.h"
 #include "service/backend_options.h"
 #include "util/brpc_client_cache.h"
+#include "util/debug_points.h"
 #include "util/mem_info.h"
 #include "util/ref_count_closure.h"
 #include "util/stopwatch.hpp"
@@ -102,6 +103,7 @@ Status DeltaWriterV2::init() {
         return Status::OK();
     }
     // build tablet schema in request level
+    DBUG_EXECUTE_IF("DeltaWriterV2.init.stream_size", { _streams.clear(); });
     if (_streams.size() == 0 || _streams[0]->tablet_schema(_req.index_id) == nullptr) {
         return Status::InternalError("failed to find tablet schema for {}", _req.index_id);
     }
diff --git a/be/src/olap/memtable_flush_executor.cpp b/be/src/olap/memtable_flush_executor.cpp
index ff2d16bd1f508f..7190597b54cf1f 100644
--- a/be/src/olap/memtable_flush_executor.cpp
+++ b/be/src/olap/memtable_flush_executor.cpp
@@ -81,7 +81,7 @@ Status FlushToken::submit(std::unique_ptr<MemTable> mem_table) {
     {
         std::shared_lock rdlk(_flush_status_lock);
         DBUG_EXECUTE_IF("FlushToken.submit_flush_error", {
-            _flush_status = Status::IOError("dbug_be_memtable_submit_flush_error");
+            _flush_status = Status::IOError<false>("dbug_be_memtable_submit_flush_error");
         });
         if (!_flush_status.ok()) {
             return _flush_status;
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index f68d82911ce8a9..4972a397f701e3 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -196,10 +196,17 @@ Status VTabletWriterV2::_init(RuntimeState* state, RuntimeProfile* profile) {
 
     // get table's tuple descriptor
     _output_tuple_desc = state->desc_tbl().get_tuple_descriptor(_tuple_desc_id);
+    DBUG_EXECUTE_IF("VTabletWriterV2._init._output_tuple_desc_null",
+                    { _output_tuple_desc = nullptr; });
     if (_output_tuple_desc == nullptr) {
         return Status::InternalError("unknown destination tuple descriptor, id = {}",
                                      _tuple_desc_id);
     }
+    DBUG_EXECUTE_IF("VTabletWriterV2._init._vec_output_expr_ctxs_not_equal_output_tuple_slot", {
+        return Status::InvalidArgument(
+                "output_tuple_slot_num {} should be equal to output_expr_num {}",
+                _output_tuple_desc->slots().size() + 1, _vec_output_expr_ctxs.size());
+    });
     if (_vec_output_expr_ctxs.size() > 0 &&
         _output_tuple_desc->slots().size() != _vec_output_expr_ctxs.size()) {
         LOG(WARNING) << "output tuple slot num should be equal to num of output exprs, "
@@ -264,6 +271,8 @@ Status VTabletWriterV2::_open_streams(int64_t src_id) {
 Status VTabletWriterV2::_open_streams_to_backend(int64_t dst_id,
                                                  ::doris::stream_load::LoadStreams& streams) {
     const auto* node_info = _nodes_info->find_node(dst_id);
+    DBUG_EXECUTE_IF("VTabletWriterV2._open_streams_to_backend.node_info_null",
+                    { node_info = nullptr; });
     if (node_info == nullptr) {
         return Status::InternalError("Unknown node {} in tablet location", dst_id);
     }
@@ -289,6 +298,8 @@ Status VTabletWriterV2::_build_tablet_node_mapping() {
         for (const auto& index : partition->indexes) {
             for (const auto& tablet_id : index.tablets) {
                 auto tablet_location = _location->find_tablet(tablet_id);
+                DBUG_EXECUTE_IF("VTabletWriterV2._build_tablet_node_mapping.tablet_location_null",
+                                { tablet_location = nullptr; });
                 if (tablet_location == nullptr) {
                     return Status::InternalError("unknown tablet location, tablet id = {}",
                                                  tablet_id);
@@ -338,6 +349,7 @@ void VTabletWriterV2::_generate_rows_for_tablet(std::vector<RowPartTabletIds>& r
 Status VTabletWriterV2::_select_streams(int64_t tablet_id, int64_t partition_id, int64_t index_id,
                                         Streams& streams) {
     const auto* location = _location->find_tablet(tablet_id);
+    DBUG_EXECUTE_IF("VTabletWriterV2._select_streams.location_null", { location = nullptr; });
     if (location == nullptr) {
         return Status::InternalError("unknown tablet location, tablet id = {}", tablet_id);
     }
@@ -484,6 +496,8 @@ Status VTabletWriterV2::close(Status exec_status) {
         status = _send_new_partition_batch();
     }
 
+    DBUG_EXECUTE_IF("VTabletWriterV2.close.cancel",
+                    { status = Status::InternalError("load cancel"); });
     if (status.ok()) {
         // only if status is ok can we call this _profile->total_time_counter().
         // if status is not ok, this sink may not be prepared, so that _profile is null
diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy
index b28b2e64004148..849a1298301e15 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -58,7 +58,7 @@ excludeGroups = ""
 excludeSuites = "test_profile,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_information_schema_external"
 
 // this directories will not be executed
-excludeDirectories = "workload_manager_p1,fault_injection_p0,nereids_rules_p0/subquery"
+excludeDirectories = "workload_manager_p1,nereids_rules_p0/subquery"
 
 customConf1 = "test_custom_conf_value"
 
diff --git a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy
index 9ddf13322c58f0..99f1173acfaaa2 100644
--- a/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy
+++ b/regression-test/suites/fault_injection_p0/test_build_index_fault.groovy
@@ -16,7 +16,7 @@
 // under the License.
 
 
-suite("test_build_index_fault", "inverted_index"){
+suite("test_build_index_fault", "inverted_index, nonConcurrent"){
     // prepare test table
     def timeout = 60000
     def delta_time = 1000
diff --git a/regression-test/suites/fault_injection_p0/test_mem_gc_when_load_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_mem_gc_when_load_fault_injection.groovy
index 99ae467ff4d246..852566c0e05fb3 100644
--- a/regression-test/suites/fault_injection_p0/test_mem_gc_when_load_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_mem_gc_when_load_fault_injection.groovy
@@ -18,7 +18,7 @@
 import org.codehaus.groovy.runtime.IOGroovyMethods
 import org.apache.doris.regression.util.Http
 
-suite("mem_gc_when_load") {
+suite("mem_gc_when_load", "nonConcurrent") {
     // init query case data
     sql """
         CREATE TABLE IF NOT EXISTS `baseall` (
diff --git a/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy b/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy
index e1a4a1a3a76a71..a1a7be381ec164 100644
--- a/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy
+++ b/regression-test/suites/fault_injection_p0/test_memtable_flush_fault.groovy
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("test_memtable_flush_fault") {
+suite("test_memtable_flush_fault", "nonConcurrent") {
     GetDebugPoint().clearDebugPointsForAllBEs()
     def testTable = "test_memtable_flush_fault"
     sql """ DROP TABLE IF EXISTS ${testTable}"""
diff --git a/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy
index a4ded1a58e0f3d..a850111dee85fa 100644
--- a/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_segcompaction_fault_injection.groovy
@@ -18,26 +18,25 @@
 import org.codehaus.groovy.runtime.IOGroovyMethods
 import org.apache.doris.regression.util.Http
 
-def tableName = "segcompaction_correctness_test"
-def create_table_sql = """
-            CREATE TABLE IF NOT EXISTS ${tableName} (
-                `col_0` BIGINT NOT NULL,`col_1` VARCHAR(20),`col_2` VARCHAR(20),`col_3` VARCHAR(20),`col_4` VARCHAR(20),
-                `col_5` VARCHAR(20),`col_6` VARCHAR(20),`col_7` VARCHAR(20),`col_8` VARCHAR(20),`col_9` VARCHAR(20),
-                `col_10` VARCHAR(20),`col_11` VARCHAR(20),`col_12` VARCHAR(20),`col_13` VARCHAR(20),`col_14` VARCHAR(20),
-                `col_15` VARCHAR(20),`col_16` VARCHAR(20),`col_17` VARCHAR(20),`col_18` VARCHAR(20),`col_19` VARCHAR(20),
-                `col_20` VARCHAR(20),`col_21` VARCHAR(20),`col_22` VARCHAR(20),`col_23` VARCHAR(20),`col_24` VARCHAR(20),
-                `col_25` VARCHAR(20),`col_26` VARCHAR(20),`col_27` VARCHAR(20),`col_28` VARCHAR(20),`col_29` VARCHAR(20),
-                `col_30` VARCHAR(20),`col_31` VARCHAR(20),`col_32` VARCHAR(20),`col_33` VARCHAR(20),`col_34` VARCHAR(20),
-                `col_35` VARCHAR(20),`col_36` VARCHAR(20),`col_37` VARCHAR(20),`col_38` VARCHAR(20),`col_39` VARCHAR(20),
-                `col_40` VARCHAR(20),`col_41` VARCHAR(20),`col_42` VARCHAR(20),`col_43` VARCHAR(20),`col_44` VARCHAR(20),
-                `col_45` VARCHAR(20),`col_46` VARCHAR(20),`col_47` VARCHAR(20),`col_48` VARCHAR(20),`col_49` VARCHAR(20)
-                )
-            DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1
-            PROPERTIES ( "replication_num" = "1" );
-        """
-def columns = "col_0, col_1, col_2, col_3, col_4, col_5, col_6, col_7, col_8, col_9, col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, col_18, col_19, col_20, col_21, col_22, col_23, col_24, col_25, col_26, col_27, col_28, col_29, col_30, col_31, col_32, col_33, col_34, col_35, col_36, col_37, col_38, col_39, col_40, col_41, col_42, col_43, col_44, col_45, col_46, col_47, col_48, col_49"
-
 suite("test_segcompaction_correctness", "nonConcurrent") {
+    def tableName = "segcompaction_correctness_test"
+    def create_table_sql = """
+                CREATE TABLE IF NOT EXISTS ${tableName} (
+                    `col_0` BIGINT NOT NULL,`col_1` VARCHAR(20),`col_2` VARCHAR(20),`col_3` VARCHAR(20),`col_4` VARCHAR(20),
+                    `col_5` VARCHAR(20),`col_6` VARCHAR(20),`col_7` VARCHAR(20),`col_8` VARCHAR(20),`col_9` VARCHAR(20),
+                    `col_10` VARCHAR(20),`col_11` VARCHAR(20),`col_12` VARCHAR(20),`col_13` VARCHAR(20),`col_14` VARCHAR(20),
+                    `col_15` VARCHAR(20),`col_16` VARCHAR(20),`col_17` VARCHAR(20),`col_18` VARCHAR(20),`col_19` VARCHAR(20),
+                    `col_20` VARCHAR(20),`col_21` VARCHAR(20),`col_22` VARCHAR(20),`col_23` VARCHAR(20),`col_24` VARCHAR(20),
+                    `col_25` VARCHAR(20),`col_26` VARCHAR(20),`col_27` VARCHAR(20),`col_28` VARCHAR(20),`col_29` VARCHAR(20),
+                    `col_30` VARCHAR(20),`col_31` VARCHAR(20),`col_32` VARCHAR(20),`col_33` VARCHAR(20),`col_34` VARCHAR(20),
+                    `col_35` VARCHAR(20),`col_36` VARCHAR(20),`col_37` VARCHAR(20),`col_38` VARCHAR(20),`col_39` VARCHAR(20),
+                    `col_40` VARCHAR(20),`col_41` VARCHAR(20),`col_42` VARCHAR(20),`col_43` VARCHAR(20),`col_44` VARCHAR(20),
+                    `col_45` VARCHAR(20),`col_46` VARCHAR(20),`col_47` VARCHAR(20),`col_48` VARCHAR(20),`col_49` VARCHAR(20)
+                    )
+                DUPLICATE KEY(`col_0`) DISTRIBUTED BY HASH(`col_0`) BUCKETS 1
+                PROPERTIES ( "replication_num" = "1" );
+            """
+    def columns = "col_0, col_1, col_2, col_3, col_4, col_5, col_6, col_7, col_8, col_9, col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, col_18, col_19, col_20, col_21, col_22, col_23, col_24, col_25, col_26, col_27, col_28, col_29, col_30, col_31, col_32, col_33, col_34, col_35, col_36, col_37, col_38, col_39, col_40, col_41, col_42, col_43, col_44, col_45, col_46, col_47, col_48, col_49"
     def runLoadWithSegcompaction = {
         String ak = getS3AK()
         String sk = getS3SK()
diff --git a/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy
new file mode 100644
index 00000000000000..e6e5758b2b3a6a
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_writer_v2_fault_injection.groovy
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("test_writer_v2_fault_injection", "nonConcurrent") {
+     sql """ set enable_memtable_on_sink_node=true """
+    sql """
+        CREATE TABLE IF NOT EXISTS `baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+    sql """
+        CREATE TABLE IF NOT EXISTS `test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    GetDebugPoint().clearDebugPointsForAllBEs()
+    streamLoad {
+        table "baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    def load_with_injection = { injection, error_msg->
+        try {
+            GetDebugPoint().enableDebugPointForAllBEs(injection)
+            sql "insert into test select * from baseall where k1 <= 3"
+        } catch(Exception e) {
+            logger.info(e.getMessage())
+            assertTrue(e.getMessage().contains(error_msg))
+        } finally {
+            GetDebugPoint().disableDebugPointForAllBEs(injection)
+        }
+    }
+
+    // VTabletWriterV2 _output_tuple_desc is null
+    load_with_injection("VTabletWriterV2._init._output_tuple_desc_null", "unknown destination tuple descriptor")
+    // VTabletWriterV2 _vec_output_expr_ctxs not equal _output_tuple_slot
+    load_with_injection("VTabletWriterV2._init._vec_output_expr_ctxs_not_equal_output_tuple_slot", "should be equal to output_expr_num")
+    // VTabletWriterV2 node_info is null
+    load_with_injection("VTabletWriterV2._open_streams_to_backend.node_info_null", "Unknown node")
+    // VTabletWriterV2 tablet_location is null
+    load_with_injection("VTabletWriterV2._build_tablet_node_mapping.tablet_location_null", "unknown tablet location")
+    // VTabletWriterV2 location is null
+    load_with_injection("VTabletWriterV2._select_streams.location_null", "unknown tablet location")
+    // VTabletWriterV2 cancel
+    load_with_injection("VTabletWriterV2.close.cancel", "load cancel")
+    // DeltaWriterV2 stream_size is 0
+    load_with_injection("DeltaWriterV2.init.stream_size", "failed to find tablet schema")
+
+    sql """ set enable_memtable_on_sink_node=false """
+}
\ No newline at end of file

From 5c9e0d906a572a12098ea368598d894699192aa5 Mon Sep 17 00:00:00 2001
From: airborne12 <airborne08@gmail.com>
Date: Sat, 30 Dec 2023 08:36:15 +0800
Subject: [PATCH 195/299] [Fix](segment iterator) fix wrong segment cache
 remove (#29333)

---
 be/src/olap/rowset/segment_v2/segment_iterator.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 6bc01ea0d4be6e..5c30423bb53d44 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -2074,7 +2074,8 @@ Status SegmentIterator::_read_columns_by_rowids(std::vector<ColumnId>& read_colu
 
 Status SegmentIterator::next_batch(vectorized::Block* block) {
     auto status = [&]() { RETURN_IF_CATCH_EXCEPTION({ return _next_batch_internal(block); }); }();
-    if (!status.ok()) {
+    // if rows read by batch is 0, will return end of file, we should not remove segment cache in this situation.
+    if (!status.ok() && !status.is<END_OF_FILE>()) {
         _segment->remove_from_segment_cache();
     }
     return status;

From 56058623f6fa4e11cf735e079570b45d7b4309af Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi02@gmail.com>
Date: Sat, 30 Dec 2023 08:40:03 +0800
Subject: [PATCH 196/299] [fix](expr) Re fix BE core dump while common expr
 filter delete condition column (#29328)

Additional deleted filter condition will be materialized column be at the end of the block,
after _output_column_by_sel_idx will be erase, we not need to filter it,
so erase it from _columns_to_filter in the first next_batch.
Eg:
delete from table where a = 10;
select b from table;
a column only effective in segment iterator, the block from query engine only contain the b column,
so no need to filter a column by expr.
---
 .../rowset/segment_v2/segment_iterator.cpp    | 52 +++++++------------
 .../suites/variant_p0/delete_update.groovy    |  1 +
 2 files changed, 21 insertions(+), 32 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 5c30423bb53d44..0cf8ffeffa13bb 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -1504,8 +1504,6 @@ Status SegmentIterator::_seek_columns(const std::vector<ColumnId>& column_ids, r
 // todo(wb) need a UT here
 Status SegmentIterator::_vec_init_lazy_materialization() {
     _is_pred_column.resize(_schema->columns().size(), false);
-    std::vector<bool> is_pred_column_no_del_condition;
-    is_pred_column_no_del_condition.resize(_schema->columns().size(), false);
 
     // including short/vec/delete pred
     std::set<ColumnId> pred_column_ids;
@@ -1547,7 +1545,6 @@ Status SegmentIterator::_vec_init_lazy_materialization() {
         for (auto predicate : _col_predicates) {
             auto cid = predicate->column_id();
             _is_pred_column[cid] = true;
-            is_pred_column_no_del_condition[cid] = true;
             pred_column_ids.insert(cid);
 
             // check pred using short eval or vec eval
@@ -1601,16 +1598,10 @@ Status SegmentIterator::_vec_init_lazy_materialization() {
         if (!_common_expr_columns.empty()) {
             _is_need_expr_eval = true;
             for (auto cid : _schema->column_ids()) {
-                // pred column also needs to be filtered by expr, exclude delete condition column,
-                // Delete condition column not need to be filtered, query engine does not need it,
-                // after _output_column_by_sel_idx, delete condition materialize column will be erase
-                // at the end of the block.
-                // Eg:
-                //      `delete from table where a = 10;`
-                //      `select b from table;`
-                // a column only effective in segment iterator, the block from query engine only contain the b column,
-                // so no need to filter a column by expr.
-                if (_is_common_expr_column[cid] || is_pred_column_no_del_condition[cid]) {
+                // pred column also needs to be filtered by expr, exclude additional delete condition column.
+                // if delete condition column not in the block, no filter is needed
+                // and will be removed from _columns_to_filter in the first next_batch.
+                if (_is_common_expr_column[cid] || _is_pred_column[cid]) {
                     auto loc = _schema_block_id_map[cid];
                     _columns_to_filter.push_back(loc);
                 }
@@ -2171,6 +2162,22 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
                 _current_return_columns[cid]->reserve(_opts.block_row_max);
             }
         }
+
+        // Additional deleted filter condition will be materialized column be at the end of the block,
+        // after _output_column_by_sel_idx  will be erase, we not need to filter it,
+        // so erase it from _columns_to_filter in the first next_batch.
+        // Eg:
+        //      `delete from table where a = 10;`
+        //      `select b from table;`
+        // a column only effective in segment iterator, the block from query engine only contain the b column,
+        // so no need to filter a column by expr.
+        for (auto it = _columns_to_filter.begin(); it != _columns_to_filter.end();) {
+            if (*it >= block->columns()) {
+                it = _columns_to_filter.erase(it);
+            } else {
+                ++it;
+            }
+        }
     }
     _init_current_block(block, _current_return_columns);
     _converted_column_ids.assign(_schema->columns().size(), 0);
@@ -2185,25 +2192,6 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) {
         _replace_version_col(_current_batch_rows_read);
     }
 
-    // If col >= block->columns(), it means col should not be filtered, there is a BUG.
-    // such as delete condition column was incorrectly put into columns_to_filter,
-    // which is usually at the end of the block. only check during the first next_batch.
-    if (_opts.stats->blocks_load == 0) {
-        for (const auto& col : _columns_to_filter) {
-            if (col >= block->columns()) {
-                std::ostringstream ss;
-                for (const auto& i : _columns_to_filter) {
-                    ss << i << "-";
-                }
-                throw Exception(
-                        ErrorCode::INTERNAL_ERROR,
-                        "filter block column id(index) greater than block->columns(), "
-                        "column id={}, all columns that need filter={}, block columns num={}",
-                        col, ss.str().substr(0, ss.str().length() - 1), block->columns());
-            }
-        }
-    }
-
     _opts.stats->blocks_load += 1;
     _opts.stats->raw_rows_read += _current_batch_rows_read;
 
diff --git a/regression-test/suites/variant_p0/delete_update.groovy b/regression-test/suites/variant_p0/delete_update.groovy
index e0cfcfa82f0fb0..bbd999559b42ee 100644
--- a/regression-test/suites/variant_p0/delete_update.groovy
+++ b/regression-test/suites/variant_p0/delete_update.groovy
@@ -63,6 +63,7 @@ suite("regression_test_variant_delete_and_update", "variant_type"){
     sql """update ${table_name} set v = '{"updated_value" : 1111}' where k = 7"""
     qt_sql "select * from ${table_name} order by k"
 
+    sql """delete from ${table_name} where v = 'xxx' or vs = 'yyy'"""
     sql """delete from ${table_name} where vs = 'xxx' or vs = 'yyy'"""
     qt_sql "select * from ${table_name} order by k"
 

From 2c4e52e44eee8327ba7a575824a8b6f28c143f1e Mon Sep 17 00:00:00 2001
From: qiye <jianliang5669@gmail.com>
Date: Sat, 30 Dec 2023 09:33:26 +0800
Subject: [PATCH 197/299] [fix](es catalog) only es_query function can push
 down to ES (#29320)

Issue Number: close #29318
1. Only push down `es_query` function to ES
2. Add null check where ES query result not have `_source` or `fields` fields.
---
 be/src/exec/es/es_scroll_parser.cpp           |  27 ++--
 .../elasticsearch/scripts/data/data2.json     |   3 +-
 .../elasticsearch/scripts/data/data2_es6.json |   3 +-
 .../elasticsearch/scripts/data/data3.json     |   3 +-
 .../elasticsearch/scripts/data/data3_es6.json |   3 +-
 .../scripts/index/es6_test1.json              |   9 ++
 .../scripts/index/es7_test1.json              |   9 ++
 .../external/elasticsearch/QueryBuilders.java |  16 ++-
 .../external_table_p0/es/test_es_query.out    | 118 +++++++++++++-----
 .../es/test_es_query_no_http_url.out          |   6 +-
 .../external_table_p0/es/test_es_query.groovy |  22 +++-
 11 files changed, 157 insertions(+), 62 deletions(-)

diff --git a/be/src/exec/es/es_scroll_parser.cpp b/be/src/exec/es/es_scroll_parser.cpp
index 726e86ea74ffbc..a1c3c9f0d5ebf6 100644
--- a/be/src/exec/es/es_scroll_parser.cpp
+++ b/be/src/exec/es/es_scroll_parser.cpp
@@ -471,7 +471,13 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc,
     if (obj.HasMember("fields")) {
         pure_doc_value = true;
     }
-    const rapidjson::Value& line = obj.HasMember(FIELD_SOURCE) ? obj[FIELD_SOURCE] : obj["fields"];
+    // obj may be neither have `_source` nor `fields` field.
+    const rapidjson::Value* line = nullptr;
+    if (obj.HasMember(FIELD_SOURCE)) {
+        line = &obj[FIELD_SOURCE];
+    } else if (obj.HasMember("fields")) {
+        line = &obj["fields"];
+    }
 
     for (int i = 0; i < tuple_desc->slots().size(); ++i) {
         const SlotDescriptor* slot_desc = tuple_desc->slots()[i];
@@ -496,17 +502,18 @@ Status ScrollParser::fill_columns(const TupleDescriptor* tuple_desc,
         const char* col_name = pure_doc_value ? docvalue_context.at(slot_desc->col_name()).c_str()
                                               : slot_desc->col_name().c_str();
 
-        rapidjson::Value::ConstMemberIterator itr = line.FindMember(col_name);
-        if (itr == line.MemberEnd() && slot_desc->is_nullable()) {
-            auto nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(col_ptr);
-            nullable_column->insert_data(nullptr, 0);
-            continue;
-        } else if (itr == line.MemberEnd() && !slot_desc->is_nullable()) {
-            std::string details = strings::Substitute(INVALID_NULL_VALUE, col_name);
-            return Status::RuntimeError(details);
+        if (line == nullptr || line->FindMember(col_name) == line->MemberEnd()) {
+            if (slot_desc->is_nullable()) {
+                auto* nullable_column = reinterpret_cast<vectorized::ColumnNullable*>(col_ptr);
+                nullable_column->insert_data(nullptr, 0);
+                continue;
+            } else {
+                std::string details = strings::Substitute(INVALID_NULL_VALUE, col_name);
+                return Status::RuntimeError(details);
+            }
         }
 
-        const rapidjson::Value& col = line[col_name];
+        const rapidjson::Value& col = (*line)[col_name];
 
         PrimitiveType type = slot_desc->type().type;
 
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json
index 5ca56093bd3bd7..792200fcdec8b7 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2.json
@@ -27,5 +27,6 @@
   "c_person": [
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
-  ]
+  ],
+  "message": ""
 }
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
index ba1f8a5f5c94cb..12d42f775fd1ac 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data2_es6.json
@@ -23,5 +23,6 @@
   "c_person": [
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
-  ]
+  ],
+  "message": ""
 }
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json
index 81f7dc749e3c94..c3878f30732f09 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3.json
@@ -27,5 +27,6 @@
   "c_person": [
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
-  ]
+  ],
+  "message": "I'm not null or empty"
 }
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
index cfc3d69dfc21ed..f360b15ecec708 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/data/data3_es6.json
@@ -23,5 +23,6 @@
   "c_person": [
     {"name": "Andy", "age": 18},
     {"name": "Tim", "age": 28}
-  ]
+  ],
+  "message": "I'm not null or empty"
 }
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
index a8612e668c3f2c..97c5f537c5b628 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es6_test1.json
@@ -87,6 +87,15 @@
               "type": "integer"
             }
           }
+        },
+        "message": {
+          "type": "text",
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
         }
       }
     }
diff --git a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json
index 69247490caaf82..604de9964ea1c0 100755
--- a/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json
+++ b/docker/thirdparties/docker-compose/elasticsearch/scripts/index/es7_test1.json
@@ -101,6 +101,15 @@
             "type": "integer"
           }
         }
+      },
+      "message": {
+        "type": "text",
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
       }
     }
   }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java
index 8dc8d4abea9574..350fd3857ea634 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/QueryBuilders.java
@@ -209,11 +209,8 @@ private static QueryBuilder parseFunctionCallExpr(Expr expr) {
         //    }');
         // The first child k1 compatible with expr syntax
         FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
-        if ("esquery".equals(functionCallExpr.getFnName().getFunction())) {
-            String stringValue = functionCallExpr.getChild(1).getStringValue();
-            return new QueryBuilders.EsQueryBuilder(stringValue);
-        }
-        return null;
+        String stringValue = functionCallExpr.getChild(1).getStringValue();
+        return new QueryBuilders.EsQueryBuilder(stringValue);
     }
 
     /**
@@ -276,7 +273,14 @@ public static QueryBuilder toEsDsl(Expr expr, List<Expr> notPushDownList, Map<St
             return parseInPredicate(expr, column, needDateCompat);
         }
         if (expr instanceof FunctionCallExpr) {
-            return parseFunctionCallExpr(expr);
+            FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
+            // current only esquery functionCallExpr can be push down to ES
+            if (!"esquery".equals(functionCallExpr.getFnName().getFunction())) {
+                notPushDownList.add(expr);
+                return null;
+            } else {
+                return parseFunctionCallExpr(expr);
+            }
         }
         return null;
     }
diff --git a/regression-test/data/external_table_p0/es/test_es_query.out b/regression-test/data/external_table_p0/es/test_es_query.out
index 98c74755181fe0..cc8be6d8a36699 100644
--- a/regression-test/data/external_table_p0/es/test_es_query.out
+++ b/regression-test/data/external_table_p0/es/test_es_query.out
@@ -1,9 +1,9 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql01 --
-["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
+["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N
 
 -- !sql02 --
-["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
+["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]	\N
 
 -- !sql03 --
 2022-08-08	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T04:10:10	2022-08-08T20:10:10
@@ -12,27 +12,42 @@
 2022-08-08	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10
 
 -- !sql04 --
-["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
+\N
+\N
+I'm not null or empty
 
 -- !sql05 --
-["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
+
+I'm not null or empty
 
 -- !sql06 --
+\N
+\N
+
+I'm not null or empty
+
+-- !sql20 --
+["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
+
+-- !sql21 --
+["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
+
+-- !sql22 --
 2022-08-08	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T04:10:10	2022-08-08T20:10:10
 2022-08-08	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10	2022-08-09T12:10:10
 2022-08-08	2022-08-10T12:10:10	2022-08-10T12:10:10	2022-08-10T04:10:10	2022-08-10T20:10:10
 2022-08-08	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T12:10:10	2022-08-11T11:10:10
 
 -- !sql_6_02 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
 
 -- !sql_6_03 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
 
 -- !sql_6_04 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
 
 -- !sql_6_05 --
 true	1	128	32768	-1	0	1.0	1	1	1	2020-01-01T00:00	2020-01-01 12:00:00	a	d	192.168.0.1	{"name":"Andy","age":18}
@@ -45,7 +60,7 @@ true	1	128	32768	-1	0	1.0	1	1	1	2020-01-01T00:00	2020-01-01 12:00:00	a	d	192.168
 true	1	128	32768	-1	0	1.0	1	1	1	2020-01-01T00:00	2020-01-01 12:00:00	a	d	192.168.0.1	{"name":"Andy","age":18}
 
 -- !sql_6_07 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
 
 -- !sql_6_08 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
@@ -58,13 +73,13 @@ true	1	128	32768	-1	0	1.0	1	1	1	2020-01-01T00:00	2020-01-01 12:00:00	a	d	192.168
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
 
 -- !sql_6_10 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
 
 -- !sql_6_11 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	text2	4.0	2022-08-08T00:00	2222	2022-08-08T12:10:10
 
 -- !sql_6_12 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	I'm not null or empty	string3	text3_4*5	5.0	2022-08-08T00:00	3333	2022-08-08T20:10:10
 
 -- !sql_6_13 --
 2022-08-08T20:10:10
@@ -75,17 +90,28 @@ true	1	128	32768	-1	0	1.0	1	1	1	2020-01-01T00:00	2020-01-01 12:00:00	a	d	192.168
 -- !sql_6_15 --
 2022-08-08T20:10:10
 
+-- !sql_6_16 --
+\N
+I'm not null or empty
+
+-- !sql_6_17 --
+
+I'm not null or empty
+
+-- !sql_6_18 --
+I'm not null or empty
+
 -- !sql_7_02 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
 -- !sql_7_03 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	I'm not null or empty	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
 
 -- !sql_7_04 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
 
 -- !sql_7_05 --
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
@@ -100,7 +126,7 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
 
 -- !sql_7_07 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
 -- !sql_7_08 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
@@ -118,16 +144,16 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 value1	value2
 
 -- !sql_7_11 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
 -- !sql_7_12 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
 
 -- !sql_7_13 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	I'm not null or empty	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
 
 -- !sql_7_14 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
 
 -- !sql_7_15 --
 2022-08-08T20:10:10
@@ -141,17 +167,29 @@ value1	value2
 -- !sql_7_18 --
 2022-08-08T20:10:10
 
+-- !sql_7_19 --
+\N
+\N
+I'm not null or empty
+
+-- !sql_7_20 --
+
+I'm not null or empty
+
+-- !sql_7_21 --
+I'm not null or empty
+
 -- !sql_7_19 --
 value1	value2
 
 -- !sql_8_01 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
 -- !sql_8_02 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	I'm not null or empty	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
 
 -- !sql_8_03 --
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
@@ -166,7 +204,7 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.1	{"name":"Andy","age":18}
 
 -- !sql_8_05 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
 -- !sql_8_06 --
 [1, 0, 1, 1]	[1, -2, -3, 4]	[128, 129, -129, -130]	[32768, 32769, -32769, -32770]	[-1, 0, 1, 2]	[0, 1, 2, 3]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[1, 2, 3, 4]	[1, 2, 3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	["a", "b", "c"]	["d", "e", "f"]	["192.168.0.1", "127.0.0.1"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]
@@ -184,16 +222,16 @@ true	1	128	32768	-1	0	1.0	1.0	1.0	1.0	2020-01-01	2020-01-01T12:00	a	d	192.168.0.
 value1	value2
 
 -- !sql_8_09 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
 -- !sql_8_10 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]		string2	2022-08-08T12:10:10	text2	4.0	2022-08-08T00:00	2022-08-09T12:10:10	1660018210000	2022-08-09T12:10:10	2022-08-09T12:10:10	2222
 
 -- !sql_8_11 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	I'm not null or empty	string3	2022-08-09T00:40:10	text3_4*5	5.0	2022-08-08T00:00	2022-08-10T12:10:10	1660104610000	2022-08-10T12:10:10	2022-08-10T20:10:10	3333
 
 -- !sql_8_12 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string4	2022-08-08T20:10:10	text3_4*5	6.0	2022-08-08T00:00	2022-08-11T12:10:10	1660191010000	2022-08-11T12:10:10	2022-08-11T11:10:10	4444
 
 -- !sql_8_13 --
 2022-08-08T20:10:10
@@ -207,3 +245,15 @@ value1	value2
 -- !sql_8_16 --
 2022-08-08T20:10:10
 
+-- !sql_8_17 --
+\N
+\N
+I'm not null or empty
+
+-- !sql_8_18 --
+
+I'm not null or empty
+
+-- !sql_8_19 --
+I'm not null or empty
+
diff --git a/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out b/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out
index 36981363db5301..ceedce57b18da7 100644
--- a/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out
+++ b/regression-test/data/external_table_p0/es/test_es_query_no_http_url.out
@@ -6,11 +6,11 @@
 ["2020-01-01", "2020-01-02"]	[-1, 0, 1, 2]	[0, 1, 2, 3]	["d", "e", "f"]	[128, 129, -129, -130]	["192.168.0.1", "127.0.0.1"]	string1	[1, 2, 3, 4]	2022-08-08	2022-08-08T12:10:10	text#1	["2020-01-01", "2020-01-02"]	3.14	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	["a", "b", "c"]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	2022-08-08T12:10:10	2022-08-08T12:10:10	2022-08-08T20:10:10	[1, -2, -3, 4]	[1, 0, 1, 1]	[32768, 32769, -32769, -32770]
 
 -- !sql61 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01 00:00:00", "2020-01-02 00:00:00"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	text#1	3.14	2022-08-08T00:00	12345	2022-08-08T20:10:10
 
 -- !sql71 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
 -- !sql81 --
-[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
+[1, 0, 1, 1]	[1, -2, -3, 4]	["2020-01-01", "2020-01-02"]	["2020-01-01 12:00:00", "2020-01-02 13:01:01"]	[1, 2, 3, 4]	[1, 1.1, 1.2, 1.3]	[1, 2, 3, 4]	[32768, 32769, -32769, -32770]	["192.168.0.1", "127.0.0.1"]	["a", "b", "c"]	[-1, 0, 1, 2]	["{"name":"Andy","age":18}", "{"name":"Tim","age":28}"]	[1, 2, 3, 4]	[128, 129, -129, -130]	["d", "e", "f"]	[0, 1, 2, 3]	\N	string1	2022-08-08T20:10:10	text#1	3.14	2022-08-08T00:00	2022-08-08T12:10:10	1659931810000	2022-08-08T12:10:10	2022-08-08T20:10:10	12345
 
diff --git a/regression-test/suites/external_table_p0/es/test_es_query.groovy b/regression-test/suites/external_table_p0/es/test_es_query.groovy
index c0cec5c82dd324..6c8f71dfc93d9f 100644
--- a/regression-test/suites/external_table_p0/es/test_es_query.groovy
+++ b/regression-test/suites/external_table_p0/es/test_es_query.groovy
@@ -101,7 +101,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
                 `test8` datetime NULL,
                 `c_byte` array<tinyint(4)> NULL,
                 `c_bool` array<boolean> NULL,
-                `c_integer` array<int(11)> NULL
+                `c_integer` array<int(11)> NULL,
+                `message` text NULL
             ) ENGINE=ELASTICSEARCH
             COMMENT 'ELASTICSEARCH'
             PROPERTIES (
@@ -115,7 +116,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
         order_qt_sql01 """select * from test_v1 where test2='text#1'"""
         order_qt_sql02 """select * from test_v1 where esquery(test2, '{"match":{"test2":"text#1"}}')"""
         order_qt_sql03 """select test4,test5,test6,test7,test8 from test_v1 order by test8"""
-
+        order_qt_sql04 """select message from test_v1 where message != ''"""
+        order_qt_sql05 """select message from test_v1 where message is not null"""
+        order_qt_sql06 """select message from test_v1 where not_null_or_empty(message)"""
        sql """
             CREATE TABLE `test_v2` (
                 `c_datetime` array<datev2> NULL,
@@ -152,9 +155,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
                 "http_ssl_enabled"="false"
             );
         """
-        order_qt_sql04 """select * from test_v2 where test2='text#1'"""
-        order_qt_sql05 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')"""
-        order_qt_sql06 """select test4,test5,test6,test7,test8 from test_v2 order by test8"""
+        order_qt_sql20 """select * from test_v2 where test2='text#1'"""
+        order_qt_sql21 """select * from test_v2 where esquery(test2, '{"match":{"test2":"text#1"}}')"""
+        order_qt_sql22 """select test4,test5,test6,test7,test8 from test_v2 order by test8"""
 
         sql """switch test_es_query_es6"""
         // order_qt_sql_6_01 """show tables"""
@@ -172,6 +175,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
         order_qt_sql_6_13 """select test6 from test1 where test1='string1'"""
         order_qt_sql_6_14 """select test6 from test1 where test1='string2'"""
         order_qt_sql_6_15 """select test6 from test1 where test1='string3'"""
+        order_qt_sql_6_16 """select message from test1 where message != ''"""
+        order_qt_sql_6_17 """select message from test1 where message is not null"""
+        order_qt_sql_6_18 """select message from test1 where not_null_or_empty(message)"""
 
         List<List<String>> tables6N = sql """show tables"""
         boolean notContainHide = true
@@ -211,6 +217,9 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
         order_qt_sql_7_16 """select test10 from test1 where test1='string2'"""
         order_qt_sql_7_17 """select test10 from test1 where test1='string3'"""
         order_qt_sql_7_18 """select test10 from test1 where test1='string4'"""
+        order_qt_sql_7_19 """select message from test1 where message != ''"""
+        order_qt_sql_7_20 """select message from test1 where message is not null"""
+        order_qt_sql_7_21 """select message from test1 where not_null_or_empty(message)"""
 
         List<List<String>> tables7N = sql """show tables"""
         boolean notContainHide7 = true
@@ -250,5 +259,8 @@ suite("test_es_query", "p0,external,es,external_docker,external_docker_es") {
         order_qt_sql_8_14 """select test10 from test1 where test1='string2'"""
         order_qt_sql_8_15 """select test10 from test1 where test1='string3'"""
         order_qt_sql_8_16 """select test10 from test1 where test1='string4'"""
+        order_qt_sql_8_17 """select message from test1 where message != ''"""
+        order_qt_sql_8_18 """select message from test1 where message is not null"""
+        order_qt_sql_8_19 """select message from test1 where not_null_or_empty(message)"""
     }
 }

From 94cbabf675e2a750850352d9780080a76df3802f Mon Sep 17 00:00:00 2001
From: Dongyang Li <hello_stephen@qq.com>
Date: Sat, 30 Dec 2023 10:30:44 +0800
Subject: [PATCH 198/299] [fix](case) update load_colddata_to_hdfs.groovy
 (#29325)

---
 .../cold_heat_separation_p2/load_colddata_to_hdfs.groovy       | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy
index 14c5939dfc813a..b49306b3c72587 100644
--- a/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy
+++ b/regression-test/suites/cold_heat_separation_p2/load_colddata_to_hdfs.groovy
@@ -135,11 +135,12 @@ suite("load_colddata_to_hdfs") {
         """
     }
 
+    String hdfsFs = getHdfsFs()
     sql """
         CREATE RESOURCE IF NOT EXISTS ${resource_name}
         PROPERTIES (
             "type"="hdfs",
-            "fs.defaultFS"="127.0.0.1:8120",
+            "fs.defaultFS"="${hdfsFs}",
             "hadoop.username"="hive",
             "hadoop.password"="hive",
             "dfs.nameservices" = "my_ha",

From 51cb15d032d929da9301be85dac61d7fd568cb1b Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Sat, 30 Dec 2023 10:45:06 +0800
Subject: [PATCH 199/299] [improve](move-memtable) cancel load immediately when
 back pressure in delta writer v2 (#29280)

---
 be/src/olap/delta_writer_v2.cpp               |  19 +++-
 be/src/olap/delta_writer_v2.h                 |   7 +-
 be/src/vec/sink/writer/vtablet_writer_v2.cpp  |   2 +-
 .../vec/exec/delta_writer_v2_pool_test.cpp    |  10 +-
 ...er_v2_back_pressure_fault_injection.groovy | 106 ++++++++++++++++++
 5 files changed, 132 insertions(+), 12 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_delta_writer_v2_back_pressure_fault_injection.groovy

diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp
index 643f1837982a0f..5987c63e6584a8 100644
--- a/be/src/olap/delta_writer_v2.cpp
+++ b/be/src/olap/delta_writer_v2.cpp
@@ -65,16 +65,18 @@ namespace doris {
 using namespace ErrorCode;
 
 std::unique_ptr<DeltaWriterV2> DeltaWriterV2::open(
-        WriteRequest* req, const std::vector<std::shared_ptr<LoadStreamStub>>& streams) {
+        WriteRequest* req, const std::vector<std::shared_ptr<LoadStreamStub>>& streams,
+        RuntimeState* state) {
     std::unique_ptr<DeltaWriterV2> writer(
-            new DeltaWriterV2(req, streams, StorageEngine::instance()));
+            new DeltaWriterV2(req, streams, StorageEngine::instance(), state));
     return writer;
 }
 
 DeltaWriterV2::DeltaWriterV2(WriteRequest* req,
                              const std::vector<std::shared_ptr<LoadStreamStub>>& streams,
-                             StorageEngine* storage_engine)
-        : _req(*req),
+                             StorageEngine* storage_engine, RuntimeState* state)
+        : _state(state),
+          _req(*req),
           _tablet_schema(new TabletSchema),
           _memtable_writer(new MemTableWriter(*req)),
           _streams(streams) {}
@@ -158,8 +160,13 @@ Status DeltaWriterV2::write(const vectorized::Block* block, const std::vector<ui
     }
     {
         SCOPED_RAW_TIMER(&_wait_flush_limit_time);
-        while (_memtable_writer->flush_running_count() >=
-               config::memtable_flush_running_count_limit) {
+        auto memtable_flush_running_count_limit = config::memtable_flush_running_count_limit;
+        DBUG_EXECUTE_IF("DeltaWriterV2.write.back_pressure",
+                        { memtable_flush_running_count_limit = 0; });
+        while (_memtable_writer->flush_running_count() >= memtable_flush_running_count_limit) {
+            if (_state->is_cancelled()) {
+                return Status::Cancelled(_state->cancel_reason());
+            }
             std::this_thread::sleep_for(std::chrono::milliseconds(10));
         }
     }
diff --git a/be/src/olap/delta_writer_v2.h b/be/src/olap/delta_writer_v2.h
index f896977d842539..7db79680ee82dd 100644
--- a/be/src/olap/delta_writer_v2.h
+++ b/be/src/olap/delta_writer_v2.h
@@ -64,7 +64,8 @@ class Block;
 class DeltaWriterV2 {
 public:
     static std::unique_ptr<DeltaWriterV2> open(
-            WriteRequest* req, const std::vector<std::shared_ptr<LoadStreamStub>>& streams);
+            WriteRequest* req, const std::vector<std::shared_ptr<LoadStreamStub>>& streams,
+            RuntimeState* state);
 
     ~DeltaWriterV2();
 
@@ -88,7 +89,7 @@ class DeltaWriterV2 {
 
 private:
     DeltaWriterV2(WriteRequest* req, const std::vector<std::shared_ptr<LoadStreamStub>>& streams,
-                  StorageEngine* storage_engine);
+                  StorageEngine* storage_engine, RuntimeState* state);
 
     void _build_current_tablet_schema(int64_t index_id,
                                       const OlapTableSchemaParam* table_schema_param,
@@ -96,6 +97,8 @@ class DeltaWriterV2 {
 
     void _update_profile(RuntimeProfile* profile);
 
+    RuntimeState* _state = nullptr;
+
     bool _is_init = false;
     bool _is_cancelled = false;
     WriteRequest _req;
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 4972a397f701e3..8ded964950b5b8 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -438,7 +438,7 @@ Status VTabletWriterV2::_write_memtable(std::shared_ptr<vectorized::Block> block
                 break;
             }
         }
-        return DeltaWriterV2::open(&req, streams);
+        return DeltaWriterV2::open(&req, streams, _state);
     });
     {
         SCOPED_TIMER(_wait_mem_limit_timer);
diff --git a/be/test/vec/exec/delta_writer_v2_pool_test.cpp b/be/test/vec/exec/delta_writer_v2_pool_test.cpp
index d44fd17a761b0e..a40724cf9ff998 100644
--- a/be/test/vec/exec/delta_writer_v2_pool_test.cpp
+++ b/be/test/vec/exec/delta_writer_v2_pool_test.cpp
@@ -56,9 +56,13 @@ TEST_F(DeltaWriterV2PoolTest, test_map) {
     auto map = pool.get_or_create(load_id);
     EXPECT_EQ(1, pool.size());
     WriteRequest req;
-    auto writer = map->get_or_create(100, [&req]() { return DeltaWriterV2::open(&req, {}); });
-    auto writer2 = map->get_or_create(101, [&req]() { return DeltaWriterV2::open(&req, {}); });
-    auto writer3 = map->get_or_create(100, [&req]() { return DeltaWriterV2::open(&req, {}); });
+    RuntimeState state;
+    auto writer = map->get_or_create(
+            100, [&req, &state]() { return DeltaWriterV2::open(&req, {}, &state); });
+    auto writer2 = map->get_or_create(
+            101, [&req, &state]() { return DeltaWriterV2::open(&req, {}, &state); });
+    auto writer3 = map->get_or_create(
+            100, [&req, &state]() { return DeltaWriterV2::open(&req, {}, &state); });
     EXPECT_EQ(2, map->size());
     EXPECT_EQ(writer, writer3);
     EXPECT_NE(writer, writer2);
diff --git a/regression-test/suites/fault_injection_p0/test_delta_writer_v2_back_pressure_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_delta_writer_v2_back_pressure_fault_injection.groovy
new file mode 100644
index 00000000000000..ea9e9ffb8bb574
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_delta_writer_v2_back_pressure_fault_injection.groovy
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("test_delta_writer_v2_back_pressure_fault_injection", "nonConcurrent") {
+    sql """ set enable_memtable_on_sink_node=true """
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """
+        CREATE TABLE IF NOT EXISTS `baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+    sql """
+        CREATE TABLE IF NOT EXISTS `test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    GetDebugPoint().clearDebugPointsForAllBEs()
+    streamLoad {
+        table "baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    try {
+        GetDebugPoint().enableDebugPointForAllBEs("DeltaWriterV2.write.back_pressure")
+        def thread1 = new Thread({
+            try {
+                def res = sql "insert into test select * from baseall where k1 <= 3"
+                logger.info(res.toString())
+            } catch(Exception e) {
+                logger.info(e.getMessage())
+                assertTrue(e.getMessage().contains("Communications link failure"))
+            } finally {
+                GetDebugPoint().disableDebugPointForAllBEs("DeltaWriterV2.write.back_pressure")
+            }
+        })
+        thread1.start()
+
+        sleep(1000)
+
+        def processList = sql "show processlist"
+        logger.info(processList.toString())
+        processList.each { item ->
+            logger.info(item[1].toString())
+            logger.info(item[11].toString())
+            if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
+                def res = sql "kill ${item[1]}"
+                logger.info(res.toString())
+            }
+        }
+    } catch(Exception e) {
+        logger.info(e.getMessage())
+    }
+
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """ set enable_memtable_on_sink_node=false """
+}
\ No newline at end of file

From e7d67e9411c79c843ce0c3c822905fdd7a089c2e Mon Sep 17 00:00:00 2001
From: py023 <137761033+py023@users.noreply.github.com>
Date: Sat, 30 Dec 2023 12:14:49 +0800
Subject: [PATCH 200/299] [fix](be) resolves some unused-raii and
 used-after-moved issues (#29285)

---
 be/src/agent/task_worker_pool.cpp    | 2 +-
 be/src/http/action/config_action.cpp | 1 -
 be/src/io/fs/s3_file_system.cpp      | 5 +----
 be/src/vec/columns/column_array.cpp  | 4 ++--
 be/src/vec/columns/column_map.cpp    | 2 +-
 5 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index 97bf7f700a1ae7..f9d43ba23180fd 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -590,7 +590,7 @@ ReportWorker::ReportWorker(std::string name, const TMasterInfo& master_info, int
     };
 
     auto st = Thread::create("ReportWorker", _name, report_loop, &_thread);
-    CHECK(st.ok()) << name << ": " << st;
+    CHECK(st.ok()) << _name << ": " << st;
 }
 
 ReportWorker::~ReportWorker() {
diff --git a/be/src/http/action/config_action.cpp b/be/src/http/action/config_action.cpp
index 4ee196be651b1f..a9de9dc2406b61 100644
--- a/be/src/http/action/config_action.cpp
+++ b/be/src/http/action/config_action.cpp
@@ -118,7 +118,6 @@ void ConfigAction::handle_update_config(HttpRequest* req) {
             std::string status(s.ok() ? "OK" : "BAD");
             rapidjson::Value result;
             result.SetObject();
-            rapidjson::Value(key.c_str(), key.size(), results.GetAllocator());
             result.AddMember("config_name",
                              rapidjson::Value(key.c_str(), key.size(), results.GetAllocator()),
                              results.GetAllocator());
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index c8206668498852..a88d9851ad436e 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -114,7 +114,7 @@ S3FileSystem::S3FileSystem(S3Conf&& s3_conf, std::string&& id)
         }
     }
     _executor = Aws::MakeShared<Aws::Utils::Threading::PooledThreadExecutor>(
-            id.c_str(), config::s3_transfer_executor_pool_size);
+            _id.c_str(), config::s3_transfer_executor_pool_size);
 }
 
 S3FileSystem::~S3FileSystem() = default;
@@ -211,9 +211,6 @@ Status S3FileSystem::delete_directory_impl(const Path& dir) {
                 return Status::IOError("fail to delete object: {}",
                                        error_msg(e.GetKey(), e.GetMessage()));
             }
-            VLOG_TRACE << "delete " << objects.size()
-                       << " s3 objects, endpoint: " << _s3_conf.endpoint
-                       << ", bucket: " << _s3_conf.bucket << ", prefix: " << _s3_conf.prefix;
         }
         is_trucated = result.GetIsTruncated();
         request.SetContinuationToken(result.GetNextContinuationToken());
diff --git a/be/src/vec/columns/column_array.cpp b/be/src/vec/columns/column_array.cpp
index c936c28b885edd..0c926c0f31e3ff 100644
--- a/be/src/vec/columns/column_array.cpp
+++ b/be/src/vec/columns/column_array.cpp
@@ -95,11 +95,11 @@ ColumnArray::ColumnArray(MutableColumnPtr&& nested_column, MutableColumnPtr&& of
         LOG(FATAL) << "offsets_column must be a ColumnUInt64";
     }
 
-    if (!offsets_concrete->empty() && nested_column) {
+    if (!offsets_concrete->empty() && data) {
         auto last_offset = offsets_concrete->get_data().back();
 
         /// This will also prevent possible overflow in offset.
-        if (nested_column->size() != last_offset) {
+        if (data->size() != last_offset) {
             LOG(FATAL) << "offsets_column has data inconsistent with nested_column";
         }
     }
diff --git a/be/src/vec/columns/column_map.cpp b/be/src/vec/columns/column_map.cpp
index 4d831e01f9e35c..6981c25fc23d0e 100644
--- a/be/src/vec/columns/column_map.cpp
+++ b/be/src/vec/columns/column_map.cpp
@@ -53,7 +53,7 @@ ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values, Mutable
         LOG(FATAL) << "offsets_column must be a ColumnUInt64";
     }
 
-    if (!offsets_concrete->empty() && keys && values) {
+    if (!offsets_concrete->empty() && keys_column && values_column) {
         auto last_offset = offsets_concrete->get_data().back();
 
         /// This will also prevent possible overflow in offset.

From 30b73576041d2714503a777d2a5b09efad93a020 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Sat, 30 Dec 2023 12:47:47 +0800
Subject: [PATCH 201/299] [improve](logging) make glog FLAGS_v configurable
 (#29182)

---
 be/src/common/config.cpp    | 2 ++
 be/src/common/config.h      | 2 ++
 be/src/common/logconfig.cpp | 2 +-
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 785aab61c97494..e94b4c6dadb315 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -208,6 +208,8 @@ DEFINE_Int32(sys_log_roll_num, "10");
 DEFINE_Strings(sys_log_verbose_modules, "");
 // verbose log level
 DEFINE_Int32(sys_log_verbose_level, "10");
+// verbose log FLAGS_v
+DEFINE_Int32(sys_log_verbose_flags_v, "-1");
 // log buffer level
 DEFINE_String(log_buffer_level, "");
 
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 1cbfbffb4cbf87..2d257b49510c10 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -256,6 +256,8 @@ DECLARE_Int32(sys_log_roll_num);
 DECLARE_Strings(sys_log_verbose_modules);
 // verbose log level
 DECLARE_Int32(sys_log_verbose_level);
+// verbose log FLAGS_v
+DECLARE_Int32(sys_log_verbose_flags_v);
 // log buffer level
 DECLARE_String(log_buffer_level);
 
diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp
index 1c53035317abd6..bae037066aea82 100644
--- a/be/src/common/logconfig.cpp
+++ b/be/src/common/logconfig.cpp
@@ -130,7 +130,7 @@ bool init_glog(const char* basename) {
     }
 
     // set verbose modules.
-    FLAGS_v = -1;
+    FLAGS_v = config::sys_log_verbose_flags_v;
     std::vector<std::string>& verbose_modules = config::sys_log_verbose_modules;
     int32_t vlog_level = config::sys_log_verbose_level;
     for (size_t i = 0; i < verbose_modules.size(); i++) {

From 03901b9a7a1970162fa71fd3892905496ef892c3 Mon Sep 17 00:00:00 2001
From: huanghaibin <284824253@qq.com>
Date: Sat, 30 Dec 2023 12:59:46 +0800
Subject: [PATCH 202/299] [enhancement](group_commit): refector relay wal code
 (#29183)

---
 be/src/http/action/http_stream.cpp            |  16 +-
 be/src/http/action/http_stream.h              |   2 +-
 be/src/olap/wal_dirs_info.cpp                 |   1 +
 be/src/olap/wal_info.cpp                      |  41 ++
 be/src/olap/wal_info.h                        |  38 ++
 be/src/olap/wal_manager.cpp                   |  41 +-
 be/src/olap/wal_manager.h                     |  17 +-
 be/src/olap/wal_table.cpp                     | 428 ++++++++----------
 be/src/olap/wal_table.h                       |  32 +-
 be/src/runtime/group_commit_mgr.cpp           |  26 +-
 .../runtime/stream_load/stream_load_context.h |   1 +
 be/src/vec/exec/format/wal/wal_reader.cpp     |   2 -
 be/src/vec/sink/writer/vwal_writer.cpp        |   2 +-
 be/test/olap/wal_manager_test.cpp             |   6 +-
 .../doris/analysis/NativeInsertStmt.java      |   2 +-
 gensrc/thrift/FrontendService.thrift          |  14 +-
 16 files changed, 346 insertions(+), 323 deletions(-)
 create mode 100644 be/src/olap/wal_info.cpp
 create mode 100644 be/src/olap/wal_info.h

diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp
index a18b10f491fc1a..15d2a1a18d1bed 100644
--- a/be/src/http/action/http_stream.cpp
+++ b/be/src/http/action/http_stream.cpp
@@ -256,7 +256,7 @@ void HttpStreamAction::on_chunk_data(HttpRequest* req) {
             } else {
                 LOG(INFO) << "use a portion of data to request fe to obtain column information";
                 ctx->is_read_schema = false;
-                ctx->status = _process_put(req, ctx);
+                ctx->status = process_put(req, ctx);
             }
         }
 
@@ -272,7 +272,7 @@ void HttpStreamAction::on_chunk_data(HttpRequest* req) {
         LOG(INFO) << "after all the data has been read and it has not reached 1M, it will execute "
                   << "here";
         ctx->is_read_schema = false;
-        ctx->status = _process_put(req, ctx);
+        ctx->status = process_put(req, ctx);
     }
     ctx->read_data_cost_nanos += (MonotonicNanos() - start_read_data_time);
 }
@@ -290,11 +290,15 @@ void HttpStreamAction::free_handler_ctx(std::shared_ptr<void> param) {
     ctx->exec_env()->new_load_stream_mgr()->remove(ctx->id);
 }
 
-Status HttpStreamAction::_process_put(HttpRequest* http_req,
-                                      std::shared_ptr<StreamLoadContext> ctx) {
+Status HttpStreamAction::process_put(HttpRequest* http_req,
+                                     std::shared_ptr<StreamLoadContext> ctx) {
     TStreamLoadPutRequest request;
     set_request_auth(&request, ctx->auth);
-    request.__set_load_sql(http_req->header(HTTP_SQL));
+    if (http_req != nullptr) {
+        request.__set_load_sql(http_req->header(HTTP_SQL));
+    } else {
+        request.__set_load_sql(ctx->sql_str);
+    }
     request.__set_loadId(ctx->id.to_thrift());
     request.__set_label(ctx->label);
     if (ctx->group_commit) {
@@ -330,7 +334,7 @@ Status HttpStreamAction::_process_put(HttpRequest* http_req,
     ctx->txn_id = ctx->put_result.params.txn_conf.txn_id;
     ctx->label = ctx->put_result.params.import_label;
     ctx->put_result.params.__set_wal_id(ctx->wal_id);
-    if (http_req->header(HTTP_GROUP_COMMIT) == "async_mode") {
+    if (http_req != nullptr && http_req->header(HTTP_GROUP_COMMIT) == "async_mode") {
         if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
             size_t content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
             if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
diff --git a/be/src/http/action/http_stream.h b/be/src/http/action/http_stream.h
index d4140a118d61e3..90a8b48fb2bb77 100644
--- a/be/src/http/action/http_stream.h
+++ b/be/src/http/action/http_stream.h
@@ -43,11 +43,11 @@ class HttpStreamAction : public HttpHandler {
 
     void on_chunk_data(HttpRequest* req) override;
     void free_handler_ctx(std::shared_ptr<void> ctx) override;
+    Status process_put(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
 
 private:
     Status _on_header(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
     Status _handle(HttpRequest* req, std::shared_ptr<StreamLoadContext> ctx);
-    Status _process_put(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
     void _save_stream_load_record(std::shared_ptr<StreamLoadContext> ctx, const std::string& str);
     Status _handle_group_commit(HttpRequest* http_req, std::shared_ptr<StreamLoadContext> ctx);
 
diff --git a/be/src/olap/wal_dirs_info.cpp b/be/src/olap/wal_dirs_info.cpp
index 340d896a8c60db..1b7216b0724598 100644
--- a/be/src/olap/wal_dirs_info.cpp
+++ b/be/src/olap/wal_dirs_info.cpp
@@ -169,6 +169,7 @@ size_t WalDirsInfo::get_max_available_size() {
 
 Status WalDirsInfo::update_wal_dir_limit(std::string wal_dir, size_t limit) {
     for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+        LOG(INFO) << "wal_dir_info:" << wal_dir_info->get_wal_dir();
         if (wal_dir_info->get_wal_dir() == wal_dir) {
             return wal_dir_info->update_wal_dir_limit(limit);
         }
diff --git a/be/src/olap/wal_info.cpp b/be/src/olap/wal_info.cpp
new file mode 100644
index 00000000000000..d93593cfaf026d
--- /dev/null
+++ b/be/src/olap/wal_info.cpp
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "olap/wal_info.h"
+namespace doris {
+WalInfo::WalInfo(int64_t wal_id, std::string wal_path, int64_t retry_num, int64_t start_time_ms)
+        : _wal_id(wal_id),
+          _wal_path(wal_path),
+          _retry_num(retry_num),
+          _start_time_ms(start_time_ms) {}
+WalInfo::~WalInfo() {}
+int64_t WalInfo::get_wal_id() {
+    return _wal_id;
+}
+std::string WalInfo::get_wal_path() {
+    return _wal_path;
+}
+int64_t WalInfo::get_retry_num() {
+    return _retry_num;
+}
+int64_t WalInfo::get_start_time_ms() {
+    return _start_time_ms;
+}
+void WalInfo::add_retry_num() {
+    _retry_num++;
+}
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_info.h b/be/src/olap/wal_info.h
new file mode 100644
index 00000000000000..0383ac68f2ce36
--- /dev/null
+++ b/be/src/olap/wal_info.h
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+#include "runtime/exec_env.h"
+
+namespace doris {
+class WalInfo {
+public:
+    WalInfo(int64_t wal_id, std::string wal_path, int64_t retry_num, int64_t start_time_ms);
+    ~WalInfo();
+    int64_t get_wal_id();
+    int64_t get_retry_num();
+    int64_t get_start_time_ms();
+    std::string get_wal_path();
+    void add_retry_num();
+
+private:
+    int64_t _wal_id;
+    std::string _wal_path;
+    int64_t _retry_num;
+    int64_t _start_time_ms;
+};
+
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_manager.cpp b/be/src/olap/wal_manager.cpp
index 0fe59cdf9cac3f..a9c35794698698 100644
--- a/be/src/olap/wal_manager.cpp
+++ b/be/src/olap/wal_manager.cpp
@@ -82,6 +82,9 @@ Status WalManager::init() {
     RETURN_IF_ERROR(_init_wal_dirs_conf());
     RETURN_IF_ERROR(_init_wal_dirs());
     RETURN_IF_ERROR(_init_wal_dirs_info());
+    for (auto wal_dir : _wal_dirs) {
+        RETURN_IF_ERROR(scan_wals(wal_dir));
+    }
     return Thread::create(
             "WalMgr", "replay_wal", [this]() { static_cast<void>(this->replay()); },
             &_replay_thread);
@@ -112,7 +115,7 @@ Status WalManager::_init_wal_dirs_conf() {
 Status WalManager::_init_wal_dirs() {
     bool exists = false;
     for (auto wal_dir : _wal_dirs) {
-        std::string tmp_dir = wal_dir + "/tmp";
+        std::string tmp_dir = wal_dir + "/" + tmp;
         LOG(INFO) << "wal_dir:" << wal_dir << ",tmp_dir:" << tmp_dir;
         RETURN_IF_ERROR(io::global_local_filesystem()->exists(wal_dir, &exists));
         if (!exists) {
@@ -122,7 +125,6 @@ Status WalManager::_init_wal_dirs() {
         if (!exists) {
             RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(tmp_dir));
         }
-        RETURN_IF_ERROR(scan_wals(wal_dir));
     }
     return Status::OK();
 }
@@ -164,15 +166,15 @@ Status WalManager::_init_wal_dirs_info() {
             &_update_wal_dirs_info_thread);
 }
 
-void WalManager::add_wal_status_queue(int64_t table_id, int64_t wal_id, WAL_STATUS wal_status) {
+void WalManager::add_wal_status_queue(int64_t table_id, int64_t wal_id, WalStatus wal_status) {
     std::lock_guard<std::shared_mutex> wrlock(_wal_status_lock);
     LOG(INFO) << "add wal queue "
               << ",table_id:" << table_id << ",wal_id:" << wal_id << ",status:" << wal_status;
     auto it = _wal_status_queues.find(table_id);
     if (it == _wal_status_queues.end()) {
-        std::unordered_map<int64_t, WAL_STATUS> tmp;
-        tmp.emplace(wal_id, wal_status);
-        _wal_status_queues.emplace(table_id, tmp);
+        std::unordered_map<int64_t, WalStatus> tmp_map;
+        tmp_map.emplace(wal_id, wal_status);
+        _wal_status_queues.emplace(table_id, tmp_map);
     } else {
         it->second.emplace(wal_id, wal_status);
     }
@@ -305,12 +307,12 @@ Status WalManager::scan_wals(const std::string& wal_path) {
         LOG(WARNING) << "Failed list files for dir=" << wal_path << ", st=" << st.to_string();
         return st;
     }
-    for (const auto& db_id : dbs) {
-        if (db_id.is_file) {
+    for (const auto& database_id : dbs) {
+        if (database_id.is_file || database_id.file_name == tmp) {
             continue;
         }
         std::vector<io::FileInfo> tables;
-        auto db_path = wal_path + "/" + db_id.file_name;
+        auto db_path = wal_path + "/" + database_id.file_name;
         st = io::global_local_filesystem()->list(db_path, false, &tables, &exists);
         if (!st.ok()) {
             LOG(WARNING) << "Failed list files for dir=" << db_path << ", st=" << st.to_string();
@@ -342,20 +344,16 @@ Status WalManager::scan_wals(const std::string& wal_path) {
                         int64_t wal_id =
                                 std::strtoll(wal.file_name.substr(0, pos).c_str(), NULL, 10);
                         _wal_path_map.emplace(wal_id, wal_file);
+                        int64_t db_id = std::strtoll(database_id.file_name.c_str(), NULL, 10);
                         int64_t tb_id = std::strtoll(table_id.file_name.c_str(), NULL, 10);
-                        add_wal_status_queue(tb_id, wal_id, WalManager::WAL_STATUS::REPLAY);
+                        add_wal_status_queue(tb_id, wal_id, WalManager::WalStatus::REPLAY);
+                        RETURN_IF_ERROR(add_recover_wal(db_id, tb_id, wal_id, wal_file));
                     } catch (const std::invalid_argument& e) {
                         return Status::InvalidArgument("Invalid format, {}", e.what());
                     }
                 }
             }
-            st = add_recover_wal(std::stoll(db_id.file_name), std::stoll(table_id.file_name), res);
             count += res.size();
-            if (!st.ok()) {
-                LOG(WARNING) << "Failed add replay wal, db=" << db_id.file_name
-                             << ", table=" << table_id.file_name << ", st=" << st.to_string();
-                return st;
-            }
         }
     }
     LOG(INFO) << "Finish list all wals, size:" << count;
@@ -396,7 +394,8 @@ Status WalManager::replay() {
     return Status::OK();
 }
 
-Status WalManager::add_recover_wal(int64_t db_id, int64_t table_id, std::vector<std::string> wals) {
+Status WalManager::add_recover_wal(int64_t db_id, int64_t table_id, int64_t wal_id,
+                                   std::string wal) {
     std::lock_guard<std::shared_mutex> wrlock(_lock);
     std::shared_ptr<WalTable> table_ptr;
     auto it = _table_map.find(table_id);
@@ -406,12 +405,10 @@ Status WalManager::add_recover_wal(int64_t db_id, int64_t table_id, std::vector<
     } else {
         table_ptr = it->second;
     }
-    table_ptr->add_wals(wals);
+    table_ptr->add_wal(wal_id, wal);
 #ifndef BE_TEST
-    for (auto wal : wals) {
-        RETURN_IF_ERROR(update_wal_dir_limit(_get_base_wal_path(wal)));
-        RETURN_IF_ERROR(update_wal_dir_used(_get_base_wal_path(wal)));
-    }
+    RETURN_IF_ERROR(update_wal_dir_limit(_get_base_wal_path(wal)));
+    RETURN_IF_ERROR(update_wal_dir_used(_get_base_wal_path(wal)));
 #endif
     return Status::OK();
 }
diff --git a/be/src/olap/wal_manager.h b/be/src/olap/wal_manager.h
index 8d330a64bb9fc8..4b42beaf45551e 100644
--- a/be/src/olap/wal_manager.h
+++ b/be/src/olap/wal_manager.h
@@ -47,7 +47,7 @@ class WalManager {
     ENABLE_FACTORY_CREATOR(WalManager);
 
 public:
-    enum WAL_STATUS {
+    enum WalStatus {
         PREPARE = 0,
         REPLAY,
         CREATE,
@@ -56,6 +56,7 @@ class WalManager {
 public:
     WalManager(ExecEnv* exec_env, const std::string& wal_dir);
     ~WalManager();
+    // used for wal
     Status delete_wal(int64_t wal_id, size_t block_queue_pre_allocated = 0);
     Status init();
     Status scan_wals(const std::string& wal_path);
@@ -64,13 +65,13 @@ class WalManager {
     Status create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>& wal_writer);
     Status scan();
     size_t get_wal_table_size(int64_t table_id);
-    Status add_recover_wal(int64_t db_id, int64_t table_id, std::vector<std::string> wals);
+    Status add_recover_wal(int64_t db_id, int64_t table_id, int64_t wal_id, std::string wal);
     Status add_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id, const std::string& label,
                         std::string& base_path);
     Status get_wal_path(int64_t wal_id, std::string& wal_path);
     Status get_wal_status_queue_size(const PGetWalQueueSizeRequest* request,
                                      PGetWalQueueSizeResponse* response);
-    void add_wal_status_queue(int64_t table_id, int64_t wal_id, WAL_STATUS wal_status);
+    void add_wal_status_queue(int64_t table_id, int64_t wal_id, WalStatus wal_status);
     Status erase_wal_status_queue(int64_t table_id, int64_t wal_id);
     void print_wal_status_queue();
     void stop();
@@ -80,6 +81,7 @@ class WalManager {
     void erase_wal_column_index(int64_t wal_id);
     Status get_wal_column_index(int64_t wal_id, std::vector<size_t>& column_index);
 
+    // used for limit
     Status update_wal_dir_limit(const std::string& wal_dir, size_t limit = -1);
     Status update_wal_dir_used(const std::string& wal_dir, size_t used = -1);
     Status update_wal_dir_pre_allocated(const std::string& wal_dir, size_t pre_allocated,
@@ -88,6 +90,7 @@ class WalManager {
     size_t get_max_available_size();
 
 private:
+    // used for limit
     Status _init_wal_dirs_conf();
     Status _init_wal_dirs();
     Status _init_wal_dirs_info();
@@ -99,11 +102,13 @@ class WalManager {
     // used for be ut
     size_t wal_limit_test_bytes;
 
+    const std::string tmp = "tmp";
+
 private:
+    //used for wal
     ExecEnv* _exec_env = nullptr;
     std::shared_mutex _lock;
     scoped_refptr<Thread> _replay_thread;
-    scoped_refptr<Thread> _update_wal_dirs_info_thread;
     CountDownLatch _stop_background_threads_latch;
     std::map<int64_t, std::shared_ptr<WalTable>> _table_map;
     std::vector<std::string> _wal_dirs;
@@ -111,11 +116,13 @@ class WalManager {
     std::shared_mutex _wal_status_lock;
     std::unordered_map<int64_t, std::string> _wal_path_map;
     std::unordered_map<int64_t, std::shared_ptr<WalWriter>> _wal_id_to_writer_map;
-    std::unordered_map<int64_t, std::unordered_map<int64_t, WAL_STATUS>> _wal_status_queues;
+    std::unordered_map<int64_t, std::unordered_map<int64_t, WalStatus>> _wal_status_queues;
     std::atomic<bool> _stop;
     std::shared_mutex _wal_column_id_map_lock;
     std::unordered_map<int64_t, std::vector<size_t>&> _wal_column_id_map;
     std::unique_ptr<doris::ThreadPool> _thread_pool;
+    // used for limit
+    scoped_refptr<Thread> _update_wal_dirs_info_thread;
     std::unique_ptr<WalDirsInfo> _wal_dirs_info;
 };
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_table.cpp b/be/src/olap/wal_table.cpp
index 7f98c410b0779a..a1c9a12f3e1750 100644
--- a/be/src/olap/wal_table.cpp
+++ b/be/src/olap/wal_table.cpp
@@ -17,19 +17,16 @@
 
 #include "olap/wal_table.h"
 
-#include <event2/bufferevent.h>
-#include <event2/event.h>
-#include <event2/event_struct.h>
-#include <event2/http.h>
 #include <thrift/protocol/TDebugProtocol.h>
 
-#include "evhttp.h"
+#include "http/action/http_stream.h"
 #include "http/action/stream_load.h"
 #include "http/ev_http_server.h"
 #include "http/http_common.h"
 #include "http/http_headers.h"
 #include "http/utils.h"
 #include "io/fs/local_file_system.h"
+#include "io/fs/stream_load_pipe.h"
 #include "olap/wal_manager.h"
 #include "runtime/client_cache.h"
 #include "runtime/fragment_mgr.h"
@@ -41,125 +38,149 @@
 namespace doris {
 
 WalTable::WalTable(ExecEnv* exec_env, int64_t db_id, int64_t table_id)
-        : _exec_env(exec_env), _db_id(db_id), _table_id(table_id), _stop(false) {}
+        : _exec_env(exec_env), _db_id(db_id), _table_id(table_id), _stop(false) {
+    _http_stream_action = std::make_shared<HttpStreamAction>(exec_env);
+}
 WalTable::~WalTable() {}
 
 #ifdef BE_TEST
-std::string k_request_line;
+Status k_stream_load_exec_status;
 #endif
 
-bool retry = false;
-
-void WalTable::add_wals(std::vector<std::string> wals) {
+void WalTable::add_wal(int64_t wal_id, std::string wal) {
     std::lock_guard<std::mutex> lock(_replay_wal_lock);
-    for (const auto& wal : wals) {
-        LOG(INFO) << "add replay wal " << wal;
-        _replay_wal_map.emplace(wal, replay_wal_info {0, UnixMillis(), false});
-    }
+    LOG(INFO) << "add replay wal " << wal;
+    auto wal_info = std::make_shared<WalInfo>(wal_id, wal, 0, UnixMillis());
+    _replay_wal_map.emplace(wal, wal_info);
 }
-Status WalTable::replay_wals() {
+void WalTable::pick_relay_wals() {
+    std::lock_guard<std::mutex> lock(_replay_wal_lock);
     std::vector<std::string> need_replay_wals;
     std::vector<std::string> need_erase_wals;
-    {
-        std::lock_guard<std::mutex> lock(_replay_wal_lock);
-        if (_replay_wal_map.empty()) {
-            return Status::OK();
-        }
-        VLOG_DEBUG << "Start replay wals for db=" << _db_id << ", table=" << _table_id
-                   << ", wal size=" << _replay_wal_map.size();
-        for (auto& [wal, info] : _replay_wal_map) {
-            auto& [retry_num, start_ts, replaying] = info;
-            if (replaying) {
-                LOG(INFO) << wal << " is replaying, skip this round";
-                return Status::OK();
-            }
-            if (retry_num >= config::group_commit_replay_wal_retry_num) {
-                LOG(WARNING) << "All replay wal failed, db=" << _db_id << ", table=" << _table_id
-                             << ", wal=" << wal
-                             << ", retry_num=" << config::group_commit_replay_wal_retry_num;
-                std::string rename_path = _get_tmp_path(wal);
-                LOG(INFO) << "rename wal from " << wal << " to " << rename_path;
-                std::rename(wal.c_str(), rename_path.c_str());
-                need_erase_wals.push_back(wal);
-                continue;
-            }
-            if (_need_replay(info)) {
-                need_replay_wals.push_back(wal);
+    for (auto it = _replay_wal_map.begin(); it != _replay_wal_map.end(); it++) {
+        auto wal_info = it->second;
+        if (wal_info->get_retry_num() >= config::group_commit_replay_wal_retry_num) {
+            LOG(WARNING) << "All replay wal failed, db=" << _db_id << ", table=" << _table_id
+                         << ", wal=" << it->first << ", retry_num=" << wal_info->get_retry_num();
+            auto st = _rename_to_tmp_path(it->first);
+            if (!st.ok()) {
+                LOG(WARNING) << "rename " << it->first << " fail"
+                             << ",st:" << st.to_string();
             }
+            need_erase_wals.push_back(it->first);
+            continue;
         }
-        std::sort(need_replay_wals.begin(), need_replay_wals.end());
-        for (const auto& wal : need_erase_wals) {
-            if (_replay_wal_map.erase(wal)) {
-                LOG(INFO) << "erase wal " << wal << " from _replay_wal_map";
-            } else {
-                LOG(WARNING) << "fail to erase wal " << wal << " from _replay_wal_map";
-            }
+        if (_need_replay(wal_info)) {
+            need_replay_wals.push_back(it->first);
         }
     }
+    for (const auto& wal : need_erase_wals) {
+        _replay_wal_map.erase(wal);
+    }
+    std::sort(need_replay_wals.begin(), need_replay_wals.end());
     for (const auto& wal : need_replay_wals) {
+        _replaying_queue.emplace_back(_replay_wal_map[wal]);
+        _replay_wal_map.erase(wal);
+    }
+}
+
+Status WalTable::relay_wal_one_by_one() {
+    std::vector<std::shared_ptr<WalInfo>> need_retry_wals;
+    std::vector<std::shared_ptr<WalInfo>> need_delete_wals;
+    while (!_replaying_queue.empty()) {
+        std::shared_ptr<WalInfo> wal_info = nullptr;
         {
             std::lock_guard<std::mutex> lock(_replay_wal_lock);
-            if (_stop.load()) {
-                break;
+            wal_info = _replaying_queue.front();
+            _replaying_queue.pop_front();
+        }
+        wal_info->add_retry_num();
+        auto st = _replay_wal_internal(wal_info->get_wal_path());
+        if (!st.ok()) {
+            LOG(WARNING) << "failed replay wal, db=" << _db_id << ", table=" << _table_id
+                         << ", wal=" << wal_info->get_wal_path() << ", st=" << st.to_string();
+            if (!st.is<ErrorCode::NOT_FOUND>()) {
+                need_retry_wals.push_back(wal_info);
             } else {
-                auto it = _replay_wal_map.find(wal);
-                if (it != _replay_wal_map.end()) {
-                    auto& [retry_num, start_time, replaying] = it->second;
-                    replaying = true;
-                }
+                need_delete_wals.push_back(wal_info);
             }
+        } else {
+            need_delete_wals.push_back(wal_info);
+        }
+        VLOG_NOTICE << "replay wal, db=" << _db_id << ", table=" << _table_id
+                    << ", wal=" << wal_info->get_wal_path() << ", st=" << st.to_string();
+    }
+    {
+        std::lock_guard<std::mutex> lock(_replay_wal_lock);
+        for (auto retry_wal_info : need_retry_wals) {
+            _replay_wal_map.emplace(retry_wal_info->get_wal_path(), retry_wal_info);
         }
-        auto st = _replay_wal_internal(wal);
+    }
+    for (auto delete_wal_info : need_delete_wals) {
+        auto st = _delete_wal(delete_wal_info->get_wal_id());
         if (!st.ok()) {
-            std::lock_guard<std::mutex> lock(_replay_wal_lock);
-            auto it = _replay_wal_map.find(wal);
-            if (it != _replay_wal_map.end()) {
-                auto& [retry_num, start_time, replaying] = it->second;
-                replaying = false;
-            }
-            LOG(WARNING) << "failed replay wal, drop this round, db=" << _db_id
-                         << ", table=" << _table_id << ", wal=" << wal << ", st=" << st.to_string();
-            break;
+            LOG(WARNING) << "fail to delete wal " << delete_wal_info->get_wal_path();
         }
-        VLOG_NOTICE << "replay wal, db=" << _db_id << ", table=" << _table_id << ", label=" << wal
-                    << ", st=" << st.to_string();
     }
     return Status::OK();
 }
-
-std::string WalTable::_get_tmp_path(const std::string wal) {
-    std::vector<std::string> path_element;
-    doris::vectorized::WalReader::string_split(wal, "/", path_element);
-    std::stringstream ss;
-    int index = 0;
-    while (index < path_element.size() - 3) {
-        ss << path_element[index] << "/";
-        index++;
+Status WalTable::replay_wals() {
+    {
+        std::lock_guard<std::mutex> lock(_replay_wal_lock);
+        if (_replay_wal_map.empty()) {
+            LOG(INFO) << "_replay_wal_map is empty,skip relaying";
+            return Status::OK();
+        }
+        if (!_replaying_queue.empty()) {
+            LOG(INFO) << "_replaying_queue is not empty,skip relaying";
+            return Status::OK();
+        }
     }
-    ss << "tmp/";
-    while (index < path_element.size()) {
-        if (index != path_element.size() - 1) {
-            ss << path_element[index] << "_";
-        } else {
-            ss << path_element[index];
+    VLOG_DEBUG << "Start replay wals for db=" << _db_id << ", table=" << _table_id
+               << ", wal size=" << _replay_wal_map.size();
+    pick_relay_wals();
+    RETURN_IF_ERROR(relay_wal_one_by_one());
+    return Status::OK();
+}
+
+Status WalTable::_rename_to_tmp_path(const std::string wal) {
+    io::Path wal_path = wal;
+    std::list<std::string> path_element;
+    for (int i = 0; i < 3; ++i) {
+        if (!wal_path.has_parent_path()) {
+            return Status::InternalError("parent path is not enough when rename " + wal);
         }
-        index++;
+        path_element.push_front(wal_path.filename().string());
+        wal_path = wal_path.parent_path();
     }
-    return ss.str();
+    wal_path.append(_exec_env->wal_mgr()->tmp);
+    for (auto path : path_element) {
+        wal_path.append(path);
+    }
+    bool exists = false;
+    RETURN_IF_ERROR(io::global_local_filesystem()->exists(wal_path.parent_path(), &exists));
+    if (!exists) {
+        RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(wal_path.parent_path()));
+    }
+    auto res = std::rename(wal.c_str(), wal_path.string().c_str());
+    if (res < 0) {
+        return Status::InternalError("rename fail on path " + wal);
+    }
+    LOG(INFO) << "rename wal from " << wal << " to " << wal_path.string();
+    return Status::OK();
 }
 
-bool WalTable::_need_replay(const doris::WalTable::replay_wal_info& info) {
+bool WalTable::_need_replay(std::shared_ptr<WalInfo> wal_info) {
 #ifndef BE_TEST
-    auto& [retry_num, start_ts, replaying] = info;
-    auto replay_interval =
-            pow(2, retry_num) * config::group_commit_replay_wal_retry_interval_seconds * 1000;
-    return UnixMillis() - start_ts >= replay_interval;
+    auto replay_interval = pow(2, wal_info->get_retry_num()) *
+                           config::group_commit_replay_wal_retry_interval_seconds * 1000;
+    return UnixMillis() - wal_info->get_start_time_ms() >= replay_interval;
 #else
     return true;
 #endif
 }
 
-Status WalTable::_abort_txn(int64_t db_id, int64_t wal_id) {
+Status WalTable::_try_abort_txn(int64_t db_id, int64_t wal_id) {
     TLoadTxnRollbackRequest request;
     request.__set_auth_code(0); // this is a fake, fe not check it now
     request.__set_db_id(db_id);
@@ -181,47 +202,23 @@ Status WalTable::_abort_txn(int64_t db_id, int64_t wal_id) {
 
 Status WalTable::_replay_wal_internal(const std::string& wal) {
     LOG(INFO) << "Start replay wal for db=" << _db_id << ", table=" << _table_id << ", wal=" << wal;
-    // start a new stream load
-    {
-        std::lock_guard<std::mutex> lock(_replay_wal_lock);
-        auto it = _replay_wal_map.find(wal);
-        if (it != _replay_wal_map.end()) {
-            auto& [retry_num, start_time, replaying] = it->second;
-            ++retry_num;
-            replaying = true;
-        } else {
-            LOG(WARNING) << "can not find wal in stream load replay map. db=" << _db_id
-                         << ", table=" << _table_id << ", wal=" << wal;
-            return Status::OK();
-        }
-    }
     std::shared_ptr<std::pair<int64_t, std::string>> pair = nullptr;
-    RETURN_IF_ERROR(_get_wal_info(wal, pair));
+    RETURN_IF_ERROR(_parse_wal_path(wal, pair));
     auto wal_id = pair->first;
     auto label = pair->second;
 #ifndef BE_TEST
-    auto st = _abort_txn(_db_id, wal_id);
+    auto st = _try_abort_txn(_db_id, wal_id);
     if (!st.ok()) {
         LOG(WARNING) << "abort txn " << wal_id << " fail";
     }
-    auto get_st = _get_column_info(_db_id, _table_id);
-    if (!get_st.ok()) {
-        if (get_st.is<ErrorCode::NOT_FOUND>()) {
-            {
-                std::lock_guard<std::mutex> lock(_replay_wal_lock);
-                _replay_wal_map.erase(wal);
-            }
-            RETURN_IF_ERROR(_delete_wal(wal_id));
-        }
-        return get_st;
-    }
+    RETURN_IF_ERROR(_get_column_info(_db_id, _table_id));
 #endif
-    RETURN_IF_ERROR(_send_request(wal_id, wal, label));
+    RETURN_IF_ERROR(_replay_one_txn_with_stremaload(wal_id, wal, label));
     return Status::OK();
 }
 
-Status WalTable::_get_wal_info(const std::string& wal,
-                               std::shared_ptr<std::pair<int64_t, std::string>>& pair) {
+Status WalTable::_parse_wal_path(const std::string& wal,
+                                 std::shared_ptr<std::pair<int64_t, std::string>>& pair) {
     std::vector<std::string> path_element;
     doris::vectorized::WalReader::string_split(wal, "/", path_element);
     auto pos = path_element[path_element.size() - 1].find("_");
@@ -236,171 +233,104 @@ Status WalTable::_get_wal_info(const std::string& wal,
     return Status::OK();
 }
 
-void http_request_done(struct evhttp_request* req, void* arg) {
-    std::stringstream out;
-    std::string status;
-    std::string msg;
-    std::string wal_id;
-    size_t len = 0;
-    if (req != nullptr) {
-        auto input = evhttp_request_get_input_buffer(req);
-        char* request_line = evbuffer_readln(input, &len, EVBUFFER_EOL_CRLF);
-        while (request_line != nullptr) {
-            std::string s(request_line);
-            out << request_line;
-            request_line = evbuffer_readln(input, &len, EVBUFFER_EOL_CRLF);
-        }
-        auto out_str = out.str();
-        LOG(INFO) << "replay wal out_str:" << out_str;
-        rapidjson::Document doc;
-        if (!out_str.empty()) {
-            doc.Parse(out.str().c_str());
-            status = std::string(doc["Status"].GetString());
-            msg = std::string(doc["Message"].GetString());
-            LOG(INFO) << "replay wal status:" << status << ",msg:" << msg;
-            if (status.find("Fail") != status.npos) {
-                if (msg.find("Label") != msg.npos &&
-                    msg.find("has already been used") != msg.npos) {
-                    retry = false;
-                } else {
-                    retry = true;
-                }
-            } else {
-                retry = false;
-            }
-        } else {
-            retry = true;
-        }
-    } else {
-        LOG(WARNING) << "req is null";
-    }
-
-    if (arg != nullptr) {
-        event_base_loopbreak((struct event_base*)arg);
-    } else {
-        LOG(WARNING) << "arg is null";
-    }
-}
-
-Status WalTable::_send_request(int64_t wal_id, const std::string& wal, const std::string& label) {
-#ifndef BE_TEST
-    struct event_base* base = nullptr;
-    struct evhttp_connection* conn = nullptr;
-    struct evhttp_request* req = nullptr;
-    retry = false;
-    event_init();
-    base = event_base_new();
-    conn = evhttp_connection_new("127.0.0.1", doris::config::webserver_port);
-    evhttp_connection_set_base(conn, base);
-    req = evhttp_request_new(http_request_done, base);
-    evhttp_add_header(req->output_headers, HTTP_LABEL_KEY.c_str(), label.c_str());
-    evhttp_add_header(req->output_headers, HTTP_AUTH_CODE.c_str(), std::to_string(wal_id).c_str());
-    evhttp_add_header(req->output_headers, HTTP_WAL_ID_KY.c_str(), std::to_string(wal_id).c_str());
+Status WalTable::_construct_sql_str(const std::string& wal, const std::string& label,
+                                    std::string& sql_str, std::vector<size_t>& index_vector) {
     std::string columns;
     RETURN_IF_ERROR(_read_wal_header(wal, columns));
     std::vector<std::string> column_id_element;
     doris::vectorized::WalReader::string_split(columns, ",", column_id_element);
-    std::vector<size_t> index_vector;
     std::stringstream ss_name;
     std::stringstream ss_id;
     int index_raw = 0;
     for (auto column_id_str : column_id_element) {
         try {
             int64_t column_id = std::strtoll(column_id_str.c_str(), NULL, 10);
-            auto it = _column_id_name_map.find(column_id);
-            auto it2 = _column_id_index_map.find(column_id);
-            if (it != _column_id_name_map.end() && it2 != _column_id_index_map.end()) {
-                ss_name << "`" << it->second << "`,";
-                ss_id << "c" << std::to_string(_column_id_index_map[column_id]) << ",";
+            auto it = _column_id_info_map.find(column_id);
+            if (it != _column_id_info_map.end()) {
+                ss_name << "`" << it->second->first << "`,";
+                ss_id << "c" << std::to_string(it->second->second) << ",";
                 index_vector.emplace_back(index_raw);
-                _column_id_name_map.erase(column_id);
-                _column_id_index_map.erase(column_id);
             }
             index_raw++;
         } catch (const std::invalid_argument& e) {
             return Status::InvalidArgument("Invalid format, {}", e.what());
         }
     }
-    _exec_env->wal_mgr()->add_wal_column_index(wal_id, index_vector);
     auto name = ss_name.str().substr(0, ss_name.str().size() - 1);
     auto id = ss_id.str().substr(0, ss_id.str().size() - 1);
     std::stringstream ss;
     ss << "insert into doris_internal_table_id(" << _table_id << ") WITH LABEL " << label << " ("
        << name << ") select " << id << " from http_stream(\"format\" = \"wal\", \"table_id\" = \""
        << std::to_string(_table_id) << "\")";
-    evhttp_add_header(req->output_headers, HTTP_SQL.c_str(), ss.str().c_str());
-    evbuffer* output = evhttp_request_get_output_buffer(req);
-    evbuffer_add_printf(output, "replay wal %s", std::to_string(wal_id).c_str());
-
-    evhttp_make_request(conn, req, EVHTTP_REQ_PUT, "/api/_http_stream");
-    evhttp_connection_set_timeout(req->evcon, 300);
-
-    event_base_dispatch(base);
-    evhttp_connection_free(conn);
-    event_base_free(base);
+    sql_str = ss.str().data();
+    return Status::OK();
+}
+Status WalTable::_handle_stream_load(int64_t wal_id, const std::string& wal,
+                                     const std::string& label) {
+    std::string sql_str;
+    std::vector<size_t> index_vector;
+    RETURN_IF_ERROR(_construct_sql_str(wal, label, sql_str, index_vector));
+    _exec_env->wal_mgr()->add_wal_column_index(wal_id, index_vector);
+    std::shared_ptr<StreamLoadContext> ctx = std::make_shared<StreamLoadContext>(_exec_env);
+    ctx->sql_str = sql_str;
+    ctx->wal_id = wal_id;
+    ctx->auth.auth_code = wal_id;
+    ctx->label = label;
+    auto st = _http_stream_action->process_put(nullptr, ctx);
+    if (st.ok()) {
+        // wait stream load finish
+        RETURN_IF_ERROR(ctx->future.get());
+        if (ctx->status.ok()) {
+            auto commit_st = _exec_env->stream_load_executor()->commit_txn(ctx.get());
+            st = commit_st;
+        } else if (!ctx->status.ok()) {
+            LOG(WARNING) << "handle streaming load failed, id=" << ctx->id
+                         << ", errmsg=" << ctx->status;
+            _exec_env->stream_load_executor()->rollback_txn(ctx.get());
+            st = ctx->status;
+        }
+    }
+    _exec_env->wal_mgr()->erase_wal_column_index(wal_id);
+    LOG(INFO) << "relay wal id=" << wal_id << ",st=" << st.to_string();
+    return st;
+}
 
+Status WalTable::_replay_one_txn_with_stremaload(int64_t wal_id, const std::string& wal,
+                                                 const std::string& label) {
+    bool success = false;
+#ifndef BE_TEST
+    auto st = _handle_stream_load(wal_id, wal, label);
+    auto msg = st.msg();
+    success = st.ok() || st.is<ErrorCode::PUBLISH_TIMEOUT>() ||
+              msg.find("LabelAlreadyUsedException") != msg.npos;
 #else
-    std::stringstream out;
-    out << k_request_line;
-    auto out_str = out.str();
-    rapidjson::Document doc;
-    doc.Parse(out_str.c_str());
-    auto status = std::string(doc["Status"].GetString());
-    if (status.find("Fail") != status.npos) {
-        retry = true;
-    } else {
-        retry = false;
-    }
+    success = k_stream_load_exec_status.ok();
 #endif
-    if (retry) {
-        LOG(INFO) << "fail to replay wal =" << wal;
-        std::lock_guard<std::mutex> lock(_replay_wal_lock);
-        auto it = _replay_wal_map.find(wal);
-        if (it != _replay_wal_map.end()) {
-            auto& [retry_num, start_time, replaying] = it->second;
-            replaying = false;
-        } else {
-            _replay_wal_map.emplace(wal, replay_wal_info {0, UnixMillis(), false});
-        }
-    } else {
+    if (success) {
         LOG(INFO) << "success to replay wal =" << wal;
-        RETURN_IF_ERROR(_delete_wal(wal_id));
-        std::lock_guard<std::mutex> lock(_replay_wal_lock);
-        if (_replay_wal_map.erase(wal)) {
-            LOG(INFO) << "erase " << wal << " from _replay_wal_map";
-        } else {
-            LOG(WARNING) << "fail to erase " << wal << " from _replay_wal_map";
-        }
+    } else {
+        LOG(INFO) << "fail to replay wal =" << wal;
+        return Status::InternalError("fail to replay wal =" + wal);
     }
-    _exec_env->wal_mgr()->erase_wal_column_index(wal_id);
     return Status::OK();
 }
 
 void WalTable::stop() {
-    bool done = true;
     do {
         {
             std::lock_guard<std::mutex> lock(_replay_wal_lock);
             if (!this->_stop.load()) {
                 this->_stop.store(true);
             }
-            auto it = _replay_wal_map.begin();
-            for (; it != _replay_wal_map.end(); it++) {
-                auto& [retry_num, start_time, replaying] = it->second;
-                if (replaying) {
-                    break;
-                }
-            }
-            if (it != _replay_wal_map.end()) {
-                done = false;
-            } else {
-                done = true;
+            if (_replay_wal_map.empty() && _replaying_queue.empty()) {
+                break;
             }
-        }
-        if (!done) {
+            LOG(INFO) << "stopping wal_table,wait for relay wal task done, now "
+                      << _replay_wal_map.size() << " wals wait to replay, "
+                      << _replaying_queue.size() << " wals are replaying";
             std::this_thread::sleep_for(std::chrono::milliseconds(1000));
         }
-    } while (!done);
+    } while (true);
 }
 
 size_t WalTable::size() {
@@ -429,13 +359,13 @@ Status WalTable::_get_column_info(int64_t db_id, int64_t tb_id) {
         }
         std::vector<TColumnInfo> column_element = result.columns;
         int64_t column_index = 1;
-        _column_id_name_map.clear();
-        _column_id_index_map.clear();
+        _column_id_info_map.clear();
         for (auto column : column_element) {
-            auto column_name = column.columnName;
-            auto column_id = column.columnId;
-            _column_id_name_map.emplace(column_id, column_name);
-            _column_id_index_map.emplace(column_id, column_index);
+            auto column_name = column.column_name;
+            auto column_id = column.column_id;
+            std::shared_ptr<ColumnInfo> column_pair =
+                    std::make_shared<ColumnInfo>(std::make_pair(column_name, column_index));
+            _column_id_info_map.emplace(column_id, column_pair);
             column_index++;
         }
     }
diff --git a/be/src/olap/wal_table.h b/be/src/olap/wal_table.h
index e3d66d577a2f93..251e8d51a61244 100644
--- a/be/src/olap/wal_table.h
+++ b/be/src/olap/wal_table.h
@@ -24,6 +24,8 @@
 #include "gen_cpp/FrontendService.h"
 #include "gen_cpp/FrontendService_types.h"
 #include "gen_cpp/HeartbeatService_types.h"
+#include "http/action/http_stream.h"
+#include "olap/wal_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/stream_load/stream_load_context.h"
 namespace doris {
@@ -32,25 +34,32 @@ class WalTable {
     WalTable(ExecEnv* exec_env, int64_t db_id, int64_t table_id);
     ~WalTable();
     // used when be start and there are wals need to do recovery
-    void add_wals(std::vector<std::string> wals);
+    void add_wal(int64_t wal_id, std::string wal);
+    void pick_relay_wals();
+    Status relay_wal_one_by_one();
     Status replay_wals();
     size_t size();
     void stop();
 
 public:
-    // <retry_num, start_time_ms, is_doing_replay>
-    using replay_wal_info = std::tuple<int64_t, int64_t, bool>;
+    // <column_name, column_index>
+    using ColumnInfo = std::pair<std::string, int64_t>;
 
 private:
-    Status _get_wal_info(const std::string& wal, std::shared_ptr<std::pair<int64_t, std::string>>&);
-    std::string _get_tmp_path(const std::string wal);
-    Status _send_request(int64_t wal_id, const std::string& wal, const std::string& label);
-    Status _abort_txn(int64_t db_id, int64_t wal_id);
+    Status _parse_wal_path(const std::string& wal,
+                           std::shared_ptr<std::pair<int64_t, std::string>>&);
+    Status _rename_to_tmp_path(const std::string wal);
+    Status _replay_one_txn_with_stremaload(int64_t wal_id, const std::string& wal,
+                                           const std::string& label);
+    Status _try_abort_txn(int64_t db_id, int64_t wal_id);
     Status _get_column_info(int64_t db_id, int64_t tb_id);
     Status _read_wal_header(const std::string& wal, std::string& columns);
-    bool _need_replay(const replay_wal_info& info);
+    bool _need_replay(std::shared_ptr<WalInfo>);
     Status _replay_wal_internal(const std::string& wal);
     Status _delete_wal(int64_t wal_id);
+    Status _construct_sql_str(const std::string& wal, const std::string& label,
+                              std::string& sql_str, std::vector<size_t>& index_vector);
+    Status _handle_stream_load(int64_t wal_id, const std::string& wal, const std::string& label);
 
 private:
     ExecEnv* _exec_env;
@@ -60,9 +69,10 @@ class WalTable {
     std::string _split = "_";
     mutable std::mutex _replay_wal_lock;
     // key is wal_id
-    std::map<std::string, replay_wal_info> _replay_wal_map;
+    std::map<std::string, std::shared_ptr<WalInfo>> _replay_wal_map;
+    std::list<std::shared_ptr<WalInfo>> _replaying_queue;
     std::atomic<bool> _stop;
-    std::map<int64_t, std::string> _column_id_name_map;
-    std::map<int64_t, int64_t> _column_id_index_map;
+    std::map<int64_t, std::shared_ptr<ColumnInfo>> _column_id_info_map;
+    std::shared_ptr<HttpStreamAction> _http_stream_action;
 };
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index 28df650f13c7b8..acb40ae6c78661 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -51,7 +51,11 @@ Status LoadBlockQueue::add_block(std::shared_ptr<vectorized::Block> block, bool
     if (block->rows() > 0) {
         _block_queue.push_back(block);
         if (write_wal) {
-            RETURN_IF_ERROR(_v_wal_writer->write_wal(block.get()));
+            auto st = _v_wal_writer->write_wal(block.get());
+            if (!st.ok()) {
+                _cancel_without_lock(st);
+                return st;
+            }
         }
         _all_block_queues_bytes->fetch_add(block->bytes(), std::memory_order_relaxed);
     }
@@ -278,11 +282,17 @@ Status GroupCommitTable::_create_group_commit_load(
         _load_block_queues.emplace(instance_id, load_block_queue);
         _need_plan_fragment = false;
         _exec_env->wal_mgr()->add_wal_status_queue(_table_id, txn_id,
-                                                   WalManager::WAL_STATUS::PREPARE);
+                                                   WalManager::WalStatus::PREPARE);
         //create wal
-        RETURN_IF_ERROR(
-                load_block_queue->create_wal(_db_id, _table_id, txn_id, label, _exec_env->wal_mgr(),
-                                             params.desc_tbl.slotDescriptors, be_exe_version));
+        if (!is_pipeline) {
+            RETURN_IF_ERROR(load_block_queue->create_wal(
+                    _db_id, _table_id, txn_id, label, _exec_env->wal_mgr(),
+                    params.desc_tbl.slotDescriptors, be_exe_version));
+        } else {
+            RETURN_IF_ERROR(load_block_queue->create_wal(
+                    _db_id, _table_id, txn_id, label, _exec_env->wal_mgr(),
+                    pipeline_params.desc_tbl.slotDescriptors, be_exe_version));
+        }
         _cv.notify_all();
     }
     st = _exec_plan_fragment(_db_id, _table_id, label, txn_id, is_pipeline, params,
@@ -367,10 +377,8 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_
     } else {
         std::string wal_path;
         RETURN_IF_ERROR(_exec_env->wal_mgr()->get_wal_path(txn_id, wal_path));
-        RETURN_IF_ERROR(_exec_env->wal_mgr()->add_recover_wal(db_id, table_id,
-                                                              std::vector<std::string> {wal_path}));
-        _exec_env->wal_mgr()->add_wal_status_queue(table_id, txn_id,
-                                                   WalManager::WAL_STATUS::REPLAY);
+        RETURN_IF_ERROR(_exec_env->wal_mgr()->add_recover_wal(db_id, table_id, txn_id, wal_path));
+        _exec_env->wal_mgr()->add_wal_status_queue(table_id, txn_id, WalManager::WalStatus::REPLAY);
     }
     std::stringstream ss;
     ss << "finish group commit, db_id=" << db_id << ", table_id=" << table_id << ", label=" << label
diff --git a/be/src/runtime/stream_load/stream_load_context.h b/be/src/runtime/stream_load/stream_load_context.h
index b530242743e086..e57996af9e12fb 100644
--- a/be/src/runtime/stream_load/stream_load_context.h
+++ b/be/src/runtime/stream_load/stream_load_context.h
@@ -139,6 +139,7 @@ class StreamLoadContext {
     int64_t table_id = -1;
     int64_t schema_version = -1;
     std::string label;
+    std::string sql_str;
     // optional
     std::string sub_label;
     double max_filter_ratio = 0.0;
diff --git a/be/src/vec/exec/format/wal/wal_reader.cpp b/be/src/vec/exec/format/wal/wal_reader.cpp
index 01846acc04aa18..32fd66cc309629 100644
--- a/be/src/vec/exec/format/wal/wal_reader.cpp
+++ b/be/src/vec/exec/format/wal/wal_reader.cpp
@@ -88,8 +88,6 @@ void WalReader::string_split(const std::string& str, const std::string& splits,
 Status WalReader::get_columns(std::unordered_map<std::string, TypeDescriptor>* name_to_type,
                               std::unordered_set<std::string>* missing_cols) {
     RETURN_IF_ERROR(_wal_reader->read_header(_version, _col_ids));
-    std::vector<std::string> col_element;
-    string_split(_col_ids, ",", col_element);
     RETURN_IF_ERROR(_state->exec_env()->wal_mgr()->get_wal_column_index(_wal_id, _column_index));
     return Status::OK();
 }
diff --git a/be/src/vec/sink/writer/vwal_writer.cpp b/be/src/vec/sink/writer/vwal_writer.cpp
index 2dc945a2a2f5b9..2ab37c340a0512 100644
--- a/be/src/vec/sink/writer/vwal_writer.cpp
+++ b/be/src/vec/sink/writer/vwal_writer.cpp
@@ -56,7 +56,7 @@ VWalWriter::~VWalWriter() {}
 
 Status VWalWriter::init() {
     RETURN_IF_ERROR(_wal_manager->create_wal_writer(_wal_id, _wal_writer));
-    _wal_manager->add_wal_status_queue(_tb_id, _wal_id, WalManager::WAL_STATUS::CREATE);
+    _wal_manager->add_wal_status_queue(_tb_id, _wal_id, WalManager::WalStatus::CREATE);
     std::stringstream ss;
     for (auto slot_desc : _slot_descs) {
         if (slot_desc.col_unique_id < 0) {
diff --git a/be/test/olap/wal_manager_test.cpp b/be/test/olap/wal_manager_test.cpp
index 93d8636eb225ec..c5c216f12a795a 100644
--- a/be/test/olap/wal_manager_test.cpp
+++ b/be/test/olap/wal_manager_test.cpp
@@ -44,8 +44,7 @@
 namespace doris {
 
 extern TLoadTxnBeginResult k_stream_load_begin_result;
-extern Status k_stream_load_plan_status;
-extern std::string k_request_line;
+extern Status k_stream_load_exec_status;
 
 ExecEnv* _env = nullptr;
 std::string wal_dir = std::string(getenv("DORIS_HOME")) + "/wal_test";
@@ -67,7 +66,6 @@ class WalManagerTest : public testing::Test {
         _env->_store_paths = {StorePath(std::filesystem::current_path(), 0)};
         _env->_wal_manager = WalManager::create_shared(_env, wal_dir);
         k_stream_load_begin_result = TLoadTxnBeginResult();
-        k_stream_load_plan_status = Status::OK();
     }
     void TearDown() override {
         static_cast<void>(io::global_local_filesystem()->delete_directory(wal_dir));
@@ -88,7 +86,7 @@ class WalManagerTest : public testing::Test {
 
 TEST_F(WalManagerTest, recovery_normal) {
     _env->wal_mgr()->wal_limit_test_bytes = 1099511627776;
-    k_request_line = "{\"Status\": \"Success\",    \"Message\": \"Test\"}";
+    k_stream_load_exec_status = Status::OK();
 
     std::string db_id = "1";
     int64_t tb_1_id = 1;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
index 0fcd5050fc83cb..da82c4406c323d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
@@ -1110,7 +1110,7 @@ public RedirectStatus getRedirectStatus() {
     }
 
     public void analyzeGroupCommit(Analyzer analyzer) throws AnalysisException {
-        if (isGroupCommitStreamLoadSql && (targetTable instanceof OlapTable)
+        if (isGroupCommitStreamLoadSql && targetTable != null && (targetTable instanceof OlapTable)
                 && !((OlapTable) targetTable).getTableProperty().getUseSchemaLightChange()) {
             throw new AnalysisException(
                     "table light_schema_change is false, can't do http_stream with group commit mode");
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index d672597a0db40f..209947c8b78778 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -688,16 +688,6 @@ struct TStreamLoadWithLoadStatusResult {
     6: optional i64 unselected_rows
 }
 
-struct TCheckWalRequest {
-    1: optional i64 wal_id
-    2: optional i64 db_id
-}
-
-struct TCheckWalResult {
-    1: optional Status.TStatus status
-    2: optional bool need_recovery
-}
-
 struct TKafkaRLTaskProgress {
     1: required map<i32,i64> partitionCmtOffset
 }
@@ -1303,8 +1293,8 @@ struct TGetBackendMetaResult {
 }
 
 struct TColumnInfo {
-  1: optional string columnName
-  2: optional i64 columnId
+  1: optional string column_name
+  2: optional i64 column_id
 }
 
 struct TGetColumnInfoRequest {

From e500b421421fd4d36b82e1f02a367a783340d984 Mon Sep 17 00:00:00 2001
From: wangbo <wangbo@apache.org>
Date: Sat, 30 Dec 2023 14:35:17 +0800
Subject: [PATCH 203/299] Disable global cancel policy in regression test
 (#29335)

---
 .../data/workload_manager_p0/test_workload_sched_policy.out     | 2 +-
 .../workload_manager_p0/test_workload_sched_policy.groovy       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/regression-test/data/workload_manager_p0/test_workload_sched_policy.out b/regression-test/data/workload_manager_p0/test_workload_sched_policy.out
index c5028f8363eeb6..0c86700866d195 100644
--- a/regression-test/data/workload_manager_p0/test_workload_sched_policy.out
+++ b/regression-test/data/workload_manager_p0/test_workload_sched_policy.out
@@ -1,6 +1,6 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !select_policy_tvf --
-be_policy	query_time > 10	cancel_query	10	true	0
+be_policy	query_time > 10	cancel_query	10	false	0
 fe_policy	username = root	set_session_variable "workload_group=normal"	10	false	0
 move_action_policy	query_time > 10	move_query_to_group "normal"	0	true	0
 set_action_policy	username = root	set_session_variable "workload_group=normal"	0	true	0
diff --git a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
index d59bb76dd5ef17..cabba8ee008911 100644
--- a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
+++ b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy
@@ -54,7 +54,7 @@ suite("test_workload_sched_policy") {
             "conditions(query_time > 10) " +
             "actions(cancel_query) " +
             "properties( " +
-            "'enabled' = 'true', " +
+            "'enabled' = 'false', " +
             "'priority'='10' " +
             ");"
 

From 80fb496523da2ac2816ab439744ded7faabd4f9f Mon Sep 17 00:00:00 2001
From: walter <w41ter.l@gmail.com>
Date: Sat, 30 Dec 2023 16:43:06 +0800
Subject: [PATCH 204/299] [improve](proc) Show journal size in bdbje journal
 dir (#29314)

---
 .../apache/doris/common/proc/BdbjeJournalDataProcNode.java   | 5 +++--
 .../java/org/apache/doris/journal/bdbje/BDBDebugger.java     | 2 ++
 .../src/main/java/org/apache/doris/qe/ConnectProcessor.java  | 4 ++++
 .../src/main/java/org/apache/doris/qe/StmtExecutor.java      | 4 ++++
 4 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BdbjeJournalDataProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BdbjeJournalDataProcNode.java
index c8c197b8cac976..6b18bd05a460f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/BdbjeJournalDataProcNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/BdbjeJournalDataProcNode.java
@@ -29,7 +29,7 @@
 public class BdbjeJournalDataProcNode implements ProcNodeInterface  {
 
     public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
-            .add("JournalId").add("OpType").add("Data").build();
+            .add("JournalId").add("OpType").add("Size").add("Data").build();
 
     private String dbName;
     private Long journalId;
@@ -53,7 +53,8 @@ public ProcResult fetchResult() throws AnalysisException {
 
         short opCode = entity.entity == null ? -1 : entity.entity.getOpCode();
         String data = entity.entity == null ? entity.errMsg : entity.entity.getData().toString();
-        result.addRow(Lists.newArrayList(entity.journalId.toString(), OperationType.getOpName(opCode), data));
+        result.addRow(Lists.newArrayList(entity.journalId.toString(),
+                    OperationType.getOpName(opCode), entity.size.toString(), data));
 
         return result;
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java
index 97009edff15cc4..d6f17f0cb8226b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBDebugger.java
@@ -212,6 +212,7 @@ public JournalEntityWrapper getJournalEntity(String dbName, Long journalId) {
             db.close();
             if (status == OperationStatus.SUCCESS) {
                 byte[] retData = value.getData();
+                entityWrapper.size = retData.length;
                 DataInputStream in = new DataInputStream(new ByteArrayInputStream(retData));
                 JournalEntity entity = new JournalEntity();
                 try {
@@ -240,6 +241,7 @@ public void close() {
 
     public static class JournalEntityWrapper {
         public Long journalId;
+        public Integer size;
         public JournalEntity entity;
         public String errMsg;
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 537a5553472443..3ec533c9a8e919 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -30,6 +30,7 @@
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.cluster.ClusterNamespace;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.ErrorCode;
 import org.apache.doris.common.NotImplementedException;
@@ -161,6 +162,9 @@ protected static boolean isNull(byte[] bitmap, int position) {
 
     protected void auditAfterExec(String origStmt, StatementBase parsedStmt,
             Data.PQueryStatistics statistics, boolean printFuzzyVariables) {
+        if (Config.enable_bdbje_debug_mode) {
+            return;
+        }
         AuditLogHelper.logAuditLog(ctx, origStmt, parsedStmt, statistics, printFuzzyVariables);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index 25f2f2e2e28448..a617d742f23cd0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -356,6 +356,10 @@ public boolean isForwardToMaster() {
             return false;
         }
 
+        if (Config.enable_bdbje_debug_mode) {
+            return false;
+        }
+
         // this is a query stmt, but this non-master FE can not read, forward it to master
         if (isQuery() && !Env.getCurrentEnv().isMaster()
                 && !Env.getCurrentEnv().canRead()) {

From 13f3d30652e1a6e709a26977ebe732c7241d8e5f Mon Sep 17 00:00:00 2001
From: lihuigang <35155212+lihuigang@users.noreply.github.com>
Date: Sat, 30 Dec 2023 17:25:33 +0800
Subject: [PATCH 205/299] [fix](hive-udf) Fixed the calculation result error in
 Hive bitmap UDF (#29073)

---
 .../org/apache/doris/udf/BitmapAndUDF.java    |   2 +-
 .../org/apache/doris/udf/BitmapOrUDF.java     |   2 +-
 .../org/apache/doris/udf/BitmapXorUDF.java    |   2 +-
 .../java/org/apache/doris/BitmapUDFTest.java  | 113 ++++++++++++++++++
 4 files changed, 116 insertions(+), 3 deletions(-)
 create mode 100644 fe/hive-udf/src/test/java/org/apache/doris/BitmapUDFTest.java

diff --git a/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapAndUDF.java b/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapAndUDF.java
index 8b5044b51f3b8b..5950f36d1a4548 100644
--- a/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapAndUDF.java
+++ b/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapAndUDF.java
@@ -63,7 +63,7 @@ public Object evaluate(DeferredObject[]  args) throws HiveException {
             BitmapValue bitmapValue0 = BitmapValueUtil.deserializeToBitmap(inputBytes0);
             BitmapValue bitmapValue1 = BitmapValueUtil.deserializeToBitmap(inputBytes1);
             bitmapValue0.and(bitmapValue1);
-            return BitmapValueUtil.serializeToBytes(bitmapValue1);
+            return BitmapValueUtil.serializeToBytes(bitmapValue0);
         } catch (IOException ioException) {
             throw new RuntimeException(ioException);
         }
diff --git a/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapOrUDF.java b/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapOrUDF.java
index 9ed1446d3fbaea..34e564c72a7733 100644
--- a/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapOrUDF.java
+++ b/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapOrUDF.java
@@ -63,7 +63,7 @@ public Object evaluate(DeferredObject[]  args) throws HiveException {
             BitmapValue bitmapValue0 = BitmapValueUtil.deserializeToBitmap(inputBytes0);
             BitmapValue bitmapValue1 = BitmapValueUtil.deserializeToBitmap(inputBytes1);
             bitmapValue0.or(bitmapValue1);
-            return BitmapValueUtil.serializeToBytes(bitmapValue1);
+            return BitmapValueUtil.serializeToBytes(bitmapValue0);
         } catch (IOException ioException) {
             throw new RuntimeException(ioException);
         }
diff --git a/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapXorUDF.java b/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapXorUDF.java
index fa46eeec7a29c4..91d0e765fed8ea 100644
--- a/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapXorUDF.java
+++ b/fe/hive-udf/src/main/java/org/apache/doris/udf/BitmapXorUDF.java
@@ -63,7 +63,7 @@ public Object evaluate(DeferredObject[]  args) throws HiveException {
             BitmapValue bitmapValue0 = BitmapValueUtil.deserializeToBitmap(inputBytes0);
             BitmapValue bitmapValue1 = BitmapValueUtil.deserializeToBitmap(inputBytes1);
             bitmapValue0.xor(bitmapValue1);
-            return BitmapValueUtil.serializeToBytes(bitmapValue1);
+            return BitmapValueUtil.serializeToBytes(bitmapValue0);
         } catch (IOException ioException) {
             throw new RuntimeException(ioException);
         }
diff --git a/fe/hive-udf/src/test/java/org/apache/doris/BitmapUDFTest.java b/fe/hive-udf/src/test/java/org/apache/doris/BitmapUDFTest.java
new file mode 100644
index 00000000000000..fba084a9260777
--- /dev/null
+++ b/fe/hive-udf/src/test/java/org/apache/doris/BitmapUDFTest.java
@@ -0,0 +1,113 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris;
+
+import org.apache.doris.common.BitmapValueUtil;
+import org.apache.doris.common.io.BitmapValue;
+import org.apache.doris.udf.BitmapAndUDF;
+import org.apache.doris.udf.BitmapCountUDF;
+import org.apache.doris.udf.BitmapOrUDF;
+import org.apache.doris.udf.BitmapXorUDF;
+
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaConstantBinaryObjectInspector;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+
+// hive bitmap udf test
+public class BitmapUDFTest {
+
+    private byte[] bitmapValue0Bytes;
+    private byte[] bitmapValue1Bytes;
+
+    private BinaryObjectInspector inputOI0 = new JavaConstantBinaryObjectInspector(new byte[0]);
+    private BinaryObjectInspector inputOI1 = new JavaConstantBinaryObjectInspector(new byte[0]);
+
+    @Before
+    public void initData() throws IOException {
+        BitmapValue bitmapValue0 = new BitmapValue();
+        BitmapValue bitmapValue1 = new BitmapValue();
+
+        bitmapValue0.add(1);
+        bitmapValue0.add(2);
+
+        bitmapValue1.add(2);
+        bitmapValue1.add(3);
+        bitmapValue1.add(4);
+
+        bitmapValue0Bytes = BitmapValueUtil.serializeToBytes(bitmapValue0);
+        bitmapValue1Bytes = BitmapValueUtil.serializeToBytes(bitmapValue1);
+    }
+
+    @Test
+    public void bitmapAndTest() throws Exception {
+        BitmapAndUDF bitmapAndUDF = new BitmapAndUDF();
+        bitmapAndUDF.initialize(new ObjectInspector[]{inputOI0, inputOI1});
+
+        GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[]{new GenericUDF.DeferredJavaObject(bitmapValue0Bytes), new GenericUDF.DeferredJavaObject(bitmapValue1Bytes)};
+        Object evaluate = bitmapAndUDF.evaluate(args);
+        BitmapValue resultBitmap = BitmapValueUtil.deserializeToBitmap((byte[]) evaluate);
+
+        Assert.assertEquals(1, resultBitmap.cardinality());
+        Assert.assertEquals("{2}", resultBitmap.toString());
+    }
+
+    @Test
+    public void bitmapOrTest() throws Exception {
+
+        BitmapOrUDF bitmapOrUDF = new BitmapOrUDF();
+        bitmapOrUDF.initialize(new ObjectInspector[]{inputOI0, inputOI1});
+
+        GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[]{new GenericUDF.DeferredJavaObject(bitmapValue0Bytes), new GenericUDF.DeferredJavaObject(bitmapValue1Bytes)};
+        Object evaluate = bitmapOrUDF.evaluate(args);
+        BitmapValue resultBitmap = BitmapValueUtil.deserializeToBitmap((byte[]) evaluate);
+
+        Assert.assertEquals(4, resultBitmap.cardinality());
+        Assert.assertEquals("{1,2,3,4}", resultBitmap.toString());
+    }
+
+    @Test
+    public void bitmapXorTest() throws Exception {
+        BitmapXorUDF bitmapXorUDF = new BitmapXorUDF();
+        bitmapXorUDF.initialize(new ObjectInspector[]{inputOI0, inputOI1});
+
+        GenericUDF.DeferredObject[] args = new GenericUDF.DeferredObject[]{new GenericUDF.DeferredJavaObject(bitmapValue0Bytes), new GenericUDF.DeferredJavaObject(bitmapValue1Bytes)};
+        Object evaluate = bitmapXorUDF.evaluate(args);
+        BitmapValue resultBitmap = BitmapValueUtil.deserializeToBitmap((byte[]) evaluate);
+
+        Assert.assertEquals(3, resultBitmap.cardinality());
+        Assert.assertEquals("{1,3,4}", resultBitmap.toString());
+    }
+
+    @Test
+    public void bitmapCountTest() throws Exception {
+        BitmapCountUDF bitmapCountUDF = new BitmapCountUDF();
+        bitmapCountUDF.initialize(new ObjectInspector[] { inputOI0 });
+        Object evaluate = bitmapCountUDF.evaluate(new GenericUDF.DeferredObject[] { new GenericUDF.DeferredJavaObject(bitmapValue0Bytes) });
+        Assert.assertEquals(2L, evaluate);
+
+        bitmapCountUDF.initialize(new ObjectInspector[] { inputOI1 });
+        Object evaluate1 = bitmapCountUDF.evaluate(new GenericUDF.DeferredObject[] { new GenericUDF.DeferredJavaObject(bitmapValue1Bytes) });
+        Assert.assertEquals(3L, evaluate1);
+    }
+}

From 7623b5cc3177290e9630b060e1442e6590817709 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Sat, 30 Dec 2023 20:08:23 +0800
Subject: [PATCH 206/299] [cleanup](move-memtable) remove namespace
 `stream_load` (#27441)

---
 be/src/runtime/exec_env.h                       | 10 +++-------
 be/src/runtime/exec_env_init.cpp                |  2 +-
 be/src/vec/sink/load_stream_stub_pool.cpp       |  3 ---
 be/src/vec/sink/load_stream_stub_pool.h         |  3 ---
 be/src/vec/sink/writer/vtablet_writer_v2.cpp    |  3 +--
 be/src/vec/sink/writer/vtablet_writer_v2.h      | 10 +++-------
 be/test/vec/exec/load_stream_stub_pool_test.cpp |  3 ---
 7 files changed, 8 insertions(+), 26 deletions(-)

diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index c97ff55a20647f..ddbb90b45a924a 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -51,9 +51,6 @@ struct RuntimeFilterTimerQueue;
 namespace taskgroup {
 class TaskGroupManager;
 }
-namespace stream_load {
-class LoadStreamStubPool;
-} // namespace stream_load
 namespace io {
 class S3FileBufferPool;
 class FileCacheFactory;
@@ -81,6 +78,7 @@ class ResultQueueMgr;
 class TMasterInfo;
 class LoadChannelMgr;
 class LoadStreamMgr;
+class LoadStreamStubPool;
 class StreamLoadExecutor;
 class RoutineLoadTaskExecutor;
 class SmallFileMgr;
@@ -238,9 +236,7 @@ class ExecEnv {
     }
 
 #endif
-    stream_load::LoadStreamStubPool* load_stream_stub_pool() {
-        return _load_stream_stub_pool.get();
-    }
+    LoadStreamStubPool* load_stream_stub_pool() { return _load_stream_stub_pool.get(); }
 
     vectorized::DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); }
 
@@ -352,7 +348,7 @@ class ExecEnv {
     // To save meta info of external file, such as parquet footer.
     FileMetaCache* _file_meta_cache = nullptr;
     std::unique_ptr<MemTableMemoryLimiter> _memtable_memory_limiter;
-    std::unique_ptr<stream_load::LoadStreamStubPool> _load_stream_stub_pool;
+    std::unique_ptr<LoadStreamStubPool> _load_stream_stub_pool;
     std::unique_ptr<vectorized::DeltaWriterV2Pool> _delta_writer_v2_pool;
     std::shared_ptr<WalManager> _wal_manager;
 
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index e612cb816b32a8..508ea7c7921011 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -217,7 +217,7 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
     _block_spill_mgr = new BlockSpillManager(store_paths);
     _group_commit_mgr = new GroupCommitMgr(this);
     _memtable_memory_limiter = std::make_unique<MemTableMemoryLimiter>();
-    _load_stream_stub_pool = std::make_unique<stream_load::LoadStreamStubPool>();
+    _load_stream_stub_pool = std::make_unique<LoadStreamStubPool>();
     _delta_writer_v2_pool = std::make_unique<vectorized::DeltaWriterV2Pool>();
     _wal_manager = WalManager::create_shared(this, config::group_commit_wal_path);
 
diff --git a/be/src/vec/sink/load_stream_stub_pool.cpp b/be/src/vec/sink/load_stream_stub_pool.cpp
index f7eeb96bb04ec1..c867756d8fb2be 100644
--- a/be/src/vec/sink/load_stream_stub_pool.cpp
+++ b/be/src/vec/sink/load_stream_stub_pool.cpp
@@ -23,8 +23,6 @@
 namespace doris {
 class TExpr;
 
-namespace stream_load {
-
 LoadStreams::LoadStreams(UniqueId load_id, int64_t dst_id, int num_use, LoadStreamStubPool* pool)
         : _load_id(load_id), _dst_id(dst_id), _use_cnt(num_use), _pool(pool) {}
 
@@ -87,5 +85,4 @@ void LoadStreamStubPool::erase(UniqueId load_id, int64_t dst_id) {
     _template_stubs.erase(load_id);
 }
 
-} // namespace stream_load
 } // namespace doris
diff --git a/be/src/vec/sink/load_stream_stub_pool.h b/be/src/vec/sink/load_stream_stub_pool.h
index 0eb14fc9351175..e41083825be07e 100644
--- a/be/src/vec/sink/load_stream_stub_pool.h
+++ b/be/src/vec/sink/load_stream_stub_pool.h
@@ -68,8 +68,6 @@ namespace doris {
 
 class LoadStreamStub;
 
-namespace stream_load {
-
 class LoadStreamStubPool;
 
 using Streams = std::vector<std::shared_ptr<LoadStreamStub>>;
@@ -118,5 +116,4 @@ class LoadStreamStubPool {
     std::unordered_map<std::pair<UniqueId, int64_t>, std::shared_ptr<LoadStreams>> _pool;
 };
 
-} // namespace stream_load
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 8ded964950b5b8..126dd4a1be070a 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -268,8 +268,7 @@ Status VTabletWriterV2::_open_streams(int64_t src_id) {
     return Status::OK();
 }
 
-Status VTabletWriterV2::_open_streams_to_backend(int64_t dst_id,
-                                                 ::doris::stream_load::LoadStreams& streams) {
+Status VTabletWriterV2::_open_streams_to_backend(int64_t dst_id, LoadStreams& streams) {
     const auto* node_info = _nodes_info->find_node(dst_id);
     DBUG_EXECUTE_IF("VTabletWriterV2._open_streams_to_backend.node_info_null",
                     { node_info = nullptr; });
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.h b/be/src/vec/sink/writer/vtablet_writer_v2.h
index fa21071da839b1..37558974197548 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.h
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.h
@@ -69,6 +69,7 @@
 namespace doris {
 class DeltaWriterV2;
 class LoadStreamStub;
+class LoadStreams;
 class ObjectPool;
 class RowDescriptor;
 class RuntimeState;
@@ -77,10 +78,6 @@ class TExpr;
 class TabletSchema;
 class TupleDescriptor;
 
-namespace stream_load {
-class LoadStreams;
-}
-
 namespace vectorized {
 
 class OlapTableBlockConvertor;
@@ -126,7 +123,7 @@ class VTabletWriterV2 final : public AsyncResultWriter {
 
     Status _open_streams(int64_t src_id);
 
-    Status _open_streams_to_backend(int64_t dst_id, ::doris::stream_load::LoadStreams& streams);
+    Status _open_streams_to_backend(int64_t dst_id, LoadStreams& streams);
 
     Status _incremental_open_streams(const std::vector<TOlapTablePartition>& partitions);
 
@@ -219,8 +216,7 @@ class VTabletWriterV2 final : public AsyncResultWriter {
     std::unordered_map<int64_t, std::unordered_map<int64_t, PTabletID>> _tablets_for_node;
     std::unordered_map<int64_t, std::vector<PTabletID>> _indexes_from_node;
 
-    std::unordered_map<int64_t, std::shared_ptr<::doris::stream_load::LoadStreams>>
-            _streams_for_node;
+    std::unordered_map<int64_t, std::shared_ptr<LoadStreams>> _streams_for_node;
 
     size_t _stream_index = 0;
     std::shared_ptr<DeltaWriterV2Map> _delta_writer_for_tablet;
diff --git a/be/test/vec/exec/load_stream_stub_pool_test.cpp b/be/test/vec/exec/load_stream_stub_pool_test.cpp
index 73f0950889673b..24da3bb6999f36 100644
--- a/be/test/vec/exec/load_stream_stub_pool_test.cpp
+++ b/be/test/vec/exec/load_stream_stub_pool_test.cpp
@@ -22,8 +22,6 @@
 
 namespace doris {
 
-namespace stream_load {
-
 class LoadStreamStubPoolTest : public testing::Test {
 public:
     LoadStreamStubPoolTest() = default;
@@ -50,5 +48,4 @@ TEST_F(LoadStreamStubPoolTest, test) {
     EXPECT_EQ(0, pool.templates_size());
 }
 
-} // namespace stream_load
 } // namespace doris

From 673a0137ce803b2c09f3e3c5ba4c4933c1fabd66 Mon Sep 17 00:00:00 2001
From: Luwei <814383175@qq.com>
Date: Sat, 30 Dec 2023 20:45:58 +0800
Subject: [PATCH 207/299] [fix] (schema change) fix missing data bug in schema
 change (#28938) (#29064)

---
 be/src/olap/schema_change.cpp | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 7f18826b406ef9..d19d399b1efaaa 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -470,6 +470,7 @@ Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader
                                              TabletSharedPtr new_tablet,
                                              TabletSchemaSPtr base_tablet_schema,
                                              TabletSchemaSPtr new_tablet_schema) {
+    bool eof = false;
     do {
         auto new_block = vectorized::Block::create_unique(new_tablet_schema->create_block());
         auto ref_block = vectorized::Block::create_unique(base_tablet_schema->create_block());
@@ -477,14 +478,15 @@ Status VSchemaChangeDirectly::_inner_process(RowsetReaderSharedPtr rowset_reader
         auto st = rowset_reader->next_block(ref_block.get());
         if (!st) {
             if (st.is<ErrorCode::END_OF_FILE>()) {
-                break;
+                eof = true;
+            } else {
+                return st;
             }
-            return st;
         }
 
         RETURN_IF_ERROR(_changer.change_block(ref_block.get(), new_block.get()));
         RETURN_IF_ERROR(rowset_writer->add_block(new_block.get()));
-    } while (true);
+    } while (!eof);
 
     RETURN_IF_ERROR(rowset_writer->flush());
     return Status::OK();
@@ -547,14 +549,16 @@ Status VSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_rea
 
     auto new_block = vectorized::Block::create_unique(new_tablet_schema->create_block());
 
+    bool eof = false;
     do {
         auto ref_block = vectorized::Block::create_unique(base_tablet_schema->create_block());
         auto st = rowset_reader->next_block(ref_block.get());
         if (!st) {
             if (st.is<ErrorCode::END_OF_FILE>()) {
-                break;
+                eof = true;
+            } else {
+                return st;
             }
-            return st;
         }
 
         RETURN_IF_ERROR(_changer.change_block(ref_block.get(), new_block.get()));
@@ -578,7 +582,7 @@ Status VSchemaChangeWithSorting::_inner_process(RowsetReaderSharedPtr rowset_rea
         // move unique ptr
         blocks.push_back(vectorized::Block::create_unique(new_tablet_schema->create_block()));
         swap(blocks.back(), new_block);
-    } while (true);
+    } while (!eof);
 
     RETURN_IF_ERROR(create_rowset());
 

From f5e2ea1699c14e378aa0c51c12d240e0911d8b95 Mon Sep 17 00:00:00 2001
From: lw112 <131352377+felixwluo@users.noreply.github.com>
Date: Sat, 30 Dec 2023 20:49:01 +0800
Subject: [PATCH 208/299] [fix][fe]Fixed show proc tablet inaccuracies (#29186)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both tables simply need to change the tablet id to get the same result from the show proc statement.

before：
image
Just change the tablet id to get the result, 10139 does not belong to the tablet in the table id of 10127

later:
image
---
 .../org/apache/doris/common/proc/TabletsProcDir.java     | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
index e13d1846a5aab1..d4c79db92df735 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
@@ -22,7 +22,6 @@
 import org.apache.doris.catalog.Replica;
 import org.apache.doris.catalog.Table;
 import org.apache.doris.catalog.Tablet;
-import org.apache.doris.catalog.TabletInvertedIndex;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.FeConstants;
@@ -209,8 +208,12 @@ public ProcNodeInterface lookup(String tabletIdStr) throws AnalysisException {
             throw new AnalysisException("Invalid tablet id format: " + tabletIdStr);
         }
 
-        TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
-        List<Replica> replicas = invertedIndex.getReplicasByTabletId(tabletId);
+        Tablet tablet = index.getTablet(tabletId);
+        if (tablet == null) {
+            throw new AnalysisException("Tablet[" + tabletId + "] does not exist");
+        }
+
+        List<Replica> replicas = tablet.getReplicas();
         return new ReplicasProcNode(tabletId, replicas);
     }
 

From 56648e0bd65937cf2f50779756a283c3e0051b19 Mon Sep 17 00:00:00 2001
From: Rohit Satardekar <rohitrs1983@yahoo.com>
Date: Sat, 30 Dec 2023 18:20:08 +0530
Subject: [PATCH 209/299] [Bug](security) BE SingleReplicaCompaction
 download_files function exists log print sensitive msg #28929 (#28930)

Co-authored-by: Rohit Satardekar <rohitrs1983@gmail.com>
---
 be/src/olap/single_replica_compaction.cpp | 18 ++++++++++++------
 be/src/olap/single_replica_compaction.h   |  1 +
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/be/src/olap/single_replica_compaction.cpp b/be/src/olap/single_replica_compaction.cpp
index 2a8edb60471154..72a0cdfc01c8f9 100644
--- a/be/src/olap/single_replica_compaction.cpp
+++ b/be/src/olap/single_replica_compaction.cpp
@@ -382,7 +382,7 @@ Status SingleReplicaCompaction::_download_files(DataDir* data_dir,
     // then it will try to clone from BE 2, but it will find the file 1 already exist, but file 1 with same
     // name may have different versions.
     VLOG_DEBUG << "single replica compaction begin to download files, remote path="
-               << remote_url_prefix << " local_path=" << local_path;
+               << _mask_token(remote_url_prefix) << " local_path=" << local_path;
     RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(local_path));
     RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(local_path));
 
@@ -442,11 +442,11 @@ Status SingleReplicaCompaction::_download_files(DataDir* data_dir,
 
         std::string local_file_path = local_path + file_name;
 
-        LOG(INFO) << "single replica compaction begin to download file from: " << remote_file_url
-                  << " to: " << local_file_path << ". size(B): " << file_size
-                  << ", timeout(s): " << estimate_timeout;
+        LOG(INFO) << "single replica compaction begin to download file from: "
+                  << _mask_token(remote_file_url) << " to: " << local_file_path
+                  << ". size(B): " << file_size << ", timeout(s): " << estimate_timeout;
 
-        auto download_cb = [&remote_file_url, estimate_timeout, &local_file_path,
+        auto download_cb = [this, &remote_file_url, estimate_timeout, &local_file_path,
                             file_size](HttpClient* client) {
             RETURN_IF_ERROR(client->init(remote_file_url));
             client->set_timeout_ms(estimate_timeout * 1000);
@@ -456,7 +456,8 @@ Status SingleReplicaCompaction::_download_files(DataDir* data_dir,
             uint64_t local_file_size = std::filesystem::file_size(local_file_path);
             if (local_file_size != file_size) {
                 LOG(WARNING) << "download file length error"
-                             << ", remote_path=" << remote_file_url << ", file_size=" << file_size
+                             << ", remote_path=" << _mask_token(remote_file_url)
+                             << ", file_size=" << file_size
                              << ", local_file_size=" << local_file_size;
                 return Status::InternalError("downloaded file size is not equal");
             }
@@ -582,4 +583,9 @@ Status SingleReplicaCompaction::_finish_clone(const string& clone_dir,
     return res;
 }
 
+std::string SingleReplicaCompaction::_mask_token(const std::string& str) {
+    std::regex pattern("token=[\\w|-]+");
+    return regex_replace(str, pattern, "token=******");
+}
+
 } // namespace doris
diff --git a/be/src/olap/single_replica_compaction.h b/be/src/olap/single_replica_compaction.h
index 7ccfe09f7aecbc..fd35da12645748 100644
--- a/be/src/olap/single_replica_compaction.h
+++ b/be/src/olap/single_replica_compaction.h
@@ -60,6 +60,7 @@ class SingleReplicaCompaction : public Compaction {
                            const std::string& local_path);
     Status _release_snapshot(const std::string& ip, int port, const std::string& snapshot_path);
     Status _finish_clone(const string& clone_dir, const Version& version);
+    std::string _mask_token(const std::string& str);
     CompactionType _compaction_type;
 
     DISALLOW_COPY_AND_ASSIGN(SingleReplicaCompaction);

From 56c3487a10445890a8a7b07d924d24b0494690c5 Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Sat, 30 Dec 2023 19:55:28 +0700
Subject: [PATCH 210/299] [test](ut) added UT cases for show backend stmt
 (#28603)

---
 .../doris/analysis/ShowBackendsStmtTest.java  | 86 +++++++++++++++++++
 1 file changed, 86 insertions(+)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java

diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java
new file mode 100644
index 00000000000000..f8584cc0e6f709
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackendsStmtTest.java
@@ -0,0 +1,86 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.ShowResultSetMetaData;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ShowBackendsStmtTest {
+    private Analyzer analyzer;
+    private ConnectContext ctx = new ConnectContext();
+
+    @Before
+    public void setUp() {
+        analyzer = AccessTestUtil.fetchAdminAnalyzer(true);
+        ctx.setSessionVariable(new SessionVariable());
+        ctx.setThreadLocalInfo();
+    }
+
+    @After
+    public void tearDown() {
+        ConnectContext.remove();
+    }
+
+    @Test
+    public void testAnalyze() throws Exception {
+        AtomicBoolean privilege = new AtomicBoolean(false);
+        new MockUp<AccessControllerManager>() {
+            @Mock
+            public boolean checkGlobalPriv(ConnectContext ctx, PrivPredicate wanted) {
+                return privilege.get();
+            }
+        };
+
+        ShowBackendsStmt stmt = new ShowBackendsStmt();
+        Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+
+        privilege.set(true);
+        stmt.analyze(analyzer);
+    }
+
+    @Test
+    public void getMetaData() {
+        ShowBackendsStmt stmt = new ShowBackendsStmt();
+        ShowResultSetMetaData result = stmt.getMetaData();
+        Assertions.assertEquals(result.getColumnCount(), 25);
+        result.getColumns().forEach(col -> Assertions.assertEquals(col.getType(), ScalarType.createVarchar(30)));
+    }
+
+    @Test
+    public void getRedirectStatus() {
+        ShowBackendsStmt stmt = new ShowBackendsStmt();
+        Assertions.assertEquals(RedirectStatus.FORWARD_NO_SYNC, stmt.getRedirectStatus());
+
+        ctx.getSessionVariable().forwardToMaster = false;
+        Assertions.assertEquals(RedirectStatus.NO_FORWARD, stmt.getRedirectStatus());
+    }
+}

From 018d8b1326341be0f5ba613132a3a630a70ee53c Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Sat, 30 Dec 2023 19:56:56 +0700
Subject: [PATCH 211/299] [fix](doc) spell errors fixes and align with code log
 for check-rpc-channel. (#28151)

---
 be/src/http/action/check_rpc_channel_action.cpp               | 3 +--
 .../en/docs/admin-manual/http-actions/be/check-rpc-channel.md | 4 ++--
 .../docs/admin-manual/http-actions/be/check-rpc-channel.md    | 2 +-
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/be/src/http/action/check_rpc_channel_action.cpp b/be/src/http/action/check_rpc_channel_action.cpp
index 8a1e81e6c760b0..4949b21b8f51c9 100644
--- a/be/src/http/action/check_rpc_channel_action.cpp
+++ b/be/src/http/action/check_rpc_channel_action.cpp
@@ -97,8 +97,7 @@ void CheckRPCChannelAction::handle(HttpRequest* req) {
         return;
     }
     if (response.status().status_code() == 0) {
-        std::string err =
-                fmt::format("open brpc connection to {0}:{1} succcess.", req_ip, req_port);
+        std::string err = fmt::format("open brpc connection to {0}:{1} success.", req_ip, req_port);
         LOG(WARNING) << err;
         HttpChannel::send_reply(req, HttpStatus::OK, err);
     } else {
diff --git a/docs/en/docs/admin-manual/http-actions/be/check-rpc-channel.md b/docs/en/docs/admin-manual/http-actions/be/check-rpc-channel.md
index caed50e7692674..15d4dca1c0552c 100644
--- a/docs/en/docs/admin-manual/http-actions/be/check-rpc-channel.md
+++ b/docs/en/docs/admin-manual/http-actions/be/check-rpc-channel.md
@@ -42,7 +42,7 @@ Check whether the connection cache is available
 
 * `remot_brpc_port`
 
-    Remot brpc port
+    Remote brpc port
 
 * `payload_size`
 
@@ -58,7 +58,7 @@ None
     {
         "msg":"success",
         "code":0,
-        "data": "open brpc connection to {host_to_check}:{remot_brpc_port} succcess.",
+        "data": "open brpc connection to {host_to_check}:{remot_brpc_port} success.",
         "count":0
     }
     ```
diff --git a/docs/zh-CN/docs/admin-manual/http-actions/be/check-rpc-channel.md b/docs/zh-CN/docs/admin-manual/http-actions/be/check-rpc-channel.md
index 5a796a6a1492a2..6203acc67adac1 100644
--- a/docs/zh-CN/docs/admin-manual/http-actions/be/check-rpc-channel.md
+++ b/docs/zh-CN/docs/admin-manual/http-actions/be/check-rpc-channel.md
@@ -58,7 +58,7 @@ under the License.
     {
         "msg":"success",
         "code":0,
-        "data": "open brpc connection to {host_to_check}:{remot_brpc_port} succcess.",
+        "data": "open brpc connection to {host_to_check}:{remot_brpc_port} success.",
         "count":0
     }
     ```

From 9a7c15b6358942fae98e90c4b060e844ade3f4eb Mon Sep 17 00:00:00 2001
From: zhiqiang <seuhezhiqiang@163.com>
Date: Sat, 30 Dec 2023 21:29:58 +0800
Subject: [PATCH 212/299] [fix](log) PipelineFragmentContext has already been
 destoried in try_close_task before logging it #29237

---
 be/src/pipeline/task_scheduler.cpp | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp
index d387680310452a..e0e668acbd9455 100644
--- a/be/src/pipeline/task_scheduler.cpp
+++ b/be/src/pipeline/task_scheduler.cpp
@@ -318,11 +318,6 @@ void TaskScheduler::_do_work(size_t index) {
                             fragment_ctx->is_canceled() ? PipelineTaskState::CANCELED
                                                         : PipelineTaskState::FINISHED,
                             status);
-            VLOG_DEBUG << fmt::format(
-                    "Task {} is eos, status {}.",
-                    PrintInstanceStandardInfo(task->query_context()->query_id(),
-                                              task->fragment_context()->get_fragment_instance_id()),
-                    get_state_name(task->get_state()));
             continue;
         }
 

From aa4b3d02f13f7d8291a9de7a51d6207fcd267fe5 Mon Sep 17 00:00:00 2001
From: dalong <60906603+DalongZhenZ@users.noreply.github.com>
Date: Sat, 30 Dec 2023 21:30:59 +0800
Subject: [PATCH 213/299] [bug]https://github.com/apache/doris/issues/29149
 (#29155)

Co-authored-by: 86157 <mapengfei@lnxjit.commapengfei@lnxjit.com>
---
 .../docs/query-acceleration/hight-concurrent-point-query.md     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/zh-CN/docs/query-acceleration/hight-concurrent-point-query.md b/docs/zh-CN/docs/query-acceleration/hight-concurrent-point-query.md
index 1a9f04dce0cc20..71b98d9d3a72ff 100644
--- a/docs/zh-CN/docs/query-acceleration/hight-concurrent-point-query.md
+++ b/docs/zh-CN/docs/query-acceleration/hight-concurrent-point-query.md
@@ -30,7 +30,7 @@ under the License.
 
 ## 背景 
 
-Doris 基于列存格式引擎构建，在高并发服务场景中，用户总是希望从系统中获取整行数据。但是，当表宽时，列存格式将大大放大随机读取 IO。Doris 查询引擎和计划对于某些简单的查询（如点查询）来说太重了。需要一个在 FE 的查询规划中规划短路径来处理这样的查询。FE 是 SQL 查询的访问层服务，使用 Java 编写，分析和解析 SQL 也会导致高并发查询的高 CPU 开销。为了解决上述问题，我们在 Doris 中引入了行存、短查询路径、PreparedStatement 来解决上诉问题，下面是开启这些优化的指南。
+Doris 基于列存格式引擎构建，在高并发服务场景中，用户总是希望从系统中获取整行数据。但是，当表宽时，列存格式将大大放大随机读取 IO。Doris 查询引擎和计划对于某些简单的查询（如点查询）来说太重了。需要一个在 FE 的查询规划中规划短路径来处理这样的查询。FE 是 SQL 查询的访问层服务，使用 Java 编写，分析和解析 SQL 也会导致高并发查询的高 CPU 开销。为了解决上述问题，我们在 Doris 中引入了行存、短查询路径、PreparedStatement 来解决上述问题，下面是开启这些优化的指南。
 
 ## 行存
 

From e91f6cb06958c6073cca7f1486b49cfdc9de92a8 Mon Sep 17 00:00:00 2001
From: Shaowen Yin <cosven@users.noreply.github.com>
Date: Sat, 30 Dec 2023 21:34:04 +0800
Subject: [PATCH 214/299] [chore](docs) straight_join is not supported (#28268)

---
 .../Data-Manipulation-Statements/Manipulation/SELECT.md         | 2 --
 .../Data-Manipulation-Statements/Manipulation/SELECT.md         | 2 --
 2 files changed, 4 deletions(-)

diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md
index fd79f1131a5b2f..4755b606967aca 100644
--- a/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md
+++ b/docs/en/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md
@@ -118,8 +118,6 @@ table_factor:
         ON conditional_expr }
 join_table:
     table_reference [INNER | CROSS] JOIN table_factor [join_condition]
-  | table_reference STRAIGHT_JOIN table_factor
-  | table_reference STRAIGHT_JOIN table_factor ON condition
   | table_reference LEFT [OUTER] JOIN table_reference join_condition
   | table_reference NATURAL [LEFT [OUTER]] JOIN table_factor
   | table_reference RIGHT [OUTER] JOIN table_reference join_condition
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md
index cdabcf9e5503b6..696d88e460c726 100644
--- a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Manipulation-Statements/Manipulation/SELECT.md
@@ -118,8 +118,6 @@ table_factor:
         ON conditional_expr }
 join_table:
     table_reference [INNER | CROSS] JOIN table_factor [join_condition]
-  | table_reference STRAIGHT_JOIN table_factor
-  | table_reference STRAIGHT_JOIN table_factor ON condition
   | table_reference LEFT [OUTER] JOIN table_reference join_condition
   | table_reference NATURAL [LEFT [OUTER]] JOIN table_factor
   | table_reference RIGHT [OUTER] JOIN table_reference join_condition

From 6aaa82987007885495d44530cefc5153200cdb32 Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Sat, 30 Dec 2023 22:25:27 +0800
Subject: [PATCH 215/299] [fix](Nereids) connection_id() should return bigint
 type (#29287)

---
 .../trees/expressions/functions/scalar/ConnectionId.java      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java
index 3b942b97512cde..7f16fa1a4acb69 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ConnectionId.java
@@ -22,7 +22,7 @@
 import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
 import org.apache.doris.nereids.trees.expressions.shape.LeafExpression;
 import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
-import org.apache.doris.nereids.types.VarcharType;
+import org.apache.doris.nereids.types.BigIntType;
 
 import com.google.common.collect.ImmutableList;
 
@@ -35,7 +35,7 @@ public class ConnectionId extends ScalarFunction
         implements LeafExpression, ExplicitlyCastableSignature, AlwaysNotNullable {
 
     public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
-            FunctionSignature.ret(VarcharType.SYSTEM_DEFAULT).args()
+            FunctionSignature.ret(BigIntType.INSTANCE).args()
     );
 
     public ConnectionId() {

From 10368a71a479006a5819cb080fe3d958cbf5bf2c Mon Sep 17 00:00:00 2001
From: Guangming Lu <71873108+LuGuangming@users.noreply.github.com>
Date: Sat, 30 Dec 2023 23:39:16 +0800
Subject: [PATCH 216/299] [fix][security]security optimize for executable
 binary file doris_be access should be restricted (#29303)

---
 bin/start_be.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/start_be.sh b/bin/start_be.sh
index debf0eda7d0e8b..ebf52a05ff9603 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -231,7 +231,7 @@ if [[ -f "${pidfile}" ]]; then
     fi
 fi
 
-chmod 755 "${DORIS_HOME}/lib/doris_be"
+chmod 550 "${DORIS_HOME}/lib/doris_be"
 echo "start time: $(date)" >>"${LOG_DIR}/be.out"
 
 if [[ ! -f '/bin/limit3' ]]; then

From 5b503e0a7b23799aea73a8464b88d59bb4ccf988 Mon Sep 17 00:00:00 2001
From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com>
Date: Sat, 30 Dec 2023 23:41:15 +0800
Subject: [PATCH 217/299] [chore] remove and add collaborators (#29286)

Removed: nanfeng1999, gitccl
Added: gavinchou, yujun777
---
 .asf.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.asf.yaml b/.asf.yaml
index 56f5cb95a458fe..2648776d4ddbbe 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -112,11 +112,11 @@ github:
     - platoneko
     - Lchangliang
     - freemandealer
-    - nanfeng1999
-    - gitccl
     - shuke987
     - wm1581066
     - KassieZ
+    - yujun777
+    - gavinchou
 
 notifications:
     pullrequests_status:  commits@doris.apache.org

From ac636a2abae76d6f805961018cd49cce412d0115 Mon Sep 17 00:00:00 2001
From: minghong <englefly@gmail.com>
Date: Sun, 31 Dec 2023 08:50:16 +0800
Subject: [PATCH 218/299] [fix](nereids) partition prunner evaluates "not
 x=const" on single node range #29164

---
 .../rules/OneRangePartitionEvaluator.java     |  2 +-
 .../test_multi_range_partition.groovy         | 36 +++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
index e6c2d92c8e2f9a..d63dea78e0e3e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java
@@ -458,7 +458,7 @@ public EvaluateRangeResult visitOr(Or or, EvaluateRangeInput context) {
     @Override
     public EvaluateRangeResult visitNot(Not not, EvaluateRangeInput context) {
         EvaluateRangeResult result = evaluateChildrenThenThis(not, context);
-        if (result.isRejectNot()) {
+        if (result.isRejectNot() && !result.result.equals(BooleanLiteral.TRUE)) {
             Map<Slot, ColumnRange> newRanges = Maps.newHashMap();
             for (Map.Entry<Slot, ColumnRange> entry : result.childrenResult.get(0).columnRanges.entrySet()) {
                 Slot slot = entry.getKey();
diff --git a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
index b12853db8a5937..0ee78332a16da9 100644
--- a/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
+++ b/regression-test/suites/nereids_rules_p0/partition_prune/test_multi_range_partition.groovy
@@ -272,4 +272,40 @@ suite("test_multi_range_partition") {
         sql "select * from pt where not k3 < 5;"
         contains "partitions=3/3 (p1,p2,p3)"
     }
+
+    sql "drop table if exists tt"
+    sql """
+    CREATE TABLE `tt` (
+     a bigint(20) NULL,
+     b bigint(20) null
+    ) ENGINE=OLAP
+    duplicate KEY(`a`)
+    COMMENT 'OLAP'
+    PARTITION BY RANGE(`a`, `b`)
+    (PARTITION p0 VALUES [("-2147483648", "-9223372036854775808"), ("5", "5")),
+    PARTITION p1 VALUES [("5", "5"), ("7", "7")),
+    PARTITION p2 VALUES [("7", "7"), ("2147483647", "2147483647000")))
+    DISTRIBUTED BY HASH(`b`) BUCKETS 10
+    PROPERTIES (
+    "replication_allocation" = "tag.location.default: 1",
+    "is_being_synced" = "false",
+    "storage_format" = "V2",
+    "light_schema_change" = "true",
+    "disable_auto_compaction" = "false",
+    "enable_single_replica_compaction" = "false"
+    ); 
+    """
+    sql "SET enable_nereids_planner=true"
+    sql "SET enable_fallback_to_original_planner=false"
+    sql "insert into tt values (0, 0), (6, 6), (8, 8)"
+    explain {
+        sql """SELECT
+            *
+            FROM
+            tt
+            WHERE
+            ( NOT ( a  IN ( 5 ) )
+            AND b  BETWEEN 2 AND 13 )"""
+        contains "partitions=3/3 (p0,p1,p2)"    
+    }
 }

From fcc4cfb9009ba146de23b75a58475469c563befc Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Sun, 31 Dec 2023 11:17:36 +0800
Subject: [PATCH 219/299] [Fix](Variant) add more info before crash in
 serialization (#29344)

---
 be/src/vec/columns/column_object.cpp                     | 8 ++++++++
 regression-test/suites/variant_p0/desc.groovy            | 2 +-
 regression-test/suites/variant_p0/load.groovy            | 2 +-
 regression-test/suites/variant_p0/with_index/load.groovy | 2 +-
 regression-test/suites/variant_p2/load.groovy            | 2 +-
 5 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index 782a2967f03230..fb2bf8c9072e53 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -1079,6 +1079,14 @@ bool ColumnObject::serialize_one_row_to_json_format(int row, rapidjson::StringBu
     VLOG_DEBUG << "dump structure " << JsonFunctions::print_json_value(*doc_structure);
 #endif
     for (const auto& subcolumn : subcolumns) {
+        if (subcolumn->data.data.empty() || subcolumn->data.get_finalized_column_ptr() == nullptr) {
+            // TODO this is a tmp defensive code to prevent from crash and
+            // print more info about crash info
+            LOG(WARNING) << "Dump crash debug info"
+                         << ", structure:" << JsonFunctions::print_json_value(*doc_structure)
+                         << ", num_rows: " << num_rows << ", row_position: " << row;
+            return false;
+        }
         find_and_set_leave_value(subcolumn->data.get_finalized_column_ptr(), subcolumn->path,
                                  subcolumn->data.get_least_common_type_serde(), root,
                                  doc_structure->GetAllocator(), row);
diff --git a/regression-test/suites/variant_p0/desc.groovy b/regression-test/suites/variant_p0/desc.groovy
index 1ccbaecb6c6d66..281893dd428e09 100644
--- a/regression-test/suites/variant_p0/desc.groovy
+++ b/regression-test/suites/variant_p0/desc.groovy
@@ -107,7 +107,7 @@ suite("regression_test_variant_desc", "nonConcurrent"){
         table_name = "no_sparse_columns"
         create_table.call(table_name, "4")
         sql "set enable_two_phase_read_opt = false;"
-        set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
+        set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1.0")
         sql """insert into  ${table_name} select 0, '{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}'  as json_str
             union  all select 0, '{"a": 1123}' as json_str union all select 0, '{"a" : 1234, "xxxx" : "kaana"}' as json_str from numbers("number" = "4096") limit 4096 ;"""
         qt_sql_3 """desc ${table_name}"""
diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy
index b23ae19d12f1a2..a8793bddec13d0 100644
--- a/regression-test/suites/variant_p0/load.groovy
+++ b/regression-test/suites/variant_p0/load.groovy
@@ -289,7 +289,7 @@ suite("regression_test_variant", "variant_type"){
         create_table.call(table_name, "4")
         // sql "set enable_two_phase_read_opt = false;"
         // no sparse columns
-        set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
+        set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1.0")
         load_json_data.call(table_name, """${getS3Url() + '/load/logdata.json'}""")
         qt_sql_32 """ select json_extract(v, "\$.json.parseFailed") from logdata where  json_extract(v, "\$.json.parseFailed") != 'null' order by k limit 1;"""
         qt_sql_32_1 """select cast(v:json.parseFailed as string) from  logdata where cast(v:json.parseFailed as string) is not null and k = 162 limit 1;"""
diff --git a/regression-test/suites/variant_p0/with_index/load.groovy b/regression-test/suites/variant_p0/with_index/load.groovy
index e124629169ee8e..f4bda1216ebcdc 100644
--- a/regression-test/suites/variant_p0/with_index/load.groovy
+++ b/regression-test/suites/variant_p0/with_index/load.groovy
@@ -43,7 +43,7 @@ suite("regression_test_variant_with_index", "nonConcurrent"){
         }
         assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout")
     }
-    set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0")
+    set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1.0")
     set_be_config.call("variant_threshold_rows_to_estimate_sparse_column", "0")
     def table_name = "var_with_index"
     sql "DROP TABLE IF EXISTS var_with_index"
diff --git a/regression-test/suites/variant_p2/load.groovy b/regression-test/suites/variant_p2/load.groovy
index 5813d02fb63a8f..2d232e1e255c08 100644
--- a/regression-test/suites/variant_p2/load.groovy
+++ b/regression-test/suites/variant_p2/load.groovy
@@ -88,7 +88,7 @@ suite("github_event_advance_p2", "variant_type,p2"){
 
     try {
         def table_name = "github_events"
-        set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
+        set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1.0")
         def s3load_paral_wait = {tbl, fmt, path, paral ->
             String ak = getS3AK()
             String sk = getS3SK()

From cc6013bfab4c30cb342fa05c65ca876ebd6b5096 Mon Sep 17 00:00:00 2001
From: bobhan1 <bh2444151092@outlook.com>
Date: Sun, 31 Dec 2023 11:18:37 +0800
Subject: [PATCH 220/299] [fix](partial update) Fix error message when doing
 strict mode partial update on a table with column that is non-nullable and
 has no default value #29218

---
 .../olap/rowset/segment_v2/segment_writer.cpp | 26 +++++++------
 .../segment_v2/vertical_segment_writer.cpp    | 27 ++++++-------
 .../test_partial_update_strict_mode.out       | 16 ++++++++
 .../test_partial_update_strict_mode.groovy    | 39 +++++++++++++++++++
 4 files changed, 83 insertions(+), 25 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index e2fd6d5f503ee4..5434c23ef8e7ac 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -468,21 +468,23 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block*
                 _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id,
                                                   DeleteBitmap::TEMP_VERSION_COMMON},
                                                  segment_pos);
-            }
 
-            if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) {
-                std::string error_column;
-                for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) {
-                    const TabletColumn& col = _tablet_schema->column(cid);
-                    if (!col.has_default_value() && !col.is_nullable()) {
-                        error_column = col.name();
-                        break;
+            } else {
+                if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) {
+                    std::string error_column;
+                    for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) {
+                        const TabletColumn& col = _tablet_schema->column(cid);
+                        if (!col.has_default_value() && !col.is_nullable()) {
+                            error_column = col.name();
+                            break;
+                        }
                     }
+                    return Status::Error<INVALID_SCHEMA, false>(
+                            "the unmentioned column `{}` should have default value or be nullable "
+                            "for "
+                            "newly inserted rows in non-strict mode partial update",
+                            error_column);
                 }
-                return Status::Error<INVALID_SCHEMA, false>(
-                        "the unmentioned column `{}` should have default value or be nullable for "
-                        "newly inserted rows in non-strict mode partial update",
-                        error_column);
             }
             has_default_or_nullable = true;
             use_default_or_null_flag.emplace_back(true);
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index fe3204e242e436..0aa8cea81ad3ad 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -402,21 +402,22 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da
                 _mow_context->delete_bitmap->add({_opts.rowset_ctx->rowset_id, _segment_id,
                                                   DeleteBitmap::TEMP_VERSION_COMMON},
                                                  segment_pos);
-            }
-
-            if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) {
-                std::string error_column;
-                for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) {
-                    const TabletColumn& col = _tablet_schema->column(cid);
-                    if (!col.has_default_value() && !col.is_nullable()) {
-                        error_column = col.name();
-                        break;
+            } else {
+                if (!_opts.rowset_ctx->partial_update_info->can_insert_new_rows_in_partial_update) {
+                    std::string error_column;
+                    for (auto cid : _opts.rowset_ctx->partial_update_info->missing_cids) {
+                        const TabletColumn& col = _tablet_schema->column(cid);
+                        if (!col.has_default_value() && !col.is_nullable()) {
+                            error_column = col.name();
+                            break;
+                        }
                     }
+                    return Status::Error<INVALID_SCHEMA, false>(
+                            "the unmentioned column `{}` should have default value or be nullable "
+                            "for "
+                            "newly inserted rows in non-strict mode partial update",
+                            error_column);
                 }
-                return Status::Error<INVALID_SCHEMA, false>(
-                        "the unmentioned column `{}` should have default value or be nullable for "
-                        "newly inserted rows in non-strict mode partial update",
-                        error_column);
             }
             has_default_or_nullable = true;
             use_default_or_null_flag.emplace_back(true);
diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.out b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.out
index aa1946a5e41462..93fa33d81f9e96 100644
--- a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.out
+++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.out
@@ -25,6 +25,14 @@
 1	kevin	18	shenzhen	400	2023-07-01T12:00
 3	steve	23	beijing	500	2023-07-03T12:00:02
 
+-- !sql --
+1	kevin	18	shenzhen	400	2023-07-01T12:00
+3	steve	23	beijing	500	2023-07-03T12:00:02
+
+-- !sql --
+1	kevin	18	shenzhen	400	2023-07-01T12:00
+3	steve	23	beijing	500	2023-07-03T12:00:02
+
 -- !sql --
 1	kevin	18	shenzhen	400	2023-07-01T12:00
 
@@ -51,3 +59,11 @@
 1	kevin	18	shenzhen	400	2023-07-01T12:00
 3	steve	23	beijing	500	2023-07-03T12:00:02
 
+-- !sql --
+1	kevin	18	shenzhen	400	2023-07-01T12:00
+3	steve	23	beijing	500	2023-07-03T12:00:02
+
+-- !sql --
+1	kevin	18	shenzhen	400	2023-07-01T12:00
+3	steve	23	beijing	500	2023-07-03T12:00:02
+
diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
index 0a399f5f317320..5fad6b4aca114d 100644
--- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
+++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_strict_mode.groovy
@@ -219,6 +219,45 @@ suite("test_partial_update_strict_mode", "p0") {
             sql "sync"
             qt_sql """select * from ${tableName4} order by id;"""
             sql """ DROP TABLE IF EXISTS ${tableName4}; """
+
+
+            // column `city` is non-nullable and has no default value
+            def tableName5 = "test_partial_update_strict_mode5"
+            sql """ DROP TABLE IF EXISTS ${tableName5} """
+            sql """
+                    CREATE TABLE ${tableName5} ( 
+                        `id` int(11) NULL, 
+                        `name` varchar(10) NULL,
+                        `age` int(11) NULL DEFAULT "20", 
+                        `city` varchar(10) NOT NULL, 
+                        `balance` decimalv3(9, 0) NULL, 
+                        `last_access_time` datetime NULL 
+                    ) ENGINE = OLAP UNIQUE KEY(`id`) 
+                    COMMENT 'OLAP' DISTRIBUTED BY HASH(`id`) 
+                    BUCKETS 4 PROPERTIES ( 
+                        "replication_allocation" = "tag.location.default: 1", 
+                        "storage_format" = "V2", 
+                        "enable_unique_key_merge_on_write" = "true", 
+                        "light_schema_change" = "true", 
+                        "disable_auto_compaction" = "false", 
+                        "enable_single_replica_compaction" = "false",
+                        "store_row_column" = "${use_row_store}"); """
+            sql """insert into ${tableName5} values(1,"kevin",18,"shenzhen",400,"2023-07-01 12:00:00");"""
+            sql """insert into ${tableName5} values(3,"steve",23,"beijing",500,"2023-07-03 12:00:02");"""
+            sql "sync;"
+            qt_sql """select * from ${tableName5} order by id;"""
+            sql "set enable_unique_key_partial_update=true;"
+            sql "set enable_insert_strict=true;"
+            sql "sync"
+            test {
+                sql """insert into ${tableName5}(id,balance,last_access_time) values
+                    (1,600,"2023-08-03 12:00:01"),
+                    (2,500,"2023-07-03 12:00:01"),
+                    (4,23,"2023-07-03 12:00:02");"""
+                exception "Insert has filtered data in strict mode"
+            }
+            sql "sync;"
+            qt_sql """select * from ${tableName5} order by id;"""
         }
     }
 }

From 3661c316c914a12f85de8d97f5355f8518f903e5 Mon Sep 17 00:00:00 2001
From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com>
Date: Sun, 31 Dec 2023 12:51:21 +0800
Subject: [PATCH 221/299] Revert "[improvement](create tablet) backend create
 tablet round robin among disks (#23218)" (#29347)

This reverts commit df5b5ae0cb2f30f026ec104a64b4d9a5ce2904f3.
---
 be/src/olap/data_dir.cpp       |  10 +---
 be/src/olap/data_dir.h         |   6 +-
 be/src/olap/storage_engine.cpp | 104 ++++++---------------------------
 be/src/olap/storage_engine.h   |   2 -
 4 files changed, 19 insertions(+), 103 deletions(-)

diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 80a0f7131fdad1..154673dfb4ff95 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -864,15 +864,7 @@ void DataDir::update_remote_data_size(int64_t size) {
     disks_remote_used_capacity->set_value(size);
 }
 
-size_t DataDir::disk_capacity() const {
-    return _disk_capacity_bytes;
-}
-
-size_t DataDir::disk_available() const {
-    return _available_bytes;
-}
-
-size_t DataDir::tablet_num() const {
+size_t DataDir::tablet_size() const {
     std::lock_guard<std::mutex> l(_mutex);
     return _tablet_set.size();
 }
diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h
index 8cdb54b5ccb62a..84b79a8b2876b6 100644
--- a/be/src/olap/data_dir.h
+++ b/be/src/olap/data_dir.h
@@ -127,11 +127,7 @@ class DataDir {
 
     void update_remote_data_size(int64_t size);
 
-    size_t disk_capacity() const;
-
-    size_t disk_available() const;
-
-    size_t tablet_num() const;
+    size_t tablet_size() const;
 
     void disks_compaction_score_increment(int64_t delta);
 
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index cf4687caf3e6bc..4b3827daf29394 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -450,103 +450,33 @@ Status StorageEngine::set_cluster_id(int32_t cluster_id) {
 
 std::vector<DataDir*> StorageEngine::get_stores_for_create_tablet(
         TStorageMedium::type storage_medium) {
-    struct DirInfo {
-        DataDir* data_dir;
-
-        size_t disk_available;
-        //if disk_available is high, then available_level is small
-        int available_level;
-
-        int tablet_num;
-
-        bool operator<(const DirInfo& other) const {
-            if (available_level != other.available_level) {
-                return available_level < other.available_level;
-            }
-            if (tablet_num != other.tablet_num) {
-                return tablet_num < other.tablet_num;
-            }
-            return data_dir->path_hash() < other.data_dir->path_hash();
-        }
-    };
-    std::map<size_t, int> available_levels;
-    std::vector<DirInfo> dir_infos;
-    int next_index = 0;
-    size_t max_disk_capacity = 0;
+    std::vector<DataDir*> stores;
     {
         std::lock_guard<std::mutex> l(_store_lock);
-        next_index = _store_next_index[storage_medium]++;
-        if (next_index < 0) {
-            next_index = 0;
-            _store_next_index[storage_medium] = next_index + 1;
-        }
         for (auto& it : _store_map) {
-            DataDir* data_dir = it.second;
-            if (data_dir->is_used()) {
+            if (it.second->is_used()) {
                 if (_available_storage_medium_type_count == 1 ||
-                    data_dir->storage_medium() == storage_medium) {
-                    size_t disk_available = data_dir->disk_available();
-                    DirInfo dir_info;
-                    dir_info.data_dir = data_dir;
-                    dir_info.available_level = disk_available;
-                    dir_infos.push_back(dir_info);
-                    available_levels[disk_available] = 0;
-                    size_t disk_capacity = data_dir->disk_capacity();
-                    if (max_disk_capacity < disk_capacity) {
-                        max_disk_capacity = disk_capacity;
-                    }
+                    it.second->storage_medium() == storage_medium) {
+                    stores.push_back(it.second);
                 }
             }
         }
     }
-
-    std::vector<DataDir*> stores;
-    if (dir_infos.empty()) {
-        return stores;
-    }
-
-    // if two disk available diff not exceeds 20% capacity, then they are the same available level.
-    size_t same_level_available_diff = std::max<size_t>(max_disk_capacity / 5, 1);
-    int level = 0;
-    size_t level_start_available = available_levels.rbegin()->first;
-    for (auto rit = available_levels.rbegin(); rit != available_levels.rend(); rit++) {
-        if (level_start_available - rit->first >= same_level_available_diff) {
-            level_start_available = rit->first;
-            level++;
-        }
-        rit->second = level;
-    }
-
-    for (auto& dir_info : dir_infos) {
-        dir_info.tablet_num = dir_info.data_dir->tablet_num();
-        dir_info.available_level = available_levels[dir_info.disk_available];
-    }
-
-    std::sort(dir_infos.begin(), dir_infos.end());
-
-    // Suppose there are five data dirs (D1, D2, D3, D4, D5).
-    // D1/D2/D3 contain 1 tablet,  D4/D5 contain 2 tablets.
-    // If three creating tablets threads simultaneously invoke this function to get stores,
-    // then the return stores will be as below:
-    // thread 1: (D1, D2, D3, D4, D5)
-    // thread 2: (D2, D3, D1, D5, D4)
-    // thread 3: (D3, D1, D2, D4, D5)
-    stores.reserve(dir_infos.size());
-    for (size_t i = 0; i < dir_infos.size();) {
-        size_t end = i + 1;
-        while (end < dir_infos.size() && dir_infos[i].tablet_num == dir_infos[end].tablet_num &&
-               dir_infos[i].available_level == dir_infos[end].available_level) {
-            end++;
-        }
-        // data dirs [i, end) have the same tablet size, round robin range [i, end)
-        size_t count = end - i;
-        for (size_t k = 0; k < count; k++) {
-            size_t index = i + (k + next_index) % count;
-            stores.push_back(dir_infos[index].data_dir);
+    std::random_device rd;
+    std::mt19937 g(rd());
+    std::shuffle(stores.begin(), stores.end(), g);
+    // Two random choices
+    for (int i = 0; i < stores.size(); i++) {
+        int j = i + 1;
+        if (j < stores.size()) {
+            if (stores[i]->tablet_size() > stores[j]->tablet_size()) {
+                std::swap(stores[i], stores[j]);
+            }
+            std::shuffle(stores.begin() + j, stores.end(), g);
+        } else {
+            break;
         }
-        i = end;
     }
-
     return stores;
 }
 
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 750e7a4ca21cab..5d71a99617f58e 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -491,8 +491,6 @@ class StorageEngine {
     bool _clear_segment_cache = false;
 
     std::atomic<bool> _need_clean_trash {false};
-    // next index for create tablet
-    std::map<TStorageMedium::type, int> _store_next_index;
 
     DISALLOW_COPY_AND_ASSIGN(StorageEngine);
 };

From 74202fe7e288ad875bf8f50bd6e7e24c3ea560c3 Mon Sep 17 00:00:00 2001
From: Guangming Lu <71873108+LuGuangming@users.noreply.github.com>
Date: Sun, 31 Dec 2023 13:42:56 +0800
Subject: [PATCH 222/299] [Enhancement] support user custom defined privite
 settings,xml for build fe (#29015)

Issue Number: close #29014
---
 build.sh                                            | 13 +++++++++++--
 docs/en/community/developer-guide/be-vscode-dev.md  |  5 +++++
 .../community/developer-guide/be-vscode-dev.md      |  5 +++++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/build.sh b/build.sh
index 4f5c83fca58dae..3b819929d00d39 100755
--- a/build.sh
+++ b/build.sh
@@ -590,9 +590,18 @@ if [[ "${FE_MODULES}" != '' ]]; then
         clean_fe
     fi
     if [[ "${DISABLE_JAVA_CHECK_STYLE}" = "ON" ]]; then
-        "${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -Dskip.doc=true -DskipTests -Dcheckstyle.skip=true ${MVN_OPT:+${MVN_OPT}}
+        # Allowed user customer set env param USER_SETTINGS_MVN_REPO means settings.xml file path
+        if [[ -n ${USER_SETTINGS_MVN_REPO} && -f ${USER_SETTINGS_MVN_REPO} ]]; then
+            "${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -Dskip.doc=true -DskipTests -Dcheckstyle.skip=true ${MVN_OPT:+${MVN_OPT}} -gs "${USER_SETTINGS_MVN_REPO}"
+        else
+            "${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -Dskip.doc=true -DskipTests -Dcheckstyle.skip=true ${MVN_OPT:+${MVN_OPT}}
+        fi
     else
-        "${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -Dskip.doc=true -DskipTests ${MVN_OPT:+${MVN_OPT}}
+        if [[ -n ${USER_SETTINGS_MVN_REPO} && -f ${USER_SETTINGS_MVN_REPO} ]]; then
+            "${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -Dskip.doc=true -DskipTests ${MVN_OPT:+${MVN_OPT}} -gs "${USER_SETTINGS_MVN_REPO}"
+        else
+            "${MVN_CMD}" package -pl ${FE_MODULES:+${FE_MODULES}} -Dskip.doc=true -DskipTests ${MVN_OPT:+${MVN_OPT}}
+        fi
     fi
     cd "${DORIS_HOME}"
 fi
diff --git a/docs/en/community/developer-guide/be-vscode-dev.md b/docs/en/community/developer-guide/be-vscode-dev.md
index 21a3e689cbd36e..e7964e2a3f7bb3 100644
--- a/docs/en/community/developer-guide/be-vscode-dev.md
+++ b/docs/en/community/developer-guide/be-vscode-dev.md
@@ -93,6 +93,11 @@ Note: This compilation has the following instructions:
 
 If nothing happens, the compilation should be successful, and the final deployment file will be output to the /home/workspace/doris/output/ directory. If you still encounter other problems, you can refer to the doris installation document http://doris.apache.org.
 
+Note: If you want to specify the private maven repository address separately when compiling fe, you can set the environment variable USER_SETTINGS_MVN_REPO to specify the file path to settings.xml.
+Example:
+```
+  export USER_SETTINGS_MVN_REPO="/xxx/xxx/settings.xml"
+```
 ## Deployment and debugging(GDB)
 
 1. Authorize be compilation result files
diff --git a/docs/zh-CN/community/developer-guide/be-vscode-dev.md b/docs/zh-CN/community/developer-guide/be-vscode-dev.md
index eecc24372612f5..9244ac61ac2efe 100644
--- a/docs/zh-CN/community/developer-guide/be-vscode-dev.md
+++ b/docs/zh-CN/community/developer-guide/be-vscode-dev.md
@@ -92,6 +92,11 @@ cd /home/workspace/doris
 
 如果不出意外，应该会编译成功，最终的部署文件将产出到 /home/workspace/doris/output/ 目录下。如果还遇到其他问题，可以参照 doris 的安装文档 http://doris.apache.org。
 
+注意：如果编译fe时希望单独指定私有的maven仓地址，可以设置环境变量USER_SETTINGS_MVN_REPO指定settings.xml的文件路径。
+举例：
+```
+  export USER_SETTINGS_MVN_REPO="/xxx/xxx/settings.xml"
+```
 ## 部署调试(GDB)
 
 1. 给 be 编译结果文件授权

From f92c0acba0a4fcfd87e87b0e945c1ed3083e933d Mon Sep 17 00:00:00 2001
From: Guangming Lu <71873108+LuGuangming@users.noreply.github.com>
Date: Sun, 31 Dec 2023 13:48:08 +0800
Subject: [PATCH 223/299] [Bug] When using the hive catalog the hive table name
 is uppercase, the Doris query is reported wrong #29058 (#29059)

---
 .../apache/doris/datasource/hive/event/AlterTableEvent.java   | 3 ++-
 .../apache/doris/datasource/hive/event/CreateTableEvent.java  | 3 ++-
 .../apache/doris/datasource/hive/event/MetastoreEvent.java    | 4 ++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
index 706f8cd303de95..6de71fbbc592a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
 package org.apache.doris.datasource.hive.event;
 
 import org.apache.doris.catalog.Env;
@@ -29,6 +28,7 @@
 
 import java.security.SecureRandom;
 import java.util.List;
+import java.util.Locale;
 
 /**
  * MetastoreEvent for ALTER_TABLE event type
@@ -65,6 +65,7 @@ private AlterTableEvent(NotificationEvent event, String catalogName) {
                     (JSONAlterTableMessage) MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
                             .getAlterTableMessage(event.getMessage());
             tableAfter = Preconditions.checkNotNull(alterTableMessage.getTableObjAfter());
+            tableAfter.setTableName(tableAfter.getTableName().toLowerCase(Locale.ROOT));
             tableBefore = Preconditions.checkNotNull(alterTableMessage.getTableObjBefore());
             tblNameAfter = tableAfter.getTableName();
         } catch (Exception e) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
index 3dff6420a8a665..4c3615fbda8915 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
 package org.apache.doris.datasource.hive.event;
 
 import org.apache.doris.catalog.Env;
@@ -28,6 +27,7 @@
 import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage;
 
 import java.util.List;
+import java.util.Locale;
 
 /**
  * MetastoreEvent for CREATE_TABLE event type
@@ -51,6 +51,7 @@ private CreateTableEvent(NotificationEvent event, String catalogName) throws Met
                     MetastoreEventsProcessor.getMessageDeserializer(event.getMessageFormat())
                             .getCreateTableMessage(event.getMessage());
             hmsTbl = Preconditions.checkNotNull(createTableMessage.getTableObj());
+            hmsTbl.setTableName(hmsTbl.getTableName().toLowerCase(Locale.ROOT));
         } catch (Exception e) {
             throw new MetastoreNotificationException(
                     debugString("Unable to deserialize the event message"), e);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
index 9dd4e79741ef2d..f9771562ed4bb2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
@@ -15,7 +15,6 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
 package org.apache.doris.datasource.hive.event;
 
 import org.apache.doris.datasource.hive.HMSCachedClient;
@@ -25,6 +24,7 @@
 import org.apache.logging.log4j.Logger;
 
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
 /**
@@ -77,7 +77,7 @@ protected MetastoreEvent(long eventId, String catalogName, String dbName,
 
     protected MetastoreEvent(NotificationEvent event, String catalogName) {
         this.event = event;
-        this.dbName = event.getDbName();
+        this.dbName = event.getDbName().toLowerCase(Locale.ROOT);
         this.tblName = event.getTableName();
         this.eventId = event.getEventId();
         this.eventTime = event.getEventTime() * 1000L;

From 4634c723d7fb86f434060c0e6fa6168d92630cb9 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Sun, 31 Dec 2023 13:50:13 +0800
Subject: [PATCH 224/299] [fix](move-memtable) only check missing tablets when
 commit info is not empty (#29326)

Some sink may not get any success tablets, because load stream will only report them at the last CLOSE_LOAD.
For these sinks (no commit info), we should not check missing tablets since it's expected to be empty.
---
 be/src/vec/sink/writer/vtablet_writer_v2.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 126dd4a1be070a..77df9b07f18701 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -553,9 +553,9 @@ Status VTabletWriterV2::close(Status exec_status) {
                 }
             }
         }
-        if (!_missing_tablets.empty()) {
-            std::stringstream ss("pre-commit check failed, ");
-            ss << "missing " << _missing_tablets.size() << " tablets:";
+        if (!tablet_commit_infos.empty() && !_missing_tablets.empty()) {
+            std::stringstream ss;
+            ss << "pre-commit check failed, missing " << _missing_tablets.size() << " tablets:";
             int print_limit = 3;
             for (auto tablet_id : _missing_tablets | std::ranges::views::take(print_limit)) {
                 ss << " (tablet_id=" << tablet_id;

From 6c74313f6024f9111cfb20fc5765e28ebc0f5f3d Mon Sep 17 00:00:00 2001
From: Nitin-Kashyap <66766227+Nitin-Kashyap@users.noreply.github.com>
Date: Sun, 31 Dec 2023 14:49:57 +0700
Subject: [PATCH 225/299] [Fix](ut)(show) added UT cases for show backup stmt
 (#28604)

 * Added UT for show backup
 * fixed some toSql() space and missing CLAUSE
 * Updated Doc with relevant syntax
---
 .../Show-Statements/SHOW-BACKUP.md            |  45 +++---
 .../Show-Statements/SHOW-BACKUP.md            |   1 +
 .../apache/doris/analysis/ShowBackupStmt.java |   8 +-
 .../doris/analysis/ShowBackupStmtTest.java    | 150 ++++++++++++++++++
 4 files changed, 178 insertions(+), 26 deletions(-)
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackupStmtTest.java

diff --git a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md
index 72c45126569c7b..d72b7a07fd69c8 100644
--- a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md
+++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md
@@ -38,32 +38,33 @@ grammar:
 
 ```sql
  SHOW BACKUP [FROM db_name]
+    [WHERE SnapshotName ( LIKE | = ) 'snapshot name']
 ````
 
 illustrate:
 
-1. Only the most recent BACKUP task is saved in Doris.
-1. The meaning of each column is as follows:
-            JobId: Unique job id
-            SnapshotName: The name of the backup
-            DbName: belongs to the database
-            State: current stage
-                PENDING: The initial state after submitting the job
-                SNAPSHOTING: Executing snapshot
-                UPLOAD_SNAPSHOT: Snapshot completed, ready to upload
-                UPLOADING: Snapshot uploading
-                SAVE_META: Save job meta information to a local file
-                UPLOAD_INFO: Upload job meta information
-                FINISHED: The job was successful
-                CANCELLED: Job failed
-            BackupObjs: Backed up tables and partitions
-            CreateTime: task submission time
-            SnapshotFinishedTime: Snapshot completion time
-            UploadFinishedTime: Snapshot upload completion time
-            FinishedTime: Job finish time
-            UnfinishedTasks: Displays unfinished subtask ids during SNAPSHOTING and UPLOADING stages
-            Status: If the job fails, display the failure message
-            Timeout: Job timeout, in seconds
+   1. Only the most recent BACKUP task is saved in Doris.
+   2. The meaning of each column is as follows:
+       - `JobId`: Unique job id
+       - `SnapshotName`: The name of the backup
+       - `DbName`: belongs to the database
+       - `State`: current stage
+           - `PENDING`: The initial state after submitting the job
+           - `SNAPSHOTING`: Executing snapshot
+           - `UPLOAD_SNAPSHOT`: Snapshot completed, ready to upload
+           - `UPLOADING`: Snapshot uploading
+           - `SAVE_META`: Save job meta information to a local file
+           - `UPLOAD_INFO`: Upload job meta information
+           - `FINISHED`: The job was successful
+           - `CANCELLED`: Job failed
+       - `BackupObjs`: Backed up tables and partitions
+       - `CreateTime`: task submission time
+       - `SnapshotFinishedTime`: Snapshot completion time
+       - `UploadFinishedTime`: Snapshot upload completion time
+       - `FinishedTime`: Job finish time
+       - `UnfinishedTasks`: Displays unfinished subtask ids during SNAPSHOTING and UPLOADING stages
+       - `Status`: If the job fails, display the failure message
+       - `Timeout`: Job timeout, in seconds
 
 ### Example
 
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md
index da6fe59b388d1e..2cb838e9bb3ef4 100644
--- a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-BACKUP.md
@@ -38,6 +38,7 @@ SHOW BACKUP
 
 ```sql
  SHOW BACKUP [FROM db_name]
+     [WHERE SnapshotName ( LIKE | = ) 'snapshot name' ]
 ```
 
 说明：
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java
index a302a5742b79b3..041345a198e176 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowBackupStmt.java
@@ -136,13 +136,13 @@ public ShowResultSetMetaData getMetaData() {
     @Override
     public String toSql() {
         StringBuilder builder = new StringBuilder();
-        builder.append("SHOW BACKUP");
-        if (dbName != null) {
-            builder.append(" FROM `").append(dbName).append("` ");
+        builder.append("SHOW BACKUP ");
+        if (Strings.isNullOrEmpty(dbName)) {
+            builder.append("FROM `").append(dbName).append("` ");
         }
 
         if (where != null) {
-            builder.append(where.toSql());
+            builder.append("WHERE ").append(where.toSql());
         }
 
         return builder.toString();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackupStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackupStmtTest.java
new file mode 100644
index 00000000000000..6d340e5d6f1bf0
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowBackupStmtTest.java
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.analysis.LikePredicate.Operator;
+import org.apache.doris.catalog.ScalarType;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.ShowResultSetMetaData;
+
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.jupiter.api.Assertions;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class ShowBackupStmtTest {
+    private Analyzer analyzer;
+    private ConnectContext ctx = new ConnectContext();
+
+    @Before
+    public void setUp() {
+        analyzer = AccessTestUtil.fetchAdminAnalyzer(true);
+        ctx.setSessionVariable(new SessionVariable());
+        ctx.setThreadLocalInfo();
+    }
+
+    @After
+    public void tearDown() {
+        ConnectContext.remove();
+    }
+
+    @Test
+    public void getMetaData() {
+        ShowBackupStmt stmt = new ShowBackupStmt("", null);
+        ShowResultSetMetaData result = stmt.getMetaData();
+        Assertions.assertEquals(result.getColumnCount(), 14);
+        result.getColumns().forEach(col -> Assertions.assertEquals(col.getType(), ScalarType.createVarchar(30)));
+    }
+
+    @Test
+    public void testNormalAnalyze() throws Exception {
+        ShowBackupStmt stmt = new ShowBackupStmt("", null);
+
+        AtomicBoolean privilege = new AtomicBoolean(true);
+        new MockUp<AccessControllerManager>() {
+            @Mock
+            public boolean checkDbPriv(ConnectContext ctx, String qualifiedDb, PrivPredicate wanted) {
+                return privilege.get();
+            }
+        };
+
+        stmt.analyze(analyzer);
+        Assertions.assertEquals(stmt.toSql(), "SHOW BACKUP ");
+
+        stmt = new ShowBackupStmt("",
+                new BinaryPredicate(BinaryPredicate.Operator.EQ, new SlotRef(new TableName("a.b.c"), "snapshotname"),
+                        new StringLiteral("FINISHED")));
+        stmt.analyze(analyzer);
+        Assertions.assertEquals(stmt.toSql(), "SHOW BACKUP WHERE `a`.`b`.`c`.`snapshotname` = 'FINISHED'");
+
+        stmt = new ShowBackupStmt("",
+                new LikePredicate(Operator.LIKE, new SlotRef(new TableName("a.b.c"), "snapshotname"),
+                        new StringLiteral("%.b.%")));
+        stmt.analyze(analyzer);
+        Assertions.assertEquals(stmt.toSql(), "SHOW BACKUP WHERE `a`.`b`.`c`.`snapshotname` LIKE '%.b.%'");
+    }
+
+    @Test
+    public void testExceptionAnalyze() {
+        AtomicBoolean privilege = new AtomicBoolean(false);
+        new MockUp<AccessControllerManager>() {
+            @Mock
+            public boolean checkDbPriv(ConnectContext ctx, String qualifiedDb, PrivPredicate wanted) {
+                return privilege.get();
+            }
+        };
+
+        /* when no privilege */ {
+            ShowBackupStmt stmt = new ShowBackupStmt("", null);
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+
+        /* where clause not supported */ {
+            privilege.set(true);
+            ShowBackupStmt stmt = new ShowBackupStmt("test", new BoolLiteral(true));
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+
+        /* where clause op not supported */ {
+            ShowBackupStmt stmt = new ShowBackupStmt("test",
+                    new LikePredicate(Operator.REGEXP, new SlotRef(new TableName(), "STATE_NAME"),
+                            new StringLiteral("%.b.%")));
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+
+        /* where clause op not supported*/ {
+            ShowBackupStmt stmt = new ShowBackupStmt("test", new BinaryPredicate(
+                    BinaryPredicate.Operator.GE, new SlotRef(new TableName(), "STATE_NAME"),
+                    new StringLiteral("blah")));
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+
+        /*where clause op not supported*/ {
+            ShowBackupStmt stmt = new ShowBackupStmt("test", new BinaryPredicate(
+                    BinaryPredicate.Operator.EQ, new SlotRef(new TableName(), "STATE_NAME"),
+                    new StringLiteral("blah")));
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+
+        /* where clause op type(left) not supported */ {
+            ShowBackupStmt stmt = new ShowBackupStmt("test", new BinaryPredicate(
+                    BinaryPredicate.Operator.EQ, new IntLiteral(1), new IntLiteral(1)));
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+
+        /* where clause op type(right) not supported */ {
+            ShowBackupStmt stmt = new ShowBackupStmt("test", new BinaryPredicate(
+                    BinaryPredicate.Operator.EQ, new SlotRef(new TableName(), "snapshotname"), new IntLiteral(1)));
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+
+        /* where clause operand is empty */ {
+            ShowBackupStmt stmt = new ShowBackupStmt("test", new BinaryPredicate(
+                    BinaryPredicate.Operator.EQ, new SlotRef(new TableName(), "snapshotname"), new StringLiteral("")));
+            Assertions.assertThrows(AnalysisException.class, () -> stmt.analyze(analyzer));
+        }
+    }
+}

From bbf58c5aa42d40e66bc6ccc9ed91a4fcb4bdfff7 Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Sun, 31 Dec 2023 16:26:41 +0800
Subject: [PATCH 226/299] [improve](move-memtable) cancel load rapidly when
 stream close wait (#29322)

---
 be/src/vec/sink/load_stream_stub.cpp          |  11 +-
 be/src/vec/sink/load_stream_stub.h            |  54 ++++++++-
 be/src/vec/sink/load_stream_stub_pool.cpp     |  13 ++-
 be/src/vec/sink/load_stream_stub_pool.h       |   4 +-
 be/src/vec/sink/writer/vtablet_writer_v2.cpp  |   5 +-
 .../io/fs/stream_sink_file_writer_test.cpp    |   6 +-
 .../vec/exec/load_stream_stub_pool_test.cpp   |   7 +-
 ...eam_stub_close_wait_fault_injection.groovy | 106 ++++++++++++++++++
 8 files changed, 185 insertions(+), 21 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy

diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 4b557cae93d06e..a5f4f59dda48f3 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -90,19 +90,22 @@ int LoadStreamStub::LoadStreamReplyHandler::on_received_messages(brpc::StreamId
 void LoadStreamStub::LoadStreamReplyHandler::on_closed(brpc::StreamId id) {
     LOG(INFO) << "on_closed, load_id=" << _load_id << ", stream_id=" << id;
     std::lock_guard<bthread::Mutex> lock(_mutex);
+    DBUG_EXECUTE_IF("LoadStreamStub::LoadStreamReplyHandler::on_closed.close_wait", { return; });
     _is_closed.store(true);
     _close_cv.notify_all();
 }
 
-LoadStreamStub::LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use)
-        : _use_cnt(num_use),
+LoadStreamStub::LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use, RuntimeState* state)
+        : _state(state),
+          _use_cnt(num_use),
           _load_id(load_id),
           _src_id(src_id),
           _tablet_schema_for_index(std::make_shared<IndexToTabletSchema>()),
           _enable_unique_mow_for_index(std::make_shared<IndexToEnableMoW>()) {};
 
-LoadStreamStub::LoadStreamStub(LoadStreamStub& stub)
-        : _use_cnt(stub._use_cnt.load()),
+LoadStreamStub::LoadStreamStub(LoadStreamStub& stub, RuntimeState* state)
+        : _state(state),
+          _use_cnt(stub._use_cnt.load()),
           _load_id(stub._load_id),
           _src_id(stub._src_id),
           _tablet_schema_for_index(stub._tablet_schema_for_index),
diff --git a/be/src/vec/sink/load_stream_stub.h b/be/src/vec/sink/load_stream_stub.h
index edbbbda1e64cbf..9c45926d3cc1f5 100644
--- a/be/src/vec/sink/load_stream_stub.h
+++ b/be/src/vec/sink/load_stream_stub.h
@@ -57,6 +57,7 @@
 #include "gutil/ref_counted.h"
 #include "runtime/exec_env.h"
 #include "runtime/memory/mem_tracker.h"
+#include "runtime/runtime_state.h"
 #include "runtime/thread_context.h"
 #include "runtime/types.h"
 #include "util/countdown_latch.h"
@@ -101,11 +102,29 @@ class LoadStreamStub {
         Status close_wait(int64_t timeout_ms) {
             DCHECK(timeout_ms > 0) << "timeout_ms should be greator than 0";
             std::unique_lock<bthread::Mutex> lock(_mutex);
-            if (_is_closed) {
-                return Status::OK();
+            int ret = 0;
+            MonotonicStopWatch watch;
+            watch.start();
+            while (true) {
+                if (_is_closed) {
+                    return Status::OK();
+                }
+                if (_stub->is_cancelled()) {
+                    return _stub->cancelled_state();
+                }
+                if (_stub->runtime_state()->is_cancelled()) {
+                    return Status::Cancelled(_stub->runtime_state()->cancel_reason());
+                }
+                // wait 1s once time.
+                ret = _close_cv.wait_for(lock, 1);
+                if (ret == 0) {
+                    return Status::OK();
+                }
+                if (watch.elapsed_time() / 1000 / 1000 >= timeout_ms) {
+                    return Status::InternalError("stream close wait timeout, result: {}", ret);
+                }
             }
-            int ret = _close_cv.wait_for(lock, timeout_ms * 1000);
-            return ret == 0 ? Status::OK() : Status::Error<true>(ret, "stream close_wait timeout");
+            return Status::OK();
         };
 
         std::vector<int64_t> success_tablets() {
@@ -138,10 +157,28 @@ class LoadStreamStub {
 
 public:
     // construct new stub
-    LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use);
+    LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use, RuntimeState* state);
 
     // copy constructor, shared_ptr members are shared
-    LoadStreamStub(LoadStreamStub& stub);
+    LoadStreamStub(LoadStreamStub& stub, RuntimeState* state);
+
+    void cancel(Status status) {
+        _cancel = true;
+        _cancel_status = status;
+    }
+
+    RuntimeState* runtime_state() const { return _state; }
+
+    bool is_cancelled() const { return _cancel; }
+
+    Status cancelled_state() const { return _cancel_status; }
+
+    std::string cancel_reason() const {
+        if (_state == nullptr) {
+            return "";
+        }
+        return _state->cancel_reason();
+    }
 
 // for mock this class in UT
 #ifdef BE_TEST
@@ -231,6 +268,11 @@ class LoadStreamStub {
     Status _send_with_retry(butil::IOBuf& buf);
 
 protected:
+    RuntimeState* _state = nullptr;
+
+    bool _cancel = false;
+    Status _cancel_status;
+
     std::atomic<bool> _is_init;
     bthread::Mutex _mutex;
 
diff --git a/be/src/vec/sink/load_stream_stub_pool.cpp b/be/src/vec/sink/load_stream_stub_pool.cpp
index c867756d8fb2be..94838acdc08b20 100644
--- a/be/src/vec/sink/load_stream_stub_pool.cpp
+++ b/be/src/vec/sink/load_stream_stub_pool.cpp
@@ -54,13 +54,19 @@ void LoadStreams::release() {
     }
 }
 
+void LoadStreams::cancel(Status status) {
+    for (auto& stream : _streams) {
+        stream->cancel(status);
+    }
+}
+
 LoadStreamStubPool::LoadStreamStubPool() = default;
 
 LoadStreamStubPool::~LoadStreamStubPool() = default;
 
 std::shared_ptr<LoadStreams> LoadStreamStubPool::get_or_create(PUniqueId load_id, int64_t src_id,
                                                                int64_t dst_id, int num_streams,
-                                                               int num_sink) {
+                                                               int num_sink, RuntimeState* state) {
     auto key = std::make_pair(UniqueId(load_id), dst_id);
     std::lock_guard<std::mutex> lock(_mutex);
     std::shared_ptr<LoadStreams> streams = _pool[key];
@@ -69,11 +75,12 @@ std::shared_ptr<LoadStreams> LoadStreamStubPool::get_or_create(PUniqueId load_id
     }
     DCHECK(num_streams > 0) << "stream num should be greater than 0";
     DCHECK(num_sink > 0) << "sink num should be greater than 0";
-    auto [it, _] = _template_stubs.emplace(load_id, new LoadStreamStub {load_id, src_id, num_sink});
+    auto [it, _] =
+            _template_stubs.emplace(load_id, new LoadStreamStub {load_id, src_id, num_sink, state});
     streams = std::make_shared<LoadStreams>(load_id, dst_id, num_sink, this);
     for (int32_t i = 0; i < num_streams; i++) {
         // copy construct, internal tablet schema map will be shared among all stubs
-        streams->streams().emplace_back(new LoadStreamStub {*it->second});
+        streams->streams().emplace_back(new LoadStreamStub {*it->second, state});
     }
     _pool[key] = streams;
     return streams;
diff --git a/be/src/vec/sink/load_stream_stub_pool.h b/be/src/vec/sink/load_stream_stub_pool.h
index e41083825be07e..b548d31a14ff56 100644
--- a/be/src/vec/sink/load_stream_stub_pool.h
+++ b/be/src/vec/sink/load_stream_stub_pool.h
@@ -78,6 +78,8 @@ class LoadStreams {
 
     void release();
 
+    void cancel(Status status);
+
     Streams& streams() { return _streams; }
 
 private:
@@ -95,7 +97,7 @@ class LoadStreamStubPool {
     ~LoadStreamStubPool();
 
     std::shared_ptr<LoadStreams> get_or_create(PUniqueId load_id, int64_t src_id, int64_t dst_id,
-                                               int num_streams, int num_sink);
+                                               int num_streams, int num_sink, RuntimeState* state);
 
     void erase(UniqueId load_id, int64_t dst_id);
 
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 77df9b07f18701..b0a39ac2a76338 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -113,7 +113,7 @@ Status VTabletWriterV2::_incremental_open_streams(
     }
     for (int64_t node_id : new_backends) {
         auto load_streams = ExecEnv::GetInstance()->load_stream_stub_pool()->get_or_create(
-                _load_id, _backend_id, node_id, _stream_per_node, _num_local_sink);
+                _load_id, _backend_id, node_id, _stream_per_node, _num_local_sink, _state);
         RETURN_IF_ERROR(_open_streams_to_backend(node_id, *load_streams));
         _streams_for_node[node_id] = load_streams;
     }
@@ -261,7 +261,7 @@ Status VTabletWriterV2::open(RuntimeState* state, RuntimeProfile* profile) {
 Status VTabletWriterV2::_open_streams(int64_t src_id) {
     for (auto& [dst_id, _] : _tablets_for_node) {
         auto streams = ExecEnv::GetInstance()->load_stream_stub_pool()->get_or_create(
-                _load_id, src_id, dst_id, _stream_per_node, _num_local_sink);
+                _load_id, src_id, dst_id, _stream_per_node, _num_local_sink, _state);
         RETURN_IF_ERROR(_open_streams_to_backend(dst_id, *streams));
         _streams_for_node[dst_id] = streams;
     }
@@ -457,6 +457,7 @@ Status VTabletWriterV2::_cancel(Status status) {
     }
     for (const auto& [_, streams] : _streams_for_node) {
         streams->release();
+        streams->cancel(status);
     }
     return Status::OK();
 }
diff --git a/be/test/io/fs/stream_sink_file_writer_test.cpp b/be/test/io/fs/stream_sink_file_writer_test.cpp
index 7e5bdd350f5b37..a0daa15916e0a2 100644
--- a/be/test/io/fs/stream_sink_file_writer_test.cpp
+++ b/be/test/io/fs/stream_sink_file_writer_test.cpp
@@ -51,7 +51,8 @@ static std::atomic<int64_t> g_num_request;
 class StreamSinkFileWriterTest : public testing::Test {
     class MockStreamStub : public LoadStreamStub {
     public:
-        MockStreamStub(PUniqueId load_id, int64_t src_id) : LoadStreamStub(load_id, src_id, 1) {};
+        MockStreamStub(PUniqueId load_id, int64_t src_id, RuntimeState* state)
+                : LoadStreamStub(load_id, src_id, 1, state) {};
 
         virtual ~MockStreamStub() = default;
 
@@ -85,8 +86,9 @@ class StreamSinkFileWriterTest : public testing::Test {
     virtual void SetUp() {
         _load_id.set_hi(LOAD_ID_HI);
         _load_id.set_lo(LOAD_ID_LO);
+        RuntimeState state;
         for (int src_id = 0; src_id < NUM_STREAM; src_id++) {
-            _streams.emplace_back(new MockStreamStub(_load_id, src_id));
+            _streams.emplace_back(new MockStreamStub(_load_id, src_id, &state));
         }
     }
 
diff --git a/be/test/vec/exec/load_stream_stub_pool_test.cpp b/be/test/vec/exec/load_stream_stub_pool_test.cpp
index 24da3bb6999f36..7faa5fb2e5dd23 100644
--- a/be/test/vec/exec/load_stream_stub_pool_test.cpp
+++ b/be/test/vec/exec/load_stream_stub_pool_test.cpp
@@ -30,13 +30,14 @@ class LoadStreamStubPoolTest : public testing::Test {
 
 TEST_F(LoadStreamStubPoolTest, test) {
     LoadStreamStubPool pool;
+    RuntimeState state;
     int64_t src_id = 100;
     PUniqueId load_id;
     load_id.set_hi(1);
     load_id.set_hi(2);
-    auto streams1 = pool.get_or_create(load_id, src_id, 101, 5, 1);
-    auto streams2 = pool.get_or_create(load_id, src_id, 102, 5, 1);
-    auto streams3 = pool.get_or_create(load_id, src_id, 101, 5, 1);
+    auto streams1 = pool.get_or_create(load_id, src_id, 101, 5, 1, &state);
+    auto streams2 = pool.get_or_create(load_id, src_id, 102, 5, 1, &state);
+    auto streams3 = pool.get_or_create(load_id, src_id, 101, 5, 1, &state);
     EXPECT_EQ(2, pool.size());
     EXPECT_EQ(1, pool.templates_size());
     EXPECT_EQ(streams1, streams3);
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
new file mode 100644
index 00000000000000..85ad5f7bc9d682
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("test_load_stream_stub_close_wait_fault_injection", "nonConcurrent") {
+    sql """ set enable_memtable_on_sink_node=true """
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """
+        CREATE TABLE IF NOT EXISTS `baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+    sql """
+        CREATE TABLE IF NOT EXISTS `test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    GetDebugPoint().clearDebugPointsForAllBEs()
+    streamLoad {
+        table "baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    try {
+        GetDebugPoint().enableDebugPointForAllBEs("LoadStreamStub::LoadStreamReplyHandler::on_closed.close_wait")
+        def thread1 = new Thread({
+            try {
+                def res = sql "insert into test select * from baseall where k1 <= 3"
+                logger.info(res.toString())
+            } catch(Exception e) {
+                logger.info(e.getMessage())
+                assertTrue(e.getMessage().contains("Communications link failure"))
+            } finally {
+                GetDebugPoint().disableDebugPointForAllBEs("LoadStreamStub::LoadStreamReplyHandler::on_closed.close_wait")
+            }
+        })
+        thread1.start()
+
+        sleep(1000)
+
+        def processList = sql "show processlist"
+        logger.info(processList.toString())
+        processList.each { item ->
+            logger.info(item[1].toString())
+            logger.info(item[11].toString())
+            if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
+                def res = sql "kill ${item[1]}"
+                logger.info(res.toString())
+            }
+        }
+    } catch(Exception e) {
+        logger.info(e.getMessage())
+    }
+
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """ set enable_memtable_on_sink_node=false """
+}
\ No newline at end of file

From 9a90c1cd57c7073b4a597beec5d1dcc0e7cf8acb Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Sun, 31 Dec 2023 16:30:27 +0800
Subject: [PATCH 227/299] [improve](move-memtable)add multi replica fault
 injection (#29348)

---
 be/src/io/fs/stream_sink_file_writer.cpp      |  28 +++--
 .../test_multi_replica_fault_injection.groovy | 100 ++++++++++++++++++
 2 files changed, 120 insertions(+), 8 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy

diff --git a/be/src/io/fs/stream_sink_file_writer.cpp b/be/src/io/fs/stream_sink_file_writer.cpp
index 74f34b44e6d560..b7cd4ed0bbfed3 100644
--- a/be/src/io/fs/stream_sink_file_writer.cpp
+++ b/be/src/io/fs/stream_sink_file_writer.cpp
@@ -53,25 +53,37 @@ Status StreamSinkFileWriter::appendv(const Slice* data, size_t data_cnt) {
     std::span<const Slice> slices {data, data_cnt};
     size_t stream_index = 0;
     bool ok = false;
+    bool skip_stream = false;
+    Status st;
     for (auto& stream : _streams) {
-        auto st = stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id,
-                                      _bytes_appended, slices);
+        DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", {
+            if (stream_index >= 2) {
+                skip_stream = true;
+            }
+        });
+        DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", {
+            if (stream_index >= 1) {
+                skip_stream = true;
+            }
+        });
+        if (!skip_stream) {
+            st = stream->append_data(_partition_id, _index_id, _tablet_id, _segment_id,
+                                     _bytes_appended, slices);
+        }
         DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", {
             if (stream_index >= 2) {
                 st = Status::InternalError("stream sink file writer append data failed");
-            } else {
-                stream_index++;
             }
+            stream_index++;
+            skip_stream = false;
         });
-
         DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", {
             if (stream_index >= 1) {
                 st = Status::InternalError("stream sink file writer append data failed");
-            } else {
-                stream_index++;
             }
+            stream_index++;
+            skip_stream = false;
         });
-
         DBUG_EXECUTE_IF("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", {
             st = Status::InternalError("stream sink file writer append data failed");
         });
diff --git a/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
new file mode 100644
index 00000000000000..aff1318a52fec3
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_multi_replica_fault_injection.groovy
@@ -0,0 +1,100 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("test_multi_replica_fault_injection", "nonConcurrent") {
+    def res = sql"show backends;"
+    logger.info(res.toString())
+    def beNums = 0;
+    res.each { item ->
+        beNums++;
+        logger.info(item.toString())
+    }
+    if (beNums == 3){
+        sql """ set enable_memtable_on_sink_node=true """
+        sql """
+            CREATE TABLE IF NOT EXISTS `baseall` (
+                `k0` boolean null comment "",
+                `k1` tinyint(4) null comment "",
+                `k2` smallint(6) null comment "",
+                `k3` int(11) null comment "",
+                `k4` bigint(20) null comment "",
+                `k5` decimal(9, 3) null comment "",
+                `k6` char(5) null comment "",
+                `k10` date null comment "",
+                `k11` datetime null comment "",
+                `k7` varchar(20) null comment "",
+                `k8` double max null comment "",
+                `k9` float sum null comment "",
+                `k12` string replace null comment "",
+                `k13` largeint(40) replace null comment ""
+            ) engine=olap
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "3")
+            """
+        sql """
+            CREATE TABLE IF NOT EXISTS `test` (
+                `k0` boolean null comment "",
+                `k1` tinyint(4) null comment "",
+                `k2` smallint(6) null comment "",
+                `k3` int(11) null comment "",
+                `k4` bigint(20) null comment "",
+                `k5` decimal(9, 3) null comment "",
+                `k6` char(5) null comment "",
+                `k10` date null comment "",
+                `k11` datetime null comment "",
+                `k7` varchar(20) null comment "",
+                `k8` double max null comment "",
+                `k9` float sum null comment "",
+                `k12` string replace_if_not_null null comment "",
+                `k13` largeint(40) replace null comment ""
+            ) engine=olap
+            DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "3")
+            """
+
+        GetDebugPoint().clearDebugPointsForAllBEs()
+        streamLoad {
+            table "baseall"
+            db "regression_test_fault_injection_p0"
+            set 'column_separator', ','
+            file "baseall.txt"
+        }
+
+        def load_with_injection = { injection, error_msg->
+            try {
+                GetDebugPoint().enableDebugPointForAllBEs(injection)
+                sql "insert into test select * from baseall where k1 <= 3"
+            } catch(Exception e) {
+                logger.info(e.getMessage())
+                assertTrue(e.getMessage().contains(error_msg))
+            } finally {
+                GetDebugPoint().disableDebugPointForAllBEs(injection)
+            }
+        }
+
+        // StreamSinkFileWriter appendv write segment failed one replica
+        // success
+        load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", "sucess")
+        // StreamSinkFileWriter appendv write segment failed two replica
+        load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", "replica num 1 < load required replica num 2")
+        // StreamSinkFileWriter appendv write segment failed all replica
+        load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "failed to write any replicas")
+
+        sql """ set enable_memtable_on_sink_node=false """
+    }
+}
\ No newline at end of file

From cc40cdf2d459d9080b74053f2284255197a78aba Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Sun, 31 Dec 2023 16:47:46 +0800
Subject: [PATCH 228/299] [improve](load) don't limit add_segment running count
 in load stream (#29332)

Co-authored-by: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com>
---
 be/src/runtime/load_stream.cpp | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index 8049f41893c9f8..6003f0f4d94234 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -206,25 +206,6 @@ Status TabletStream::add_segment(const PStreamHeader& header, butil::IOBuf* data
         }
     };
     auto& flush_token = _flush_tokens[new_segid % _flush_tokens.size()];
-    auto load_stream_flush_token_max_tasks = config::load_stream_flush_token_max_tasks;
-    auto load_stream_max_wait_flush_token_time_ms =
-            config::load_stream_max_wait_flush_token_time_ms;
-    DBUG_EXECUTE_IF("TabletStream.add_segment.long_wait", {
-        load_stream_flush_token_max_tasks = 0;
-        load_stream_max_wait_flush_token_time_ms = 1000;
-    });
-    MonotonicStopWatch timer;
-    timer.start();
-    while (flush_token->num_tasks() >= load_stream_flush_token_max_tasks) {
-        if (timer.elapsed_time() / 1000 / 1000 >= load_stream_max_wait_flush_token_time_ms) {
-            return Status::Error<true>(
-                    "wait flush token back pressure time is more than "
-                    "load_stream_max_wait_flush_token_time {}",
-                    load_stream_max_wait_flush_token_time_ms);
-        }
-        bthread_usleep(10 * 1000); // 10ms
-    }
-    timer.stop();
     return flush_token->submit_func(add_segment_func);
 }
 

From 3c6c652997874e77ae3fbb227d72b8e9697305f5 Mon Sep 17 00:00:00 2001
From: Luwei <814383175@qq.com>
Date: Sun, 31 Dec 2023 17:02:15 +0800
Subject: [PATCH 229/299]  [Fix](schema change) disable convert light schema
 change (#28205) (#29300)

---
 fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 4e053524552043..7b9859ae450e35 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2220,7 +2220,7 @@ public class Config extends ConfigBase {
             "暂时性配置项，开启后会自动将所有的olap表修改为可light schema change",
             "temporary config filed, will make all olap tables enable light schema change"
     })
-    public static boolean enable_convert_light_weight_schema_change = true;
+    public static boolean enable_convert_light_weight_schema_change = false;
 
     @ConfField(mutable = true, masterOnly = false, description = {
             "查询information_schema.metadata_name_ids表时,获取一个数据库中所有表用的时间",

From b07ce175de45d96321859e2f4fb0719396df692c Mon Sep 17 00:00:00 2001
From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com>
Date: Sun, 31 Dec 2023 22:25:25 +0800
Subject: [PATCH 230/299] =?UTF-8?q?Revert=20"[fix](move-memtable)=20only?=
 =?UTF-8?q?=20check=20missing=20tablets=20when=20commit=20i=E2=80=A6=20(#2?=
 =?UTF-8?q?9352)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Revert "[fix](move-memtable) only check missing tablets when commit info is not empty (#29326)"

This reverts commit 4634c723d7fb86f434060c0e6fa6168d92630cb9.

* Revert "[fix](move-memtable) check missing tablets before commit (#29223)"

This reverts commit fb0ed8c253f5241c23c71fb88f031604b0bf5a1f.
---
 be/src/vec/sink/writer/vtablet_writer_v2.cpp | 23 --------------------
 be/src/vec/sink/writer/vtablet_writer_v2.h   |  2 +-
 2 files changed, 1 insertion(+), 24 deletions(-)

diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index b0a39ac2a76338..2550fb172c57c1 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -549,32 +549,10 @@ Status VTabletWriterV2::close(Status exec_status) {
                     TTabletCommitInfo commit_info;
                     commit_info.tabletId = tablet_id;
                     commit_info.backendId = node_id;
-                    _missing_tablets.erase(tablet_id);
                     tablet_commit_infos.emplace_back(std::move(commit_info));
                 }
             }
         }
-        if (!tablet_commit_infos.empty() && !_missing_tablets.empty()) {
-            std::stringstream ss;
-            ss << "pre-commit check failed, missing " << _missing_tablets.size() << " tablets:";
-            int print_limit = 3;
-            for (auto tablet_id : _missing_tablets | std::ranges::views::take(print_limit)) {
-                ss << " (tablet_id=" << tablet_id;
-                auto backends = _location->find_tablet(tablet_id)->node_ids;
-                ss << ", backend_id=[";
-                bool first = true;
-                for (auto& backend_id : backends) {
-                    (first ? ss : ss << ',') << backend_id;
-                    first = false;
-                }
-                ss << "])";
-            }
-            if (_missing_tablets.size() > print_limit) {
-                ss << ", ...";
-            }
-            LOG(INFO) << ss.str() << ", load_id=" << print_id(_load_id);
-            return Status::InternalError(ss.str());
-        }
         _state->tablet_commit_infos().insert(_state->tablet_commit_infos().end(),
                                              std::make_move_iterator(tablet_commit_infos.begin()),
                                              std::make_move_iterator(tablet_commit_infos.end()));
@@ -605,7 +583,6 @@ Status VTabletWriterV2::_close_load(const Streams& streams) {
     for (auto [tablet_id, tablet] : _tablets_for_node[node_id]) {
         if (_tablet_finder->partition_ids().contains(tablet.partition_id())) {
             tablets_to_commit.push_back(tablet);
-            _missing_tablets.insert(tablet_id);
         }
     }
     for (const auto& stream : streams) {
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.h b/be/src/vec/sink/writer/vtablet_writer_v2.h
index 37558974197548..e99bbd92696cfb 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.h
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.h
@@ -211,7 +211,7 @@ class VTabletWriterV2 final : public AsyncResultWriter {
     RuntimeState* _state = nullptr;     // not owned, set when open
     RuntimeProfile* _profile = nullptr; // not owned, set when open
 
-    std::unordered_set<int64_t> _missing_tablets;
+    std::unordered_set<int64_t> _opened_partitions;
 
     std::unordered_map<int64_t, std::unordered_map<int64_t, PTabletID>> _tablets_for_node;
     std::unordered_map<int64_t, std::vector<PTabletID>> _indexes_from_node;

From 5985d216f33b1aa516d2a319165c229198e528fd Mon Sep 17 00:00:00 2001
From: zhangdong <493738387@qq.com>
Date: Sun, 31 Dec 2023 23:10:30 +0800
Subject: [PATCH 231/299] [feature](mtmv)support cancel mtmv task command
 (#29252)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- `CANCEL MATERIALIZED VIEW TASK taskId on mvName`
- CANCEL MATERIALIZED VIEW TASK, tasks("type"="mv") and jobs("type"="mv") support check auth use priv of mv
- tasks and jobs add column mvName and mvDbName，you can use `select * from tasks("type"="mv") where MvName="xxx"` get all tasks of mv
- fix `desc mv all` error
- fix p0 The task sequence is incorrect
---
 .../org/apache/doris/nereids/DorisParser.g4   |  1 +
 .../apache/doris/analysis/DescribeStmt.java   |  4 +-
 .../doris/job/extensions/mtmv/MTMVJob.java    | 43 ++++++++-
 .../doris/job/extensions/mtmv/MTMVTask.java   | 38 +++++++-
 .../job/extensions/mtmv/MTMVTaskContext.java  |  9 ++
 .../apache/doris/mtmv/MTMVHookService.java    | 10 ++
 .../org/apache/doris/mtmv/MTMVJobManager.java |  8 +-
 .../doris/mtmv/MTMVRelationManager.java       |  6 ++
 .../org/apache/doris/mtmv/MTMVService.java    |  9 ++
 .../nereids/parser/LogicalPlanBuilder.java    | 10 ++
 .../doris/nereids/trees/plans/PlanType.java   |  1 +
 .../plans/commands/CancelMTMVTaskCommand.java | 50 ++++++++++
 .../commands/info/CancelMTMVTaskInfo.java     | 91 +++++++++++++++++++
 .../plans/commands/info/PauseMTMVInfo.java    |  7 ++
 .../plans/commands/info/RefreshMTMVInfo.java  |  9 ++
 .../plans/commands/info/ResumeMTMVInfo.java   |  7 ++
 .../trees/plans/visitor/CommandVisitor.java   |  5 +
 .../JobsTableValuedFunction.java              | 10 +-
 .../tablefunction/MetadataGenerator.java      | 17 ++++
 .../TasksTableValuedFunction.java             | 10 +-
 .../doris/regression/suite/Suite.groovy       |  2 +-
 .../suites/mtmv_p0/test_build_mtmv.groovy     |  4 +
 .../suites/mtmv_p0/test_task_mtmv.groovy      | 62 +++++++++++++
 23 files changed, 396 insertions(+), 17 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CancelMTMVTaskInfo.java
 create mode 100644 regression-test/suites/mtmv_p0/test_task_mtmv.groovy

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 0aab15c884a925..880986a6cec9f9 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -96,6 +96,7 @@ statement
     | DROP MATERIALIZED VIEW (IF EXISTS)? mvName=multipartIdentifier      #dropMTMV
     | PAUSE MATERIALIZED VIEW JOB ON mvName=multipartIdentifier      #pauseMTMV
     | RESUME MATERIALIZED VIEW JOB ON mvName=multipartIdentifier      #resumeMTMV
+    | CANCEL MATERIALIZED VIEW TASK taskId=INTEGER_VALUE ON mvName=multipartIdentifier      #cancelMTMVTask
     | ALTER TABLE table=relation
         ADD CONSTRAINT constraintName=errorCapturingIdentifier
         constraint                                                        #addConstraint
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
index cc5e8adb260161..40c576b1c4f361 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
@@ -187,7 +187,7 @@ public void analyze(Analyzer analyzer) throws UserException {
                 // show base table schema only
                 String procString = "/catalogs/" + catalog.getId() + "/" + db.getId() + "/" + table.getId() + "/"
                         + TableProcDir.INDEX_SCHEMA + "/";
-                if (table.getType() == TableType.OLAP) {
+                if (table instanceof OlapTable) {
                     procString += ((OlapTable) table).getBaseIndexId();
                 } else {
                     if (partitionNames != null) {
@@ -212,7 +212,7 @@ public void analyze(Analyzer analyzer) throws UserException {
                 }
             } else {
                 Util.prohibitExternalCatalog(dbTableName.getCtl(), this.getClass().getSimpleName() + " ALL");
-                if (table.getType() == TableType.OLAP) {
+                if (table instanceof OlapTable) {
                     isOlapTable = true;
                     OlapTable olapTable = (OlapTable) table;
                     Set<String> bfColumns = olapTable.getCopiedBfColumns();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java
index 669a2806a543d9..678155170bd541 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVJob.java
@@ -17,6 +17,7 @@
 
 package org.apache.doris.job.extensions.mtmv;
 
+import org.apache.doris.analysis.UserIdentity;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.Env;
@@ -25,12 +26,15 @@
 import org.apache.doris.catalog.TableIf.TableType;
 import org.apache.doris.common.DdlException;
 import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.common.UserException;
 import org.apache.doris.common.io.Text;
 import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.job.base.AbstractJob;
 import org.apache.doris.job.common.JobType;
 import org.apache.doris.job.common.TaskType;
 import org.apache.doris.job.extensions.mtmv.MTMVTask.MTMVTaskTriggerMode;
+import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.persist.gson.GsonUtils;
 import org.apache.doris.qe.ShowResultSetMetaData;
 import org.apache.doris.thrift.TCell;
@@ -67,11 +71,14 @@ public class MTMVJob extends AbstractJob<MTMVTask, MTMVTaskContext> {
     public static final ImmutableList<Column> SCHEMA = ImmutableList.of(
             new Column("Id", ScalarType.createStringType()),
             new Column("Name", ScalarType.createStringType()),
+            new Column("MvId", ScalarType.createStringType()),
+            new Column("MvName", ScalarType.createStringType()),
+            new Column("MvDatabaseId", ScalarType.createStringType()),
+            new Column("MvDatabaseName", ScalarType.createStringType()),
             new Column("ExecuteType", ScalarType.createStringType()),
             new Column("RecurringStrategy", ScalarType.createStringType()),
             new Column("Status", ScalarType.createStringType()),
-            new Column("CreateTime", ScalarType.createStringType()),
-            new Column("Comment", ScalarType.createStringType()));
+            new Column("CreateTime", ScalarType.createStringType()));
 
     public static final ImmutableMap<String, Integer> COLUMN_TO_INDEX;
 
@@ -118,6 +125,7 @@ protected void checkJobParamsInternal() {
 
     @Override
     public List<MTMVTask> createTasks(TaskType taskType, MTMVTaskContext taskContext) {
+        LOG.info("begin create mtmv task, jobId: {}, taskContext: {}", super.getJobId(), taskContext);
         if (taskContext == null) {
             taskContext = new MTMVTaskContext(MTMVTaskTriggerMode.SYSTEM);
         }
@@ -126,6 +134,7 @@ public List<MTMVTask> createTasks(TaskType taskType, MTMVTaskContext taskContext
         ArrayList<MTMVTask> tasks = new ArrayList<>();
         tasks.add(task);
         super.initTasks(tasks, taskType);
+        LOG.info("finish create mtmv task, task: {}", task);
         return tasks;
     }
 
@@ -144,6 +153,8 @@ public boolean isReadyForScheduling(MTMVTaskContext taskContext) {
         List<MTMVTask> runningTasks = getRunningTasks();
         for (MTMVTask task : runningTasks) {
             if (task.getTaskContext() == null || task.getTaskContext().getTriggerMode() == MTMVTaskTriggerMode.SYSTEM) {
+                LOG.warn("isReadyForScheduling return false, because current taskContext is null, exist task: {}",
+                        task);
                 return false;
             }
         }
@@ -195,14 +206,40 @@ public TRow getTvfInfo() {
         TRow trow = new TRow();
         trow.addToColumnValue(new TCell().setStringVal(String.valueOf(super.getJobId())));
         trow.addToColumnValue(new TCell().setStringVal(super.getJobName()));
+        String dbName = "";
+        String mvName = "";
+        try {
+            MTMV mtmv = getMTMV();
+            dbName = mtmv.getQualifiedDbName();
+            mvName = mtmv.getName();
+        } catch (UserException e) {
+            LOG.warn("can not find mv", e);
+        }
+        trow.addToColumnValue(new TCell().setStringVal(String.valueOf(mtmvId)));
+        trow.addToColumnValue(new TCell().setStringVal(mvName));
+        trow.addToColumnValue(new TCell().setStringVal(String.valueOf(dbId)));
+        trow.addToColumnValue(new TCell().setStringVal(dbName));
         trow.addToColumnValue(new TCell().setStringVal(super.getJobConfig().getExecuteType().name()));
         trow.addToColumnValue(new TCell().setStringVal(super.getJobConfig().convertRecurringStrategyToString()));
         trow.addToColumnValue(new TCell().setStringVal(super.getJobStatus().name()));
         trow.addToColumnValue(new TCell().setStringVal(TimeUtils.longToTimeString(super.getCreateTimeMs())));
-        trow.addToColumnValue(new TCell().setStringVal(super.getComment()));
         return trow;
     }
 
+    public boolean hasPriv(UserIdentity userIdentity, PrivPredicate wanted) {
+        MTMV mtmv;
+        try {
+            mtmv = getMTMV();
+        } catch (UserException e) {
+            LOG.warn("can not find mv", e);
+            return false;
+        }
+        return Env.getCurrentEnv().getAccessManager()
+                .checkTblPriv(userIdentity, InternalCatalog.INTERNAL_CATALOG_NAME,
+                        mtmv.getQualifiedDbName(), mtmv.getName(),
+                        wanted);
+    }
+
     private MTMV getMTMV() throws DdlException, MetaNotFoundException {
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId);
         return (MTMV) db.getTableOrMetaException(mtmvId, TableType.MATERIALIZED_VIEW);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index 7c11f0549ad859..cbfea9e7139f05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -25,7 +25,9 @@
 import org.apache.doris.catalog.ScalarType;
 import org.apache.doris.catalog.TableIf.TableType;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
 import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.MetaNotFoundException;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.TimeUtils;
 import org.apache.doris.job.exception.JobException;
@@ -73,7 +75,9 @@ public class MTMVTask extends AbstractTask {
             new Column("JobId", ScalarType.createStringType()),
             new Column("JobName", ScalarType.createStringType()),
             new Column("MvId", ScalarType.createStringType()),
+            new Column("MvName", ScalarType.createStringType()),
             new Column("MvDatabaseId", ScalarType.createStringType()),
+            new Column("MvDatabaseName", ScalarType.createStringType()),
             new Column("Status", ScalarType.createStringType()),
             new Column("ErrorMsg", ScalarType.createStringType()),
             new Column("CreateTime", ScalarType.createStringType()),
@@ -213,8 +217,7 @@ public void before() throws JobException {
         LOG.info("mtmv task before, taskId: {}", super.getTaskId());
         super.before();
         try {
-            Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId);
-            mtmv = (MTMV) db.getTableOrMetaException(mtmvId, TableType.MATERIALIZED_VIEW);
+            mtmv = getMTMV();
             if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) {
                 OlapTable relatedTable = (OlapTable) MTMVUtil.getTable(mtmv.getMvPartitionInfo().getRelatedTable());
                 MTMVUtil.alignMvPartition(mtmv, relatedTable);
@@ -225,6 +228,11 @@ public void before() throws JobException {
         }
     }
 
+    private MTMV getMTMV() throws DdlException, MetaNotFoundException {
+        Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId);
+        return (MTMV) db.getTableOrMetaException(mtmvId, TableType.MATERIALIZED_VIEW);
+    }
+
     @Override
     public void runTask() throws JobException {
         LOG.info("mtmv task runTask, taskId: {}", super.getTaskId());
@@ -246,8 +254,19 @@ public TRow getTvfInfo() {
         trow.addToColumnValue(new TCell().setStringVal(String.valueOf(super.getTaskId())));
         trow.addToColumnValue(new TCell().setStringVal(String.valueOf(super.getJobId())));
         trow.addToColumnValue(new TCell().setStringVal(super.getJobName()));
+        String dbName = "";
+        String mvName = "";
+        try {
+            MTMV mtmv = getMTMV();
+            dbName = mtmv.getQualifiedDbName();
+            mvName = mtmv.getName();
+        } catch (UserException e) {
+            LOG.warn("can not find mv", e);
+        }
         trow.addToColumnValue(new TCell().setStringVal(String.valueOf(mtmvId)));
+        trow.addToColumnValue(new TCell().setStringVal(mvName));
         trow.addToColumnValue(new TCell().setStringVal(String.valueOf(dbId)));
+        trow.addToColumnValue(new TCell().setStringVal(dbName));
         trow.addToColumnValue(new TCell()
                 .setStringVal(super.getStatus() == null ? FeConstants.null_string : super.getStatus().toString()));
         trow.addToColumnValue(new TCell().setStringVal(super.getErrMsg()));
@@ -355,4 +374,19 @@ private List<Long> calculateNeedRefreshPartitions() throws AnalysisException {
     public MTMVTaskContext getTaskContext() {
         return taskContext;
     }
+
+    @Override
+    public String toString() {
+        return "MTMVTask{"
+                + "dbId=" + dbId
+                + ", mtmvId=" + mtmvId
+                + ", taskContext=" + taskContext
+                + ", needRefreshPartitions=" + needRefreshPartitions
+                + ", completedPartitions=" + completedPartitions
+                + ", refreshMode=" + refreshMode
+                + ", mtmv=" + mtmv
+                + ", relation=" + relation
+                + ", executor=" + executor
+                + "} " + super.toString();
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTaskContext.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTaskContext.java
index adb6fd5ef7178a..1c53f5dd301ddb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTaskContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTaskContext.java
@@ -55,4 +55,13 @@ public MTMVTaskTriggerMode getTriggerMode() {
     public boolean isComplete() {
         return isComplete;
     }
+
+    @Override
+    public String toString() {
+        return "MTMVTaskContext{"
+                + "triggerMode=" + triggerMode
+                + ", partitions=" + partitions
+                + ", isComplete=" + isComplete
+                + '}';
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java
index 41bc506f5c0d62..d9ab9984581227 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVHookService.java
@@ -23,6 +23,7 @@
 import org.apache.doris.common.MetaNotFoundException;
 import org.apache.doris.job.exception.JobException;
 import org.apache.doris.job.extensions.mtmv.MTMVTask;
+import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.ResumeMTMVInfo;
@@ -107,13 +108,22 @@ public interface MTMVHookService {
 
     /**
      * Triggered when pause mtmv
+     *
      * @param info
      */
     void pauseMTMV(PauseMTMVInfo info) throws MetaNotFoundException, DdlException, JobException;
 
     /**
      * Triggered when resume mtmv
+     *
      * @param info
      */
     void resumeMTMV(ResumeMTMVInfo info) throws MetaNotFoundException, DdlException, JobException;
+
+    /**
+     * cancel mtmv task
+     *
+     * @param info
+     */
+    void cancelMTMVTask(CancelMTMVTaskInfo info) throws DdlException, MetaNotFoundException, JobException;
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
index 8cf225c59c9220..f53a7b60868ace 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVJobManager.java
@@ -37,6 +37,7 @@
 import org.apache.doris.job.extensions.mtmv.MTMVTaskContext;
 import org.apache.doris.mtmv.MTMVRefreshEnum.BuildMode;
 import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshTrigger;
+import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.ResumeMTMVInfo;
@@ -66,7 +67,6 @@ public void createMTMV(MTMV mtmv) throws DdlException {
         MTMVJob job = new MTMVJob(mtmv.getDatabase().getId(), mtmv.getId());
         job.setJobId(Env.getCurrentEnv().getNextId());
         job.setJobName(mtmv.getJobInfo().getJobName());
-        job.setComment(mtmv.getName());
         job.setCreateUser(ConnectContext.get().getCurrentUserIdentity());
         job.setJobStatus(JobStatus.RUNNING);
         job.setJobConfig(getJobConfig(mtmv));
@@ -204,6 +204,12 @@ public void resumeMTMV(ResumeMTMVInfo info) throws MetaNotFoundException, DdlExc
         Env.getCurrentEnv().getJobManager().alterJobStatus(job.getJobId(), JobStatus.RUNNING);
     }
 
+    @Override
+    public void cancelMTMVTask(CancelMTMVTaskInfo info) throws DdlException, MetaNotFoundException, JobException {
+        MTMVJob job = getJobByTableNameInfo(info.getMvName());
+        job.cancelTaskById(info.getTaskId());
+    }
+
     private MTMVJob getJobByTableNameInfo(TableNameInfo info) throws DdlException, MetaNotFoundException {
         Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(info.getDb());
         MTMV mtmv = (MTMV) db.getTableOrMetaException(info.getTbl(), TableType.MATERIALIZED_VIEW);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
index 7be43771e442f8..35bf777acb5625 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelationManager.java
@@ -27,6 +27,7 @@
 import org.apache.doris.job.exception.JobException;
 import org.apache.doris.job.extensions.mtmv.MTMVTask;
 import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState;
+import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.ResumeMTMVInfo;
@@ -200,6 +201,11 @@ public void resumeMTMV(ResumeMTMVInfo info) throws MetaNotFoundException, DdlExc
 
     }
 
+    @Override
+    public void cancelMTMVTask(CancelMTMVTaskInfo info) {
+
+    }
+
     private void processBaseTableChange(Table table, String msgPrefix) {
         BaseTableInfo baseTableInfo = new BaseTableInfo(table);
         Set<BaseTableInfo> mtmvsByBaseTable = getMtmvsByBaseTable(baseTableInfo);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
index 3ab05c92a463aa..5b7c18882373ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVService.java
@@ -26,6 +26,7 @@
 import org.apache.doris.job.exception.JobException;
 import org.apache.doris.job.extensions.mtmv.MTMVTask;
 import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
+import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.RefreshMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.ResumeMTMVInfo;
@@ -159,4 +160,12 @@ public void resumeMTMV(ResumeMTMVInfo info) throws MetaNotFoundException, DdlExc
             mtmvHookService.resumeMTMV(info);
         }
     }
+
+    public void cancelMTMVTask(CancelMTMVTaskInfo info) throws MetaNotFoundException, DdlException, JobException {
+        Objects.requireNonNull(info);
+        LOG.info("cancelMTMVTask, CancelMTMVTaskInfo: {}", info);
+        for (MTMVHookService mtmvHookService : hooks.values()) {
+            mtmvHookService.cancelMTMVTask(info);
+        }
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 71890516628c38..745ae179469b90 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -55,6 +55,7 @@
 import org.apache.doris.nereids.DorisParser.BracketRelationHintContext;
 import org.apache.doris.nereids.DorisParser.BuildModeContext;
 import org.apache.doris.nereids.DorisParser.CallProcedureContext;
+import org.apache.doris.nereids.DorisParser.CancelMTMVTaskContext;
 import org.apache.doris.nereids.DorisParser.CollateContext;
 import org.apache.doris.nereids.DorisParser.ColumnDefContext;
 import org.apache.doris.nereids.DorisParser.ColumnDefsContext;
@@ -326,6 +327,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.BatchInsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.Constraint;
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
@@ -351,6 +353,7 @@
 import org.apache.doris.nereids.trees.plans.commands.info.AlterMTMVRenameInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.BulkLoadDataDesc;
 import org.apache.doris.nereids.trees.plans.commands.info.BulkStorageDesc;
+import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo;
 import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo;
@@ -712,6 +715,13 @@ public ResumeMTMVCommand visitResumeMTMV(ResumeMTMVContext ctx) {
         return new ResumeMTMVCommand(new ResumeMTMVInfo(new TableNameInfo(nameParts)));
     }
 
+    @Override
+    public CancelMTMVTaskCommand visitCancelMTMVTask(CancelMTMVTaskContext ctx) {
+        List<String> nameParts = visitMultipartIdentifier(ctx.mvName);
+        long taskId = Long.parseLong(ctx.taskId.getText());
+        return new CancelMTMVTaskCommand(new CancelMTMVTaskInfo(new TableNameInfo(nameParts), taskId));
+    }
+
     @Override
     public AlterMTMVCommand visitAlterMTMV(AlterMTMVContext ctx) {
         List<String> nameParts = visitMultipartIdentifier(ctx.mvName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 3db4a43834507d..3fe662b7baff6c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -135,5 +135,6 @@ public enum PlanType {
     DROP_MTMV_COMMAND,
     PAUSE_MTMV_COMMAND,
     RESUME_MTMV_COMMAND,
+    CANCEL_MTMV_TASK_COMMAND,
     CALL_COMMAND
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java
new file mode 100644
index 00000000000000..e5c075da7e4433
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CancelMTMVTaskCommand.java
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.CancelMTMVTaskInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import java.util.Objects;
+
+/**
+ * cancel mtmv task
+ */
+public class CancelMTMVTaskCommand extends Command implements ForwardWithSync, NotAllowFallback {
+    private final CancelMTMVTaskInfo cancelMTMVTaskInfo;
+
+    public CancelMTMVTaskCommand(CancelMTMVTaskInfo cancelMTMVTaskInfo) {
+        super(PlanType.CANCEL_MTMV_TASK_COMMAND);
+        this.cancelMTMVTaskInfo = Objects.requireNonNull(cancelMTMVTaskInfo, "require cancelMTMVTaskInfo object");
+    }
+
+    @Override
+    public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
+        cancelMTMVTaskInfo.analyze(ctx);
+        Env.getCurrentEnv().getMtmvService().cancelMTMVTask(cancelMTMVTaskInfo);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitCancelMTMVTaskCommand(this, context);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CancelMTMVTaskInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CancelMTMVTaskInfo.java
new file mode 100644
index 00000000000000..257769fe359f6b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CancelMTMVTaskInfo.java
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.info;
+
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.exceptions.AnalysisException;
+import org.apache.doris.qe.ConnectContext;
+
+import java.util.Objects;
+
+/**
+ * cancel mtmv task info
+ */
+public class CancelMTMVTaskInfo {
+    private final TableNameInfo mvName;
+    private final long taskId;
+
+    public CancelMTMVTaskInfo(TableNameInfo mvName, long taskId) {
+        this.mvName = Objects.requireNonNull(mvName, "require mvName object");
+        this.taskId = taskId;
+    }
+
+    /**
+     * analyze pause info
+     *
+     * @param ctx ConnectContext
+     */
+    public void analyze(ConnectContext ctx) {
+        mvName.analyze(ctx);
+        if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), mvName.getDb(),
+                mvName.getTbl(), PrivPredicate.CREATE)) {
+            String message = ErrorCode.ERR_TABLEACCESS_DENIED_ERROR.formatErrorMsg("CREATE",
+                    ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
+                    mvName.getDb() + ": " + mvName.getTbl());
+            throw new AnalysisException(message);
+        }
+        try {
+            Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(mvName.getDb());
+            db.getTableOrMetaException(mvName.getTbl(), TableType.MATERIALIZED_VIEW);
+        } catch (MetaNotFoundException | DdlException e) {
+            throw new AnalysisException(e.getMessage());
+        }
+    }
+
+    /**
+     * getMvName
+     *
+     * @return TableNameInfo
+     */
+    public TableNameInfo getMvName() {
+        return mvName;
+    }
+
+    /**
+     * get taskId
+     *
+     * @return taskId
+     */
+    public long getTaskId() {
+        return taskId;
+    }
+
+    @Override
+    public String toString() {
+        return "CancelMTMVTaskInfo{"
+                + "mvName=" + mvName
+                + ", taskId=" + taskId
+                + '}';
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PauseMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PauseMTMVInfo.java
index 15bbcff6f72f99..c744e6667096fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PauseMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PauseMTMVInfo.java
@@ -69,4 +69,11 @@ public void analyze(ConnectContext ctx) {
     public TableNameInfo getMvName() {
         return mvName;
     }
+
+    @Override
+    public String toString() {
+        return "PauseMTMVInfo{"
+                + "mvName=" + mvName
+                + '}';
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java
index e6a4368a850c83..315b30ae87d141 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RefreshMTMVInfo.java
@@ -100,4 +100,13 @@ public List<String> getPartitions() {
     public boolean isComplete() {
         return isComplete;
     }
+
+    @Override
+    public String toString() {
+        return "RefreshMTMVInfo{"
+                + "mvName=" + mvName
+                + ", partitions=" + partitions
+                + ", isComplete=" + isComplete
+                + '}';
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ResumeMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ResumeMTMVInfo.java
index a7f23bedfc7500..541d5c878e3576 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ResumeMTMVInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/ResumeMTMVInfo.java
@@ -69,4 +69,11 @@ public void analyze(ConnectContext ctx) {
     public TableNameInfo getMvName() {
         return mvName;
     }
+
+    @Override
+    public String toString() {
+        return "ResumeMTMVInfo{"
+                + "mvName=" + mvName
+                + '}';
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index a48a8aaf981f28..297e80114de2cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -21,6 +21,7 @@
 import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.BatchInsertIntoTableCommand;
 import org.apache.doris.nereids.trees.plans.commands.CallCommand;
+import org.apache.doris.nereids.trees.plans.commands.CancelMTMVTaskCommand;
 import org.apache.doris.nereids.trees.plans.commands.Command;
 import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand;
 import org.apache.doris.nereids.trees.plans.commands.CreatePolicyCommand;
@@ -123,6 +124,10 @@ default R visitResumeMTMVCommand(ResumeMTMVCommand resumeMTMVCommand, C context)
         return visitCommand(resumeMTMVCommand, context);
     }
 
+    default R visitCancelMTMVTaskCommand(CancelMTMVTaskCommand cancelMTMVTaskCommand, C context) {
+        return visitCommand(cancelMTMVTaskCommand, context);
+    }
+
     default R visitCallCommand(CallCommand callCommand, C context) {
         return visitCommand(callCommand, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JobsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JobsTableValuedFunction.java
index 2a3a698eb9dbbc..260a483d184a58 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JobsTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/JobsTableValuedFunction.java
@@ -17,11 +17,12 @@
 
 package org.apache.doris.tablefunction;
 
-import org.apache.doris.analysis.UserIdentity;
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
 import org.apache.doris.job.common.JobType;
 import org.apache.doris.job.extensions.insert.InsertJob;
 import org.apache.doris.job.extensions.mtmv.MTMVJob;
+import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.thrift.TJobsMetadataParams;
@@ -64,9 +65,10 @@ public JobsTableValuedFunction(Map<String, String> params) throws AnalysisExcept
             throw new AnalysisException("Invalid job metadata query");
         }
         this.jobType = jobType;
-        UserIdentity userIdentity = ConnectContext.get().getCurrentUserIdentity();
-        if (!userIdentity.isRootUser()) {
-            throw new AnalysisException("only root user can operate");
+        if (jobType != JobType.MV) {
+            if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+                throw new AnalysisException("only ADMIN priv can operate");
+            }
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 9c773b37dca50d..5b4069ae8e4e64 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -31,6 +31,7 @@
 import org.apache.doris.datasource.CatalogIf;
 import org.apache.doris.datasource.InternalCatalog;
 import org.apache.doris.job.common.JobType;
+import org.apache.doris.job.extensions.mtmv.MTMVJob;
 import org.apache.doris.job.task.AbstractTask;
 import org.apache.doris.mtmv.MTMVUtil;
 import org.apache.doris.mysql.privilege.PrivPredicate;
@@ -586,12 +587,20 @@ private static TFetchSchemaTableDataResult jobMetadataResult(TMetadataTableReque
         TJobsMetadataParams jobsMetadataParams = params.getJobsMetadataParams();
         String type = jobsMetadataParams.getType();
         JobType jobType = JobType.valueOf(type);
+        TUserIdentity currentUserIdent = jobsMetadataParams.getCurrentUserIdent();
+        UserIdentity userIdentity = UserIdentity.fromThrift(currentUserIdent);
         List<TRow> dataBatch = Lists.newArrayList();
         TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
 
         List<org.apache.doris.job.base.AbstractJob> jobList = Env.getCurrentEnv().getJobManager().queryJobs(jobType);
 
         for (org.apache.doris.job.base.AbstractJob job : jobList) {
+            if (job instanceof MTMVJob) {
+                MTMVJob mtmvJob = (MTMVJob) job;
+                if (!mtmvJob.hasPriv(userIdentity, PrivPredicate.SHOW)) {
+                    continue;
+                }
+            }
             dataBatch.add(job.getTvfInfo());
         }
         result.setDataBatch(dataBatch);
@@ -607,12 +616,20 @@ private static TFetchSchemaTableDataResult taskMetadataResult(TMetadataTableRequ
         TTasksMetadataParams tasksMetadataParams = params.getTasksMetadataParams();
         String type = tasksMetadataParams.getType();
         JobType jobType = JobType.valueOf(type);
+        TUserIdentity currentUserIdent = tasksMetadataParams.getCurrentUserIdent();
+        UserIdentity userIdentity = UserIdentity.fromThrift(currentUserIdent);
         List<TRow> dataBatch = Lists.newArrayList();
         TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult();
 
         List<org.apache.doris.job.base.AbstractJob> jobList = Env.getCurrentEnv().getJobManager().queryJobs(jobType);
 
         for (org.apache.doris.job.base.AbstractJob job : jobList) {
+            if (job instanceof MTMVJob) {
+                MTMVJob mtmvJob = (MTMVJob) job;
+                if (!mtmvJob.hasPriv(userIdentity, PrivPredicate.SHOW)) {
+                    continue;
+                }
+            }
             List<AbstractTask> tasks = job.queryAllTasks();
             for (AbstractTask task : tasks) {
                 TRow tvfInfo = task.getTvfInfo();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TasksTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TasksTableValuedFunction.java
index c2df1cc000aab2..c7015f57ff6197 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TasksTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/TasksTableValuedFunction.java
@@ -17,11 +17,12 @@
 
 package org.apache.doris.tablefunction;
 
-import org.apache.doris.analysis.UserIdentity;
 import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.Env;
 import org.apache.doris.job.common.JobType;
 import org.apache.doris.job.extensions.insert.InsertTask;
 import org.apache.doris.job.extensions.mtmv.MTMVTask;
+import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.nereids.exceptions.AnalysisException;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.thrift.TMetaScanRange;
@@ -64,9 +65,10 @@ public TasksTableValuedFunction(Map<String, String> params) throws AnalysisExcep
             throw new AnalysisException("Invalid task metadata query");
         }
         this.jobType = jobType;
-        UserIdentity userIdentity = ConnectContext.get().getCurrentUserIdentity();
-        if (!userIdentity.isRootUser()) {
-            throw new AnalysisException("only root user can operate");
+        if (jobType != JobType.MV) {
+            if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) {
+                throw new AnalysisException("only ADMIN priv can operate");
+            }
         }
     }
 
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
index 942f44f8d8d278..cd7a01b8b97460 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
@@ -853,7 +853,7 @@ class Suite implements GroovyInterceptable {
 
     void waitingMTMVTaskFinished(String jobName) {
         Thread.sleep(2000);
-        String showTasks = "select TaskId,JobId,JobName,MvId,Status from tasks('type'='mv') where JobName = '${jobName}'"
+        String showTasks = "select TaskId,JobId,JobName,MvId,Status from tasks('type'='mv') where JobName = '${jobName}' order by CreateTime ASC"
         String status = "NULL"
         List<List<Object>> result
         long startTime = System.currentTimeMillis()
diff --git a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
index eb4560f8b06de7..05bc5e5322e4f4 100644
--- a/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_build_mtmv.groovy
@@ -88,6 +88,10 @@ suite("test_build_mtmv") {
     logger.info("showCreateTableResult: " + showCreateTableResult.toString())
     assertTrue(showCreateTableResult.toString().contains("CREATE MATERIALIZED VIEW `multi_mv_test_create_mtmv` (\n  `aa` BIGINT NULL COMMENT 'aaa',\n  `bb` VARCHAR(20) NULL\n) ENGINE=MATERIALIZED_VIEW\nCOMMENT 'comment1'\nDISTRIBUTED BY RANDOM BUCKETS 2\nPROPERTIES"))
 
+    def descTableAllResult = sql """desc ${mvName} all"""
+    logger.info("descTableAllResult: " + descTableAllResult.toString())
+    assertTrue(descTableAllResult.toString().contains("${mvName}"))
+
     // if not exist
     try {
         sql """
diff --git a/regression-test/suites/mtmv_p0/test_task_mtmv.groovy b/regression-test/suites/mtmv_p0/test_task_mtmv.groovy
new file mode 100644
index 00000000000000..a3e59afd66c1e0
--- /dev/null
+++ b/regression-test/suites/mtmv_p0/test_task_mtmv.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_task_mtmv") {
+    def tableName = "t_test_task_mtmv_user"
+    def mvName = "multi_mv_test_task_mtmv"
+    def dbName = "regression_test_mtmv_p0"
+    sql """drop materialized view if exists ${mvName};"""
+    sql """drop table if exists `${tableName}`"""
+    sql """
+        CREATE TABLE IF NOT EXISTS `${tableName}` (
+            event_day DATE,
+            id BIGINT,
+            username VARCHAR(20)
+        )
+        DISTRIBUTED BY HASH(id) BUCKETS 10 
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+        """
+
+    sql """
+        CREATE MATERIALIZED VIEW ${mvName}
+        BUILD DEFERRED REFRESH COMPLETE ON MANUAL
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS 
+        SELECT * FROM ${tableName};
+    """
+     sql """
+        REFRESH MATERIALIZED VIEW ${mvName}
+    """
+    def jobName = getJobName(dbName, mvName);
+    waitingMTMVTaskFinished(jobName)
+    def taskIdArr = sql """ select TaskId from tasks('type'='mv') where MvName = '${mvName}';"""
+    def taskId = taskIdArr.get(0).get(0);
+    logger.info("taskId: " + taskId.toString())
+     try {
+        sql """
+                cancel MATERIALIZED VIEW TASK ${taskId} on ${mvName};
+            """
+        } catch (Exception e) {
+           log.info("cancel error msg: " + e.getMessage())
+           assertTrue(e.getMessage().contains("no running task"));
+        }
+    sql """drop materialized view if exists ${mvName};"""
+
+}

From 01a0f0915ffac8a067559cc97220deadd9a72ab9 Mon Sep 17 00:00:00 2001
From: zhangdong <493738387@qq.com>
Date: Mon, 1 Jan 2024 08:12:26 +0800
Subject: [PATCH 232/299] [fix](mtmv)fix  insert overwrite will generate
 garbage temporary partition when restarting FE (#29075)

---
 .../java/org/apache/doris/catalog/Env.java    |  23 +++
 .../insertoverwrite/InsertOverwriteLog.java   |  81 ++++++++
 .../InsertOverwriteManager.java               | 194 ++++++++++++++++++
 .../insertoverwrite/InsertOverwriteTask.java  |  96 +++++++++
 .../insertoverwrite/InsertOverwriteUtil.java  | 133 ++++++++++++
 .../apache/doris/journal/JournalEntity.java   |   6 +
 .../commands/InsertOverwriteTableCommand.java | 175 +++-------------
 .../org/apache/doris/persist/EditLog.java     |  10 +
 .../apache/doris/persist/OperationType.java   |   2 +
 .../doris/persist/meta/MetaPersistMethod.java |   6 +
 .../persist/meta/PersistMetaModules.java      |   3 +-
 11 files changed, 587 insertions(+), 142 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteLog.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteTask.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index aba414ecdf6ce0..b4b483738516c0 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -146,6 +146,7 @@
 import org.apache.doris.httpv2.entity.ResponseBody;
 import org.apache.doris.httpv2.meta.MetaBaseAction;
 import org.apache.doris.httpv2.rest.RestApiStatusCode;
+import org.apache.doris.insertoverwrite.InsertOverwriteManager;
 import org.apache.doris.job.base.AbstractJob;
 import org.apache.doris.job.extensions.mtmv.MTMVTask;
 import org.apache.doris.job.manager.JobManager;
@@ -515,6 +516,8 @@ public class Env {
 
     private MTMVService mtmvService;
 
+    private InsertOverwriteManager insertOverwriteManager;
+
     public List<TFrontendInfo> getFrontendInfos() {
         List<TFrontendInfo> res = new ArrayList<>();
 
@@ -748,6 +751,7 @@ private Env(boolean isCheckpointCatalog) {
         this.workloadActionPublisherThread = new WorkloadActionPublishThread("WorkloadActionPublisher",
                 Config.workload_action_interval_ms, systemInfo);
         this.mtmvService = new MTMVService();
+        this.insertOverwriteManager = new InsertOverwriteManager();
     }
 
     public static void destroyCheckpoint() {
@@ -807,6 +811,10 @@ public MTMVService getMtmvService() {
         return mtmvService;
     }
 
+    public InsertOverwriteManager getInsertOverwriteManager() {
+        return insertOverwriteManager;
+    }
+
     public TabletScheduler getTabletScheduler() {
         return tabletScheduler;
     }
@@ -1495,6 +1503,8 @@ private void transferToMaster() {
         // so no need to check 'isReady' flag in this method
         postProcessAfterMetadataReplayed(false);
 
+        insertOverwriteManager.allTaskFail();
+
         toMasterProgress = "start daemon threads";
 
         // start all daemon threads that only running on MASTER FE
@@ -1626,6 +1636,7 @@ private void startMasterOnlyDaemonThreads() {
         // binlog gcer
         binlogGcer.start();
         columnIdFlusher.start();
+        insertOverwriteManager.start();
     }
 
     // start threads that should running on all FE
@@ -2148,6 +2159,18 @@ public long loadAnalysisManager(DataInputStream in, long checksum) throws IOExce
         return checksum;
     }
 
+    public long loadInsertOverwrite(DataInputStream in, long checksum) throws IOException {
+        this.insertOverwriteManager = InsertOverwriteManager.read(in);
+        LOG.info("finished replay iot from image");
+        return checksum;
+    }
+
+    public long saveInsertOverwrite(CountingDataOutputStream out, long checksum) throws IOException {
+        this.insertOverwriteManager.write(out);
+        LOG.info("finished save iot to image");
+        return checksum;
+    }
+
     // Only called by checkpoint thread
     // return the latest image file's absolute path
     public String saveImage() throws IOException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteLog.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteLog.java
new file mode 100644
index 00000000000000..0d5118438041de
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteLog.java
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.insertoverwrite;
+
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class InsertOverwriteLog implements Writable {
+
+    public enum InsertOverwriteOpType {
+        ADD,
+        DROP,
+        CANCEL
+    }
+
+    @SerializedName(value = "id")
+    private long taskId;
+    @SerializedName(value = "task")
+    private InsertOverwriteTask task;
+    @SerializedName(value = "type")
+    private InsertOverwriteOpType opType;
+
+    public InsertOverwriteLog(long taskId, InsertOverwriteTask task,
+            InsertOverwriteOpType opType) {
+        this.taskId = taskId;
+        this.task = task;
+        this.opType = opType;
+    }
+
+    public long getTaskId() {
+        return taskId;
+    }
+
+    public InsertOverwriteTask getTask() {
+        return task;
+    }
+
+    public InsertOverwriteOpType getOpType() {
+        return opType;
+    }
+
+    @Override
+    public String toString() {
+        return "InsertOverwriteLog{"
+                + "taskId=" + taskId
+                + ", task=" + task
+                + ", opType=" + opType
+                + '}';
+    }
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+        Text.writeString(out, GsonUtils.GSON.toJson(this));
+    }
+
+    public static InsertOverwriteLog read(DataInput in) throws IOException {
+        return GsonUtils.GSON.fromJson(Text.readString(in), InsertOverwriteLog.class);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
new file mode 100644
index 00000000000000..9835efa531b2ef
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteManager.java
@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.insertoverwrite;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.common.util.MasterDaemon;
+import org.apache.doris.insertoverwrite.InsertOverwriteLog.InsertOverwriteOpType;
+import org.apache.doris.persist.gson.GsonUtils;
+
+import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+public class InsertOverwriteManager extends MasterDaemon implements Writable {
+    private static final Logger LOG = LogManager.getLogger(InsertOverwriteManager.class);
+
+    private static final long CLEAN_INTERVAL_SECOND = 10;
+
+    @SerializedName(value = "tasks")
+    private Map<Long, InsertOverwriteTask> tasks = Maps.newConcurrentMap();
+
+    public InsertOverwriteManager() {
+        super("InsertOverwriteDropDirtyPartitions", CLEAN_INTERVAL_SECOND * 1000);
+    }
+
+    /**
+     * register insert overwrite task
+     *
+     * @param dbId
+     * @param tableId
+     * @param tempPartitionNames
+     * @return taskId
+     */
+    public long registerTask(long dbId, long tableId, List<String> tempPartitionNames) {
+        long taskId = Env.getCurrentEnv().getNextId();
+        InsertOverwriteTask task = new InsertOverwriteTask(dbId, tableId,
+                tempPartitionNames);
+        tasks.put(taskId, task);
+        Env.getCurrentEnv().getEditLog()
+                .logInsertOverwrite(new InsertOverwriteLog(taskId, task, InsertOverwriteOpType.ADD));
+        return taskId;
+    }
+
+    /**
+     * when insert overwrite fail, try drop temp partition
+     *
+     * @param taskId
+     */
+    public void taskFail(long taskId) {
+        boolean rollback = rollback(taskId);
+        if (rollback) {
+            removeTask(taskId);
+        } else {
+            cancelTask(taskId);
+        }
+    }
+
+    /**
+     * when insert overwrite success, drop task
+     *
+     * @param taskId
+     */
+    public void taskSuccess(long taskId) {
+        removeTask(taskId);
+    }
+
+    /**
+     * for transferToMaster, try drop all temp partitions
+     */
+    public void allTaskFail() {
+        LOG.info("try drop all temp partitions when transferToMaster");
+        HashMap<Long, InsertOverwriteTask> copyTasks = Maps.newHashMap(tasks);
+        for (Entry<Long, InsertOverwriteTask> entry : copyTasks.entrySet()) {
+            taskFail(entry.getKey());
+        }
+    }
+
+    private void cancelTask(long taskId) {
+        if (tasks.containsKey(taskId)) {
+            LOG.info("cancel insert overwrite task: {}", tasks.get(taskId));
+            tasks.get(taskId).setCancel(true);
+            Env.getCurrentEnv().getEditLog()
+                    .logInsertOverwrite(new InsertOverwriteLog(taskId, null, InsertOverwriteOpType.CANCEL));
+        }
+    }
+
+    private void removeTask(long taskId) {
+        if (tasks.containsKey(taskId)) {
+            LOG.info("remove insert overwrite task: {}", tasks.get(taskId));
+            tasks.remove(taskId);
+            Env.getCurrentEnv().getEditLog()
+                    .logInsertOverwrite(new InsertOverwriteLog(taskId, null, InsertOverwriteOpType.DROP));
+        }
+    }
+
+    /**
+     * drop temp partitions
+     *
+     * @param taskId
+     * @return if success
+     */
+    private boolean rollback(long taskId) {
+        InsertOverwriteTask task = tasks.get(taskId);
+        OlapTable olapTable;
+        try {
+            olapTable = task.getTable();
+        } catch (DdlException e) {
+            LOG.warn("can not get table, task: {}", task);
+            return true;
+        }
+        return InsertOverwriteUtil.dropPartitions(olapTable, task.getTempPartitionNames());
+    }
+
+    /**
+     * replay logs
+     *
+     * @param insertOverwriteLog
+     */
+    public void replayInsertOverwriteLog(InsertOverwriteLog insertOverwriteLog) {
+        switch (insertOverwriteLog.getOpType()) {
+            case ADD:
+                tasks.put(insertOverwriteLog.getTaskId(), insertOverwriteLog.getTask());
+                break;
+            case DROP:
+                tasks.remove(insertOverwriteLog.getTaskId());
+                break;
+            case CANCEL:
+                InsertOverwriteTask task = tasks.get(insertOverwriteLog.getTaskId());
+                if (task != null) {
+                    task.setCancel(true);
+                }
+                break;
+            default:
+                LOG.warn("error insertOverwriteLog: {}", insertOverwriteLog.toString());
+        }
+    }
+
+    /**
+     * Regularly drop partitions that have failed dropped
+     */
+    @Override
+    protected void runAfterCatalogReady() {
+        LOG.info("start clean insert overwrite temp partitions");
+        HashMap<Long, InsertOverwriteTask> copyTasks = Maps.newHashMap(tasks);
+        for (Entry<Long, InsertOverwriteTask> entry : copyTasks.entrySet()) {
+            if (entry.getValue().isCancel()) {
+                boolean rollback = rollback(entry.getKey());
+                if (rollback) {
+                    removeTask(entry.getKey());
+                }
+            }
+        }
+    }
+
+
+    @Override
+    public void write(DataOutput out) throws IOException {
+        String json = GsonUtils.GSON.toJson(this);
+        Text.writeString(out, json);
+    }
+
+    public static InsertOverwriteManager read(DataInput in) throws IOException {
+        String json = Text.readString(in);
+        return GsonUtils.GSON.fromJson(json, InsertOverwriteManager.class);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteTask.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteTask.java
new file mode 100644
index 00000000000000..b37dba6a6f2ad4
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteTask.java
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.insertoverwrite;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf.TableType;
+import org.apache.doris.common.DdlException;
+
+import com.google.gson.annotations.SerializedName;
+
+import java.util.List;
+
+public class InsertOverwriteTask {
+    @SerializedName(value = "cancel")
+    private boolean cancel;
+    @SerializedName(value = "dbId")
+    private long dbId;
+    @SerializedName(value = "tId")
+    private long tableId;
+    @SerializedName(value = "tpns")
+    private List<String> tempPartitionNames;
+
+    public InsertOverwriteTask() {
+    }
+
+    public InsertOverwriteTask(long dbId, long tableId, List<String> tempPartitionNames) {
+        this.dbId = dbId;
+        this.tableId = tableId;
+        this.tempPartitionNames = tempPartitionNames;
+        this.cancel = false;
+    }
+
+    public boolean isCancel() {
+        return cancel;
+    }
+
+    public void setCancel(boolean cancel) {
+        this.cancel = cancel;
+    }
+
+    public long getDbId() {
+        return dbId;
+    }
+
+    public void setDbId(long dbId) {
+        this.dbId = dbId;
+    }
+
+    public long getTableId() {
+        return tableId;
+    }
+
+    public void setTableId(long tableId) {
+        this.tableId = tableId;
+    }
+
+    public List<String> getTempPartitionNames() {
+        return tempPartitionNames;
+    }
+
+    public void setTempPartitionNames(List<String> tempPartitionNames) {
+        this.tempPartitionNames = tempPartitionNames;
+    }
+
+    public OlapTable getTable() throws DdlException {
+        return (OlapTable) Env.getCurrentEnv().getInternalCatalog().getDbOrDdlException(dbId)
+                .getTableOrDdlException(tableId,
+                        TableType.OLAP);
+    }
+
+    @Override
+    public String toString() {
+        return "InsertOverwriteTask{"
+                + "cancel=" + cancel
+                + ", dbId=" + dbId
+                + ", tableId=" + tableId
+                + ", tempPartitionNames=" + tempPartitionNames
+                + '}';
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java
new file mode 100644
index 00000000000000..af906d2653e7d9
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/insertoverwrite/InsertOverwriteUtil.java
@@ -0,0 +1,133 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.insertoverwrite;
+
+import org.apache.doris.analysis.AddPartitionLikeClause;
+import org.apache.doris.analysis.DropPartitionClause;
+import org.apache.doris.analysis.PartitionNames;
+import org.apache.doris.analysis.ReplacePartitionClause;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.util.PropertyAnalyzer;
+
+import com.google.common.collect.Maps;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+public class InsertOverwriteUtil {
+    private static final Logger LOG = LogManager.getLogger(InsertOverwriteUtil.class);
+
+    /**
+     * add temp partitions
+     *
+     * @param olapTable
+     * @param partitionNames
+     * @param tempPartitionNames
+     * @throws DdlException
+     */
+    public static void addTempPartitions(OlapTable olapTable, List<String> partitionNames,
+            List<String> tempPartitionNames) throws DdlException {
+        for (int i = 0; i < partitionNames.size(); i++) {
+            Env.getCurrentEnv().addPartitionLike((Database) olapTable.getDatabase(), olapTable.getName(),
+                    new AddPartitionLikeClause(tempPartitionNames.get(i), partitionNames.get(i), true));
+        }
+    }
+
+    /**
+     * replace partitions
+     *
+     * @param olapTable
+     * @param partitionNames
+     * @param tempPartitionNames
+     * @throws DdlException
+     */
+    public static void replacePartition(OlapTable olapTable, List<String> partitionNames,
+            List<String> tempPartitionNames) throws DdlException {
+        try {
+            if (!olapTable.writeLockIfExist()) {
+                return;
+            }
+            Map<String, String> properties = Maps.newHashMap();
+            properties.put(PropertyAnalyzer.PROPERTIES_USE_TEMP_PARTITION_NAME, "false");
+            ReplacePartitionClause replacePartitionClause = new ReplacePartitionClause(
+                    new PartitionNames(false, partitionNames),
+                    new PartitionNames(true, tempPartitionNames), properties);
+            Env.getCurrentEnv()
+                    .replaceTempPartition((Database) olapTable.getDatabase(), olapTable, replacePartitionClause);
+        } finally {
+            olapTable.writeUnlock();
+        }
+
+    }
+
+    /**
+     * generate temp partitionName
+     *
+     * @param partitionNames
+     * @return
+     */
+    public static List<String> generateTempPartitionNames(List<String> partitionNames) {
+        List<String> tempPartitionNames = new ArrayList<String>(partitionNames.size());
+        for (String partitionName : partitionNames) {
+            String tempPartitionName = "iot_temp_" + partitionName;
+            if (tempPartitionName.length() > 50) {
+                tempPartitionName = tempPartitionName.substring(0, 30) + Math.abs(Objects.hash(tempPartitionName))
+                        + "_" + System.currentTimeMillis();
+            }
+            tempPartitionNames.add(tempPartitionName);
+        }
+        return tempPartitionNames;
+    }
+
+    /**
+     * drop temp partitions
+     *
+     * @param olapTable
+     * @param tempPartitionNames
+     * @return
+     */
+    public static boolean dropPartitions(OlapTable olapTable, List<String> tempPartitionNames) {
+        try {
+            if (!olapTable.writeLockIfExist()) {
+                return true;
+            }
+            for (String partitionName : tempPartitionNames) {
+                if (olapTable.getPartition(partitionName, true) == null) {
+                    continue;
+                }
+                Env.getCurrentEnv().dropPartition(
+                        (Database) olapTable.getDatabase(), olapTable,
+                        new DropPartitionClause(true, partitionName, true, true));
+                LOG.info("successfully drop temp partition [{}] for [{}]", partitionName, olapTable.getName());
+            }
+        } catch (DdlException e) {
+            LOG.info("drop partition failed for [{}]", olapTable.getName(), e);
+            return false;
+        } finally {
+            olapTable.writeUnlock();
+        }
+        return true;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
index aae8f4e4e7f837..6f6bc945096ea5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java
@@ -44,6 +44,7 @@
 import org.apache.doris.datasource.InitDatabaseLog;
 import org.apache.doris.datasource.InitTableLog;
 import org.apache.doris.ha.MasterInfo;
+import org.apache.doris.insertoverwrite.InsertOverwriteLog;
 import org.apache.doris.job.base.AbstractJob;
 import org.apache.doris.journal.bdbje.Timestamp;
 import org.apache.doris.load.DeleteInfo;
@@ -888,6 +889,11 @@ public void readFields(DataInput in) throws IOException {
                 isRead = true;
                 break;
             }
+            case OperationType.OP_INSERT_OVERWRITE: {
+                data = InsertOverwriteLog.read(in);
+                isRead = true;
+                break;
+            }
             case OperationType.OP_ALTER_REPOSITORY: {
                 data = Repository.read(in);
                 isRead = true;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertOverwriteTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertOverwriteTableCommand.java
index 95a36146c71e3a..501f75b78dbffb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertOverwriteTableCommand.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/InsertOverwriteTableCommand.java
@@ -17,21 +17,13 @@
 
 package org.apache.doris.nereids.trees.plans.commands;
 
-import org.apache.doris.analysis.AddPartitionLikeClause;
-import org.apache.doris.analysis.AlterClause;
-import org.apache.doris.analysis.AlterTableStmt;
-import org.apache.doris.analysis.Analyzer;
-import org.apache.doris.analysis.DropPartitionClause;
-import org.apache.doris.analysis.PartitionNames;
-import org.apache.doris.analysis.ReplacePartitionClause;
-import org.apache.doris.analysis.TableName;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.common.ErrorCode;
 import org.apache.doris.common.ErrorReport;
 import org.apache.doris.common.UserException;
-import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.insertoverwrite.InsertOverwriteUtil;
 import org.apache.doris.mysql.privilege.PrivPredicate;
 import org.apache.doris.nereids.NereidsPlanner;
 import org.apache.doris.nereids.analyzer.UnboundTableSink;
@@ -45,7 +37,6 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
 import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
 import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.DdlExecutor;
 import org.apache.doris.qe.QueryState.MysqlStateType;
 import org.apache.doris.qe.StmtExecutor;
 
@@ -56,14 +47,10 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.UUID;
 
 /**
  * insert into select command implementation
@@ -133,50 +120,25 @@ public void run(ConnectContext ctx, StmtExecutor executor) throws Exception {
                     targetTable.getQualifiedDbName() + ": " + targetTable.getName());
         }
 
-        TableName tableName = new TableName(InternalCatalog.INTERNAL_CATALOG_NAME,
-                targetTable.getQualifiedDbName(), targetTable.getName());
         ConnectContext.get().setSkipAuth(true);
-        try {
-            List<String> partitionNames = ((UnboundTableSink<?>) logicalQuery).getPartitions();
-            if (CollectionUtils.isEmpty(partitionNames)) {
-                partitionNames = Lists.newArrayList(targetTable.getPartitionNames());
-            }
-            List<String> tempPartitionNames = addTempPartitions(ctx, tableName, partitionNames);
-            boolean insertRes = insertInto(ctx, executor, tempPartitionNames, tableName);
-            if (!insertRes) {
-                return;
-            }
-            replacePartition(ctx, tableName, partitionNames, tempPartitionNames);
-        } finally {
-            ConnectContext.get().setSkipAuth(false);
+        List<String> partitionNames = ((UnboundTableSink<?>) logicalQuery).getPartitions();
+        if (CollectionUtils.isEmpty(partitionNames)) {
+            partitionNames = Lists.newArrayList(targetTable.getPartitionNames());
         }
-    }
-
-    /**
-     * replacing partitionNames with tempPartitionNames
-     *
-     * @param ctx ctx
-     * @param tableName tableName
-     * @param partitionNames partitionNames
-     * @param tempPartitionNames tempPartitionNames
-     * @throws UserException UserException
-     */
-    private void replacePartition(ConnectContext ctx, TableName tableName, List<String> partitionNames,
-            List<String> tempPartitionNames)
-            throws UserException {
-        // overwrite old partition with tmp partition
+        List<String> tempPartitionNames = InsertOverwriteUtil.generateTempPartitionNames(partitionNames);
+        long taskId = Env.getCurrentEnv().getInsertOverwriteManager()
+                .registerTask(targetTable.getDatabase().getId(), targetTable.getId(), tempPartitionNames);
         try {
-            List<AlterClause> ops = new ArrayList<>();
-            Map<String, String> properties = new HashMap<>();
-            properties.put("use_temp_partition_name", "false");
-            ops.add(new ReplacePartitionClause(new PartitionNames(false, partitionNames),
-                    new PartitionNames(true, tempPartitionNames), properties));
-            AlterTableStmt alterTableStmt = new AlterTableStmt(tableName, ops);
-            Env.getCurrentEnv().alterTable(alterTableStmt);
+            InsertOverwriteUtil.addTempPartitions(targetTable, partitionNames, tempPartitionNames);
+            insertInto(ctx, executor, tempPartitionNames);
+            InsertOverwriteUtil.replacePartition(targetTable, partitionNames, tempPartitionNames);
+            Env.getCurrentEnv().getInsertOverwriteManager().taskSuccess(taskId);
         } catch (Exception e) {
-            LOG.warn("IOT overwrite table partitions error", e);
-            rollback(ctx, tableName, tempPartitionNames);
+            LOG.warn("insert into overwrite failed");
+            Env.getCurrentEnv().getInsertOverwriteManager().taskFail(taskId);
             throw e;
+        } finally {
+            ConnectContext.get().setSkipAuth(false);
         }
     }
 
@@ -186,96 +148,27 @@ private void replacePartition(ConnectContext ctx, TableName tableName, List<Stri
      * @param ctx ctx
      * @param executor executor
      * @param tempPartitionNames tempPartitionNames
-     * @param tableName tableName
      */
-    private boolean insertInto(ConnectContext ctx, StmtExecutor executor, List<String> tempPartitionNames,
-            TableName tableName) {
-        try {
-            UnboundTableSink<?> sink = (UnboundTableSink<?>) logicalQuery;
-            UnboundTableSink<?> copySink = new UnboundTableSink<>(
-                    sink.getNameParts(),
-                    sink.getColNames(),
-                    sink.getHints(),
-                    true,
-                    tempPartitionNames,
-                    sink.isPartialUpdate(),
-                    sink.getDMLCommandType(),
-                    (LogicalPlan) (sink.child(0)));
-            // for overwrite situation, we disable auto create partition.
-            InsertIntoTableCommand insertCommand = new InsertIntoTableCommand(copySink, labelName);
-            insertCommand.setAllowAutoPartition(false);
-            insertCommand.run(ctx, executor);
-            if (ctx.getState().getStateType() == MysqlStateType.ERR) {
-                String errMsg = Strings.emptyToNull(ctx.getState().getErrorMessage());
-                LOG.warn("InsertInto state error:{}", errMsg);
-                rollback(ctx, tableName, tempPartitionNames);
-                return false;
-            }
-            return true;
-        } catch (Exception e) {
-            LOG.warn("InsertInto error", e);
-            rollback(ctx, tableName, tempPartitionNames);
-            return false;
-        }
-    }
-
-    /**
-     * add some tempPartitions
-     *
-     * @param ctx ctx
-     * @param tableName tableName
-     * @param partitionNames partitionNames
-     * @return tempPartitionNames
-     * @throws Exception Exception
-     */
-    private List<String> addTempPartitions(ConnectContext ctx, TableName tableName, List<String> partitionNames)
+    private void insertInto(ConnectContext ctx, StmtExecutor executor, List<String> tempPartitionNames)
             throws Exception {
-        List<String> tempPartitionNames = new ArrayList<>();
-        try {
-            // create tmp partitions with uuid
-            for (String partitionName : partitionNames) {
-                UUID uuid = UUID.randomUUID();
-                // to comply with naming rules
-                String tempPartName = "tmp_partition_" + uuid.toString().replace('-', '_');
-                List<AlterClause> ops = new ArrayList<>();
-                ops.add(new AddPartitionLikeClause(tempPartName, partitionName, true));
-
-                AlterTableStmt alterTableStmt = new AlterTableStmt(tableName, ops);
-                Analyzer tempAnalyzer = new Analyzer(Env.getCurrentEnv(), ctx);
-                alterTableStmt.analyze(tempAnalyzer);
-                DdlExecutor.execute(ctx.getEnv(), alterTableStmt);
-                // only when execution succeeded, put the temp partition name into list
-                tempPartitionNames.add(tempPartName);
-            }
-            return tempPartitionNames;
-        } catch (Exception e) {
-            LOG.warn("IOT create tmp table partitions error", e);
-            rollback(ctx, tableName, tempPartitionNames);
-            throw e;
-        }
-    }
-
-    /**
-     * delete temp partitions
-     *
-     * @param ctx ctx
-     * @param targetTableName targetTableName
-     * @param tempPartitionNames tempPartitionNames
-     */
-    private void rollback(ConnectContext ctx, TableName targetTableName,
-            List<String> tempPartitionNames) {
-        try {
-            for (String partitionName : tempPartitionNames) {
-                List<AlterClause> ops = new ArrayList<>();
-                ops.add(new DropPartitionClause(true, partitionName, true, true));
-                AlterTableStmt dropTablePartitionStmt = new AlterTableStmt(targetTableName, ops);
-                Analyzer tempAnalyzer = new Analyzer(Env.getCurrentEnv(), ctx);
-                dropTablePartitionStmt.analyze(tempAnalyzer);
-                DdlExecutor.execute(ctx.getEnv(), dropTablePartitionStmt);
-            }
-        } catch (Exception ex) {
-            LOG.warn("IOT drop partitions error", ex);
-            ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, "Unexpected exception: " + ex.getMessage());
+        UnboundTableSink<?> sink = (UnboundTableSink<?>) logicalQuery;
+        UnboundTableSink<?> copySink = new UnboundTableSink<>(
+                sink.getNameParts(),
+                sink.getColNames(),
+                sink.getHints(),
+                true,
+                tempPartitionNames,
+                sink.isPartialUpdate(),
+                sink.getDMLCommandType(),
+                (LogicalPlan) (sink.child(0)));
+        // for overwrite situation, we disable auto create partition.
+        InsertIntoTableCommand insertCommand = new InsertIntoTableCommand(copySink, labelName);
+        insertCommand.setAllowAutoPartition(false);
+        insertCommand.run(ctx, executor);
+        if (ctx.getState().getStateType() == MysqlStateType.ERR) {
+            String errMsg = Strings.emptyToNull(ctx.getState().getErrorMessage());
+            LOG.warn("InsertInto state error:{}", errMsg);
+            throw new UserException(errMsg);
         }
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
index f78f417e66f545..cbbae9d1788403 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
@@ -52,6 +52,7 @@
 import org.apache.doris.datasource.InitCatalogLog;
 import org.apache.doris.datasource.InitDatabaseLog;
 import org.apache.doris.ha.MasterInfo;
+import org.apache.doris.insertoverwrite.InsertOverwriteLog;
 import org.apache.doris.job.base.AbstractJob;
 import org.apache.doris.journal.Journal;
 import org.apache.doris.journal.JournalCursor;
@@ -1132,6 +1133,11 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
                     env.getAlterInstance().processAlterMTMV(alterMtmv, true);
                     break;
                 }
+                case OperationType.OP_INSERT_OVERWRITE: {
+                    final InsertOverwriteLog insertOverwriteLog = (InsertOverwriteLog) journal.getData();
+                    env.getInsertOverwriteManager().replayInsertOverwriteLog(insertOverwriteLog);
+                    break;
+                }
                 case OperationType.OP_ALTER_REPOSITORY: {
                     Repository repository = (Repository) journal.getData();
                     env.getBackupHandler().getRepoMgr().alterRepo(repository, true);
@@ -1963,6 +1969,10 @@ public void logAlterMTMV(AlterMTMV log) {
 
     }
 
+    public void logInsertOverwrite(InsertOverwriteLog log) {
+        logEdit(OperationType.OP_INSERT_OVERWRITE, log);
+    }
+
     public String getNotReadyReason() {
         if (journal == null) {
             return "journal is null";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
index 87b6b5bacfadbe..d26ddc12f4901f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java
@@ -353,6 +353,8 @@ public class OperationType {
 
     public static final short OP_ALTER_REPOSITORY = 460;
 
+    public static final short OP_INSERT_OVERWRITE = 461;
+
     /**
      * Get opcode name by op code.
      **/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
index 37741d84af66c9..2d777a50b624ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/MetaPersistMethod.java
@@ -240,6 +240,12 @@ public static MetaPersistMethod create(String name) throws NoSuchMethodException
                 break;
             case "JobTaskManager":
                 break;
+            case "insertOverwrite":
+                metaPersistMethod.readMethod =
+                        Env.class.getDeclaredMethod("loadInsertOverwrite", DataInputStream.class, long.class);
+                metaPersistMethod.writeMethod =
+                        Env.class.getDeclaredMethod("saveInsertOverwrite", CountingDataOutputStream.class, long.class);
+                break;
             default:
                 break;
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
index efaa732baa0670..32bf866f2288a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/persist/meta/PersistMetaModules.java
@@ -39,7 +39,8 @@ public class PersistMetaModules {
             "globalVariable", "cluster", "broker", "resources", "exportJob", "syncJob", "backupHandler",
             "paloAuth", "transactionState", "colocateTableIndex", "routineLoadJobs", "loadJobV2", "smallFiles",
             "plugins", "deleteHandler", "sqlBlockRule", "policy", "globalFunction", "workloadGroups",
-            "binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "workloadSchedPolicy");
+            "binlogs", "resourceGroups", "AnalysisMgrV2", "AsyncJobManager", "workloadSchedPolicy",
+            "insertOverwrite");
 
     // Modules in this list is deprecated and will not be saved in meta file. (also should not be in MODULE_NAMES)
     public static final ImmutableList<String> DEPRECATED_MODULE_NAMES = ImmutableList.of(

From 3d59f24403b76d80e917ee18eb35eb7fa0deecde Mon Sep 17 00:00:00 2001
From: Mingyu Chen <morningman@163.com>
Date: Mon, 1 Jan 2024 17:20:45 +0800
Subject: [PATCH 233/299] [fix](compatibility) add some keywords (#29251)

1. Add `BINARY` keywords
    `BINARY` is used for tell MySQL to treat a String(or String column) as case sensitive.
     eg: `"abc" = "ABC"` is true, but `BINARY "abc" = "ABC"` is false.

    But in Doris, `"abc" = "ABC"` is false by default.

     I add this `BINARY` keyword just for compatibility, it will take no effect.

2. Add `PARTITIONS` and `AUTO_INCREMENT` as reserved words.

    `PARTITIONS` is the table name in `information_schema` database.
    `AUTO_INCREMENT` is a column name of a table in `information_schema`
---
 .../antlr4/org/apache/doris/nereids/DorisLexer.g4 |  1 +
 .../org/apache/doris/nereids/DorisParser.g4       |  6 ++++--
 fe/fe-core/src/main/cup/sql_parser.cup            |  4 ++++
 .../doris/nereids/parser/NereidsParserTest.java   | 15 +++++++++++++++
 4 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index c7e823ac65d97b..a1eea971611de9 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -121,6 +121,7 @@ BEGIN: 'BEGIN';
 BETWEEN: 'BETWEEN';
 BIGINT: 'BIGINT';
 BIN: 'BIN';
+BINARY: 'BINARY';
 BINLOG: 'BINLOG';
 BITAND: 'BITAND';
 BITMAP: 'BITMAP';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 880986a6cec9f9..a91f207e47c5e8 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -678,7 +678,7 @@ primaryExpression
     | LEFT_PAREN query RIGHT_PAREN                                                             #subqueryExpression
     | ATSIGN identifierOrText                                                                  #userVariable
     | DOUBLEATSIGN (kind=(GLOBAL | SESSION) DOT)? identifier                                   #systemVariable
-    | identifier                                                                               #columnReference
+    | BINARY? identifier                                                                       #columnReference
     | base=primaryExpression DOT fieldName=identifier                                          #dereference
     | LEFT_PAREN expression RIGHT_PAREN                                                        #parenthesizedExpression
     | KEY (dbName=identifier DOT)? keyName=identifier                                          #encryptKey
@@ -760,7 +760,7 @@ constant
     | type=(DATE | DATEV1 | DATEV2 | TIMESTAMP) STRING_LITERAL                                 #typeConstructor
     | number                                                                                   #numericLiteral
     | booleanValue                                                                             #booleanLiteral
-    | STRING_LITERAL                                                                           #stringLiteral
+    | BINARY? STRING_LITERAL                                                                   #stringLiteral
     | LEFT_BRACKET (items+=constant)? (COMMA items+=constant)* RIGHT_BRACKET                   #arrayLiteral
     | LEFT_BRACE (items+=constant COLON items+=constant)?
        (COMMA items+=constant COLON items+=constant)* RIGHT_BRACE                              #mapLiteral
@@ -898,6 +898,7 @@ nonReserved
     | ARRAY
     | AT
     | AUTHORS
+    | AUTO_INCREMENT
     | BACKENDS
     | BACKUP
     | BEGIN
@@ -1067,6 +1068,7 @@ nonReserved
     | PASSWORD_HISTORY
     | PASSWORD_LOCK_TIME
     | PASSWORD_REUSE
+    | PARTITIONS
     | PATH
     | PAUSE
     | PERCENT
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 26b221c128c6e3..976ad030dfacfc 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -7557,6 +7557,8 @@ keyword ::=
     {: RESULT = id; :}
     | KW_ARRAY:id
     {: RESULT = id; :}
+    | KW_AUTO_INCREMENT:id
+    {: RESULT = id; :}
     | KW_BACKUP:id
     {: RESULT = id; :}
     | KW_BEGIN:id
@@ -7783,6 +7785,8 @@ keyword ::=
     {: RESULT = id; :}
     | KW_PARAMETER:id
     {: RESULT = id; :}
+    | KW_PARTITIONS:id
+    {: RESULT = id; :}
     | KW_PASSWORD:id
     {: RESULT = id; :}
     | KW_PASSWORD_EXPIRE:id
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index dc7e336433f245..6dbcacee40feaf 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -457,4 +457,19 @@ public void testParseCollate() {
         NereidsParser nereidsParser = new NereidsParser();
         nereidsParser.parseSingle(sql);
     }
+
+    @Test
+    public void testParseBinaryKeyword() {
+        String sql = "SELECT BINARY 'abc' FROM t";
+        NereidsParser nereidsParser = new NereidsParser();
+        nereidsParser.parseSingle(sql);
+    }
+
+    @Test
+    public void testParseReserveKeyword() {
+        // partitions and auto_increment are reserve keywords
+        String sql = "SELECT BINARY 'abc' FROM information_schema.partitions order by AUTO_INCREMENT";
+        NereidsParser nereidsParser = new NereidsParser();
+        nereidsParser.parseSingle(sql);
+    }
 }

From 216b53058d373b1327d068963eee7ca94269f06c Mon Sep 17 00:00:00 2001
From: zhangdong <493738387@qq.com>
Date: Mon, 1 Jan 2024 17:46:21 +0800
Subject: [PATCH 234/299] [doc](mtmv)mtmv doc (#27549)

---
 .../async-materialized-view.md                | 216 ++++++++++++++++++
 .../sql-functions/table-functions/jobs.md     |  90 ++++++++
 .../sql-functions/table-functions/mv_infos.md | 100 ++++++++
 .../sql-functions/table-functions/tasks.md    | 106 +++++++++
 .../Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md    |  75 ++++++
 .../Create/CREATE-ASYNC-MATERIALIZED-VIEW.md  | 216 ++++++++++++++++++
 .../Drop/DROP-ASYNC-MATERIALIZED-VIEW.md      |  68 ++++++
 .../CANCEL-MATERIALIZED-VIEW-TASK.md          |  56 +++++
 .../PAUSE-MATERIALIZED-VIEW.md                |  56 +++++
 .../REFRESH-MATERIALIZED-VIEW.md              |  72 ++++++
 .../RESUME-MATERIALIZED-VIEW.md               |  56 +++++
 .../async-materialized-view.md                | 216 ++++++++++++++++++
 .../sql-functions/table-functions/jobs.md     |  90 ++++++++
 .../sql-functions/table-functions/mv_infos.md | 100 ++++++++
 .../sql-functions/table-functions/tasks.md    | 106 +++++++++
 .../Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md    |  75 ++++++
 .../Create/CREATE-ASYNC-MATERIALIZED-VIEW.md  | 215 +++++++++++++++++
 .../Drop/DROP-ASYNC-MATERIALIZED-VIEW.md      |  68 ++++++
 .../CANCEL-MATERIALIZED-VIEW-TASK.md          |  56 +++++
 .../PAUSE-MATERIALIZED-VIEW.md                |  56 +++++
 .../REFRESH-MATERIALIZED-VIEW.md              |  72 ++++++
 .../RESUME-MATERIALIZED-VIEW.md               |  56 +++++
 22 files changed, 2221 insertions(+)
 create mode 100644 docs/en/docs/query-acceleration/async-materialized-view.md
 create mode 100644 docs/en/docs/sql-manual/sql-functions/table-functions/jobs.md
 create mode 100644 docs/en/docs/sql-manual/sql-functions/table-functions/mv_infos.md
 create mode 100644 docs/en/docs/sql-manual/sql-functions/table-functions/tasks.md
 create mode 100644 docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md
 create mode 100644 docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md
 create mode 100644 docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md
 create mode 100644 docs/en/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md
 create mode 100644 docs/en/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md
 create mode 100644 docs/en/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md
 create mode 100644 docs/en/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md
 create mode 100644 docs/zh-CN/docs/query-acceleration/async-materialized-view.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/table-functions/jobs.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/table-functions/mv_infos.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-functions/table-functions/tasks.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md
 create mode 100644 docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md

diff --git a/docs/en/docs/query-acceleration/async-materialized-view.md b/docs/en/docs/query-acceleration/async-materialized-view.md
new file mode 100644
index 00000000000000..2a77ff2d4b4a65
--- /dev/null
+++ b/docs/en/docs/query-acceleration/async-materialized-view.md
@@ -0,0 +1,216 @@
+---
+{
+    "title": "Asynchronous materialized view",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Asynchronous materialized view
+
+## Construction and maintenance of materialized views
+
+### Create materialized views
+
+Prepare two tables and data
+```sql
+use tpch;
+
+CREATE TABLE IF NOT EXISTS orders  (
+    o_orderkey       integer not null,
+    o_custkey        integer not null,
+    o_orderstatus    char(1) not null,
+    o_totalprice     decimalv3(15,2) not null,
+    o_orderdate      date not null,
+    o_orderpriority  char(15) not null,
+    o_clerk          char(15) not null,
+    o_shippriority   integer not null,
+    o_comment        varchar(79) not null
+    )
+    DUPLICATE KEY(o_orderkey, o_custkey)
+    PARTITION BY RANGE(o_orderdate)(
+    FROM ('2023-10-17') TO ('2023-10-20') INTERVAL 1 DAY)
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES ("replication_num" = "1");
+
+insert into orders values
+   (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'),
+   (2, 2, 'ok', 109.2, '2023-10-18', 'c','d',2, 'mm'),
+   (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy');
+
+CREATE TABLE IF NOT EXISTS lineitem (
+    l_orderkey    integer not null,
+    l_partkey     integer not null,
+    l_suppkey     integer not null,
+    l_linenumber  integer not null,
+    l_quantity    decimalv3(15,2) not null,
+    l_extendedprice  decimalv3(15,2) not null,
+    l_discount    decimalv3(15,2) not null,
+    l_tax         decimalv3(15,2) not null,
+    l_returnflag  char(1) not null,
+    l_linestatus  char(1) not null,
+    l_shipdate    date not null,
+    l_commitdate  date not null,
+    l_receiptdate date not null,
+    l_shipinstruct char(25) not null,
+    l_shipmode     char(10) not null,
+    l_comment      varchar(44) not null
+    )
+    DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber)
+    PARTITION BY RANGE(l_shipdate)
+    (FROM ('2023-10-17') TO ('2023-10-20') INTERVAL 1 DAY)
+    DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3
+    PROPERTIES ("replication_num" = "1");
+
+insert into lineitem values
+ (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'),
+ (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'),
+ (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx');
+```
+Create materialized views
+```sql
+CREATE MATERIALIZED VIEW mv1 
+        BUILD DEFERRED REFRESH AUTO ON MANUAL
+        partition by(l_shipdate)
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS 
+        select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total
+            from lineitem
+            left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate
+            group by
+            l_shipdate,
+            o_orderdate,
+            l_partkey,
+            l_suppkey;
+```
+
+Specific syntax can be viewed [CREATE MATERIALIZED VIEW](../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md)
+
+### View materialized view meta information
+
+```sql
+select * from mv_infos("database"="tpch") where Name="mv1";
+```
+
+The unique features of materialized views can be viewed through [mv_infos()](../sql-manual/sql-functions/table-functions/mv_infos.md)
+
+Properties related to table, still viewed through [SHOW TABLES](../sql-manual/sql-reference/Show-Statements/SHOW-TABLES.md)
+
+### Refresh materialized view
+
+The materialized view supports different refresh strategies, such as scheduled refresh and manual refresh. It also supports different refresh granularity, such as full refresh, incremental refresh of partition granularity, etc. Here we take manually refreshing partial partitions of the materialized view as an example.
+
+First, check the list of materialized view partitions
+```sql
+SHOW PARTITIONS FROM mv1;
+```
+
+Refresh partition named `p_20231017_20231018`
+```sql
+REFRESH MATERIALIZED VIEW mv1 partitions(p_20231017_20231018);
+```
+
+Specific syntax can be viewed [REFRESH MATERIALIZED VIEW](../sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md)
+
+### task management
+
+Each materialized view defaults to a job responsible for refreshing data, which is used to describe the refresh strategy and other information of the materialized view. Each time a refresh is triggered, a task is generated,
+Task is used to describe specific refresh information, such as the time used for refreshing, which partitions were refreshed, etc
+
+#### View jobs in materialized views
+
+```sql
+select * from jobs("type"="mv") order by CreateTime;
+```
+
+Specific syntax can be viewed [jobs("type"="mv")](../sql-manual/sql-functions/table-functions/jobs.md)
+
+#### Pause materialized view job scheduled scheduling
+
+```sql
+PAUSE MATERIALIZED VIEW JOB ON mv1;
+```
+
+Can pause the scheduled scheduling of materialized views
+
+Specific syntax can be viewed [PAUSE MATERIALIZED VIEW JOB](../sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md)
+
+#### RESUME materialized view job scheduling
+
+```sql
+RESUME MATERIALIZED VIEW JOB ON mv1;
+```
+
+Can RESUME scheduled scheduling of materialized views
+
+Specific syntax can be viewed [RESUME MATERIALIZED VIEW JOB](../sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md)
+
+#### Viewing tasks in materialized views
+
+```sql
+select * from tasks("type"="mv");
+```
+
+Specific syntax can be viewed [tasks("type"="mv")](../sql-manual/sql-functions/table-functions/tasks.md)
+
+#### Cancel the task of objectifying the view
+
+```sql
+CANCEL MATERIALIZED VIEW TASK realTaskId on mv1;
+```
+
+Can cancel the operation of this task
+
+Specific syntax can be viewed [CANCEL MATERIALIZED VIEW TASK](../sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md)
+
+### Modifying materialized views
+
+Modify the properties of materialized views
+```sql
+ALTER MATERIALIZED VIEW mv1 set("grace_period"="3333");
+```
+
+Modify the name of the materialized view, the refresh method of the materialized view, and the unique properties of the materialized view can be viewed through [ALTER MATERIALIZED VIEW](../sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md)
+
+The materialized view itself is also a Table, so Table related properties, such as the number of copies, are still modified through the syntax related to `ALTER TABLE`.
+
+### Delete materialized view
+
+```sql
+DROP MATERIALIZED VIEW mv1;
+```
+
+The materialized view has a dedicated deletion syntax and cannot be deleted through the drop table,
+
+Specific syntax can be viewed [DROP MATERIALIZED VIEW](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md)
+
+## The use of materialized views
+
+### Directly view data from materialized views
+
+The materialized view itself is also a Table, so it can be directly queried
+
+```sql
+select * FROM mv1;
+```
+
+### Transparent rewriting
\ No newline at end of file
diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/jobs.md b/docs/en/docs/sql-manual/sql-functions/table-functions/jobs.md
new file mode 100644
index 00000000000000..602ef985dbabaf
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-functions/table-functions/jobs.md
@@ -0,0 +1,90 @@
+---
+{
+    "title": "JOBS",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `jobs`
+
+### Name
+
+jobs
+
+### description
+
+Table function, generating a temporary task table, which can view job information in a certain task type.
+
+This function is used in the from clause.
+
+#### syntax
+
+`jobs("type"="")`
+
+jobs("type"="mv")Table structure:
+```sql
+mysql> desc function jobs("type"="mv");
++-------------------+------+------+-------+---------+-------+
+| Field             | Type | Null | Key   | Default | Extra |
++-------------------+------+------+-------+---------+-------+
+| Id                | TEXT | No   | false | NULL    | NONE  |
+| Name              | TEXT | No   | false | NULL    | NONE  |
+| MvId              | TEXT | No   | false | NULL    | NONE  |
+| MvName            | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseId      | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseName    | TEXT | No   | false | NULL    | NONE  |
+| ExecuteType       | TEXT | No   | false | NULL    | NONE  |
+| RecurringStrategy | TEXT | No   | false | NULL    | NONE  |
+| Status            | TEXT | No   | false | NULL    | NONE  |
+| CreateTime        | TEXT | No   | false | NULL    | NONE  |
++-------------------+------+------+-------+---------+-------+
+10 rows in set (0.00 sec)
+```
+
+* Id: job ID.
+* Name: job name.
+* MvId: Materialized View ID
+* MvName: Materialized View Name
+* MvDatabaseId: DB ID of the materialized view
+* MvDatabaseName: Name of the database to which the materialized view belongs
+* ExecuteType: Execution type
+* RecurringStrategy: Loop strategy
+* Status: Job status
+* CreateTime: Task creation time
+
+### example
+
+1. View jobs in all materialized views
+
+```sql
+mysql> select * from jobs("type"="mv");
+```
+
+2. View job with name `inner_mtmv_75043`
+
+```sql
+mysql> select * from jobs("type"="mv") where Name="inner_mtmv_75043";
+```
+
+### keywords
+
+    jobs
diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/mv_infos.md b/docs/en/docs/sql-manual/sql-functions/table-functions/mv_infos.md
new file mode 100644
index 00000000000000..e9c9c24c9a7d5e
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-functions/table-functions/mv_infos.md
@@ -0,0 +1,100 @@
+---
+{
+    "title": "MV_INFOS",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `mv_infos`
+
+### Name
+
+mv_infos
+
+### description
+
+Table function, generating temporary tables for asynchronous materialized views, which can view information about asynchronous materialized views created in a certain database.
+
+This function is used in the from clause.
+
+#### syntax
+
+`mv_infos("database"="")`
+
+mv_infos() Table structure:
+```sql
+mysql> desc function mv_infos("database"="tpch100");
++--------------------+---------+------+-------+---------+-------+
+| Field              | Type    | Null | Key   | Default | Extra |
++--------------------+---------+------+-------+---------+-------+
+| Id                 | BIGINT  | No   | false | NULL    | NONE  |
+| Name               | TEXT    | No   | false | NULL    | NONE  |
+| JobName            | TEXT    | No   | false | NULL    | NONE  |
+| State              | TEXT    | No   | false | NULL    | NONE  |
+| SchemaChangeDetail | TEXT    | No   | false | NULL    | NONE  |
+| RefreshState       | TEXT    | No   | false | NULL    | NONE  |
+| RefreshInfo        | TEXT    | No   | false | NULL    | NONE  |
+| QuerySql           | TEXT    | No   | false | NULL    | NONE  |
+| EnvInfo            | TEXT    | No   | false | NULL    | NONE  |
+| MvProperties       | TEXT    | No   | false | NULL    | NONE  |
+| MvPartitionInfo    | TEXT    | No   | false | NULL    | NONE  |
+| SyncWithBaseTables | BOOLEAN | No   | false | NULL    | NONE  |
++--------------------+---------+------+-------+---------+-------+
+12 rows in set (0.01 sec)
+```
+
+* Id: Materialized View ID
+* Name: Materialized View Name
+* JobName: The job name corresponding to the materialized view
+* State: Materialized View State
+* SchemaChangeDetail: The reason why the materialized view State becomes a SchemeChange
+* RefreshState: Materialized view refresh status
+* RefreshInfo: Refreshing strategy information defined by materialized views
+* QuerySql: Query statements defined by materialized views
+* EnvInfo: Environmental information during the creation of materialized views
+* MvProperties: Materialized visual attributes
+* MvPartitionInfo: Partition information of materialized views
+* SyncWithBaseTables：Is it synchronized with the base table data? To see which partition is not synchronized, please use [SHOW PARTITIONS](../sql-reference/Show-Statements/SHOW-PARTITIONS.md)
+
+### example
+
+1. View all materialized views under db1
+
+```sql
+mysql> select * from mv_infos("database"="db1");
+```
+
+2. View the materialized view named mv1 under db1
+
+```sql
+mysql> select * from mv_infos("database"="db1") where Name = "mv1";
+```
+
+3. View the status of the materialized view named mv1 under db1
+
+```sql
+mysql> select State from mv_infos("database"="db1") where Name = "mv1";
+```
+
+### keywords
+
+    mv, infos
diff --git a/docs/en/docs/sql-manual/sql-functions/table-functions/tasks.md b/docs/en/docs/sql-manual/sql-functions/table-functions/tasks.md
new file mode 100644
index 00000000000000..e924cc8d6d90d0
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-functions/table-functions/tasks.md
@@ -0,0 +1,106 @@
+---
+{
+    "title": "TASKS",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `tasks`
+
+### Name
+
+tasks
+
+### description
+
+Table function, generating a temporary task table, which can view task information in a certain task type.
+
+This function is used in the from clause.
+
+#### syntax
+
+`tasks("type"="")`
+
+tasks("type"="mv") Table structure:
+```sql
+mysql> desc function tasks("type"="mv");
++-----------------------+------+------+-------+---------+-------+
+| Field                 | Type | Null | Key   | Default | Extra |
++-----------------------+------+------+-------+---------+-------+
+| TaskId                | TEXT | No   | false | NULL    | NONE  |
+| JobId                 | TEXT | No   | false | NULL    | NONE  |
+| JobName               | TEXT | No   | false | NULL    | NONE  |
+| MvId                  | TEXT | No   | false | NULL    | NONE  |
+| MvName                | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseId          | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseName        | TEXT | No   | false | NULL    | NONE  |
+| Status                | TEXT | No   | false | NULL    | NONE  |
+| ErrorMsg              | TEXT | No   | false | NULL    | NONE  |
+| CreateTime            | TEXT | No   | false | NULL    | NONE  |
+| StartTime             | TEXT | No   | false | NULL    | NONE  |
+| FinishTime            | TEXT | No   | false | NULL    | NONE  |
+| DurationMs            | TEXT | No   | false | NULL    | NONE  |
+| TaskContext           | TEXT | No   | false | NULL    | NONE  |
+| RefreshMode           | TEXT | No   | false | NULL    | NONE  |
+| NeedRefreshPartitions | TEXT | No   | false | NULL    | NONE  |
+| CompletedPartitions   | TEXT | No   | false | NULL    | NONE  |
+| Progress              | TEXT | No   | false | NULL    | NONE  |
++-----------------------+------+------+-------+---------+-------+
+18 rows in set (0.00 sec)
+```
+
+* TaskId: task id
+* JobId: job id
+* JobName: job Name
+* MvId: Materialized View ID
+* MvName: Materialized View Name
+* MvDatabaseId: DB ID of the materialized view
+* MvDatabaseName: Name of the database to which the materialized view belongs
+* Status: task status
+* ErrorMsg: Task failure information
+* CreateTime: Task creation time
+* StartTime: Task start running time
+* FinishTime: Task End Run Time
+* DurationMs: Task runtime
+* TaskContext: Task running parameters
+* RefreshMode: refresh mode
+* NeedRefreshPartitions: The partition information that needs to be refreshed for this task
+* CompletedPartitions: The partition information that has been refreshed for this task
+* Progress: Task running progress
+
+### example
+
+1. View tasks for all materialized views
+
+```sql
+mysql> select * from tasks("type"="mv");
+```
+
+2. View all tasks with jobName `inner_mtmv_75043`
+
+```sql
+mysql> select * from tasks("type"="mv") where JobName="inner_mtmv_75043";
+```
+
+### keywords
+
+    tasks
diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..11f24aed95fa2a
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md
@@ -0,0 +1,75 @@
+---
+{
+    "title": "ALTER-ASYNC-MATERIALIZED-VIEW",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## ALTER-ASYNC-MATERIALIZED-VIEW
+
+### Name
+
+ALTER ASYNC MATERIALIZED VIEW
+
+### Description
+
+This statement is used to modify asynchronous materialized views.
+
+#### syntax
+
+```sql
+ALTER MATERIALIZED VIEW mvName=multipartIdentifier ((RENAME newName=identifier)
+       | (REFRESH (refreshMethod | refreshTrigger | refreshMethod refreshTrigger))
+       | (SET  LEFT_PAREN fileProperties=propertyItemList RIGHT_PAREN))
+```
+
+#### illustrate
+
+##### RENAME
+
+Used to change the name of the materialized view
+
+For example, changing the name of mv1 to mv2
+```sql
+ALTER MATERIALIZED VIEW mv1 rename mv2;
+```
+
+##### refreshMethod
+
+Same as [creating asynchronous materialized views](../Create/CREATE-ASYNC-MATERIALIZED-VIEW.md)
+
+##### refreshTrigger
+
+Same as [creating asynchronous materialized views](../Create/CREATE-ASYNC-MATERIALIZED-VIEW.md)
+
+##### SET
+Modify properties unique to materialized views
+
+For example, modifying the grace_period of mv1 to 3000ms
+```sql
+ALTER MATERIALIZED VIEW mv1 set("grace_period"="3000");
+```
+
+### Keywords
+
+    ALTER, ASYNC, MATERIALIZED, VIEW
+
diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..31450326d7a59d
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md
@@ -0,0 +1,216 @@
+---
+{
+    "title": "CREATE-ASYNC-MATERIALIZED-VIEW",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## CREATE-ASYNC-MATERIALIZED-VIEW
+
+### Name
+
+CREATE ASYNC MATERIALIZED VIEW
+
+### Description
+
+This statement is used to create an asynchronous materialized view.
+
+#### syntax
+
+```sql
+CREATE MATERIALIZED VIEW (IF NOT EXISTS)? mvName=multipartIdentifier
+        (LEFT_PAREN cols=simpleColumnDefs RIGHT_PAREN)? buildMode?
+        (REFRESH refreshMethod? refreshTrigger?)?
+        (KEY keys=identifierList)?
+        (COMMENT STRING_LITERAL)?
+        (PARTITION BY LEFT_PAREN partitionKey = identifier RIGHT_PAREN)?
+        (DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM) (BUCKETS (INTEGER_VALUE | AUTO))?)?
+        propertyClause?
+        AS query
+```
+
+#### illustrate
+
+##### simpleColumnDefs
+
+Used to define the materialized view column information, if not defined, it will be automatically derived
+
+```sql
+simpleColumnDefs
+: cols+=simpleColumnDef (COMMA cols+=simpleColumnDef)*
+    ;
+
+simpleColumnDef
+: colName=identifier (COMMENT comment=STRING_LITERAL)?
+    ;
+```
+
+For example, define two columns aa and bb, where the annotation for aa is "name"
+```sql
+CREATE MATERIALIZED VIEW mv1
+(aa comment "name",bb)
+```
+
+##### buildMode
+
+Used to define whether the materialized view is refreshed immediately after creation, default to IMMEDIATE
+
+IMMEDIATE：Refresh Now
+
+DEFERRED：Delay refresh
+
+```sql
+buildMode
+: BUILD (IMMEDIATE | DEFERRED)
+;
+```
+
+For example, specifying the materialized view to refresh immediately
+
+```sql
+CREATE MATERIALIZED VIEW mv1
+BUILD IMMEDIATE
+```
+
+##### refreshMethod
+
+Used to define the refresh method for materialized views, default to AUTO
+
+COMPLETE：Full refresh
+
+AUTO：Try to refresh incrementally as much as possible. If incremental refresh is not possible, refresh in full
+
+```sql
+refreshMethod
+: COMPLETE | AUTO
+;
+```
+
+For example, specifying full refresh of materialized views
+```sql
+CREATE MATERIALIZED VIEW mv1
+REFRESH COMPLETE
+```
+
+##### refreshTrigger
+
+Trigger method for refreshing data in materialized views, default to MANUAL
+
+MANUAL：Manual refresh
+
+SCHEDULE：Timed refresh
+
+```sql
+refreshTrigger
+: ON MANUAL
+| ON SCHEDULE refreshSchedule
+;
+    
+refreshSchedule
+: EVERY INTEGER_VALUE mvRefreshUnit (STARTS STRING_LITERAL)?
+;
+    
+mvRefreshUnit
+: MINUTE | HOUR | DAY | WEEK
+;    
+```
+
+For example: executed every 2 hours, starting from 21:07:09 on December 13, 2023
+```sql
+CREATE MATERIALIZED VIEW mv1
+REFRESH ON SCHEDULE EVERY 2 HOUR STARTS "2023-12-13 21:07:09"
+```
+
+##### key
+The materialized view is the DUPLICATE KEY model, therefore the specified columns are arranged in sequence
+
+```sql
+identifierList
+: LEFT_PAREN identifierSeq RIGHT_PAREN
+    ;
+
+identifierSeq
+: ident+=errorCapturingIdentifier (COMMA ident+=errorCapturingIdentifier)*
+;
+```
+
+For example, specifying k1 and k2 as sorting sequences
+```sql
+CREATE MATERIALIZED VIEW mv1
+KEY(k1,k2)
+```
+
+##### partition
+There are two partition methods for materialized views. If no partition is specified, there is only one partition by default. If a partition field is specified, 
+it will automatically deduce which base table the field comes from and synchronize all partitions of the base table (constraint: the base table can only have one partition field and cannot allow null values)
+
+For example, if the base table is a range partition with a partition field of `create_time` and partitioning by day, and `partition by(ct) as select create_time as ct from t1` is specified when creating a materialized view, 
+then the materialized view will also be a range partition with a partition field of 'ct' and partitioning by day
+
+#### property
+The materialized view can specify both the properties of the table and the properties unique to the materialized view.
+
+The properties unique to materialized views include:
+
+`grace_period`: Maximum delay time allowed for materialized view data during query rewriting
+
+`excluded_trigger_tables`: Table names ignored during data refresh, separated by commas. For example, ` table1, table2`
+
+`refresh_partition_num`: The number of partitions refreshed in a single insert statement, defaults to 1
+
+##### query
+
+Create a query statement for the materialized view, and the result is the data in the materialized view
+
+Random functions are not supported, for example:
+```sql
+SELECT random() as dd,k3 FROM user
+```
+
+### Example
+
+1. Create a materialized view mv1 that refreshes immediately and then once a week, with the data source being the hive catalog
+
+   ```sql
+   CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 1 WEEK
+    DISTRIBUTED BY RANDOM BUCKETS 2
+    PROPERTIES (
+    "replication_num" = "1"
+    )
+    AS SELECT * FROM hive_catalog.db1.user;
+   ```
+
+2. Create a materialized view with multiple table joins
+
+   ```sql
+   CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 1 WEEK
+    DISTRIBUTED BY RANDOM BUCKETS 2
+    PROPERTIES (
+    "replication_num" = "1"
+    )
+    AS select user.k1,user.k3,com.k4 from user join com on user.k1=com.k1;
+   ```
+
+### Keywords
+
+    CREATE, ASYNC, MATERIALIZED, VIEW
+
diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..a8bf6abafd1d1a
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md
@@ -0,0 +1,68 @@
+---
+{
+    "title": "DROP-ASYNC-MATERIALIZED-VIEW",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## DROP-ASYNC-MATERIALIZED-VIEW
+
+### Name
+
+DROP ASYNC MATERIALIZED VIEW
+
+### Description
+
+This statement is used to delete asynchronous materialized views.
+
+syntax:
+
+```sql
+DROP MATERIALIZED VIEW (IF EXISTS)? mvName=multipartIdentifier
+```
+
+
+1. IF EXISTS:
+   If the materialized view does not exist, do not throw an error. If this keyword is not declared and the materialized view does not exist, an error will be reported.
+
+2. mv_name:
+   The name of the materialized view to be deleted. Required field.
+
+### Example
+
+1. Delete table materialized view mv1
+
+```sql
+DROP MATERIALIZED VIEW mv1;
+```
+2.If present, delete the materialized view of the specified database
+
+```sql
+DROP MATERIALIZED VIEW IF EXISTS db1.mv1;
+```
+
+### Keywords
+
+    DROP, ASYNC, MATERIALIZED, VIEW
+
+### Best Practice
+
diff --git a/docs/en/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md
new file mode 100644
index 00000000000000..9b779fe9271bb3
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md
@@ -0,0 +1,56 @@
+---
+{
+    "title": "CANCEL-MATERIALIZED-VIEW-TASK",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## CANCEL-MATERIALIZED-VIEW-TASK
+
+### Name
+
+CANCEL MATERIALIZED VIEW TASK
+
+### Description
+
+This statement is used to cancel the task of materialized views
+
+syntax:
+
+```sql
+CANCEL MATERIALIZED VIEW TASK taskId=INTEGER_VALUE ON mvName=multipartIdentifier
+```
+
+### Example
+
+1. Cancel the task with id 1 in materialized view mv1
+
+    ```sql
+    CANCEL MATERIALIZED VIEW TASK 1 on mv1;
+    ```
+   
+### Keywords
+
+    CANCEL, MATERIALIZED, VIEW, TASK
+
+### Best Practice
+
diff --git a/docs/en/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..7eec77abe38ac6
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md
@@ -0,0 +1,56 @@
+---
+{
+    "title": "PAUSE-MATERIALIZED-VIEW",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## PAUSE-MATERIALIZED-VIEW
+
+### Name
+
+PAUSE MATERIALIZED VIEW
+
+### Description
+
+This statement is used to pause the scheduled scheduling of materialized views
+
+syntax:
+
+```sql
+PAUSE MATERIALIZED VIEW JOB ON mvName=multipartIdentifier
+```
+
+### Example
+
+1. Pause scheduled scheduling of materialized view mv1
+
+    ```sql
+    PAUSE MATERIALIZED VIEW mv1;
+    ```
+   
+### Keywords
+
+    PAUSE, MATERIALIZED, VIEW
+
+### Best Practice
+
diff --git a/docs/en/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..e51424e90a8529
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md
@@ -0,0 +1,72 @@
+---
+{
+    "title": "REFRESH-MATERIALIZED-VIEW",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## REFRESH-MATERIALIZED-VIEW
+
+### Name
+
+REFRESH MATERIALIZED VIEW
+
+### Description
+
+This statement is used to manually refresh the specified asynchronous materialized view
+
+syntax:
+
+```sql
+REFRESH MATERIALIZED VIEW mvName=multipartIdentifier (partitionSpec | COMPLETE)? 
+```
+
+Explanation:
+
+Asynchronous refresh of data for a materialized view
+
+### Example
+
+1. Refresh materialized view mv1 (automatically calculate the partition to be refreshed)
+
+    ```sql
+    REFRESH MATERIALIZED VIEW mv1;
+    ```
+
+2. Refresh partition named p_19950801_19950901和p_19950901_19951001
+
+    ```sql
+    REFRESH MATERIALIZED VIEW mv1 partitions(p_19950801_19950901,p_19950901_19951001);
+    ```
+ 
+3. Force refresh of all materialized view data
+
+    ```sql
+    REFRESH MATERIALIZED VIEW mv1 complete;
+    ```
+   
+### Keywords
+
+    REFRESH, MATERIALIZED, VIEW
+
+### Best Practice
+
diff --git a/docs/en/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..2f89fb2288de66
--- /dev/null
+++ b/docs/en/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md
@@ -0,0 +1,56 @@
+---
+{
+    "title": "RESUME-MATERIALIZED-VIEW",
+    "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## RESUME-MATERIALIZED-VIEW
+
+### Name
+
+RESUME MATERIALIZED VIEW
+
+### Description
+
+This statement is used to temporarily restore the scheduled scheduling of materialized views
+
+syntax:
+
+```sql
+RESUME MATERIALIZED VIEW JOB ON mvName=multipartIdentifier
+```
+
+### Example
+
+1. Timed scheduling for restoring materialized view mv1
+
+    ```sql
+    RESUME MATERIALIZED VIEW mv1;
+    ```
+   
+### Keywords
+
+    RESUME, MATERIALIZED, VIEW
+
+### Best Practice
+
diff --git a/docs/zh-CN/docs/query-acceleration/async-materialized-view.md b/docs/zh-CN/docs/query-acceleration/async-materialized-view.md
new file mode 100644
index 00000000000000..15a222813c54b4
--- /dev/null
+++ b/docs/zh-CN/docs/query-acceleration/async-materialized-view.md
@@ -0,0 +1,216 @@
+---
+{
+    "title": "异步物化视图",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# 异步物化视图
+
+## 物化视图的构建和维护
+
+### 创建物化视图
+
+准备两张表和数据
+```sql
+use tpch;
+
+CREATE TABLE IF NOT EXISTS orders  (
+    o_orderkey       integer not null,
+    o_custkey        integer not null,
+    o_orderstatus    char(1) not null,
+    o_totalprice     decimalv3(15,2) not null,
+    o_orderdate      date not null,
+    o_orderpriority  char(15) not null,
+    o_clerk          char(15) not null,
+    o_shippriority   integer not null,
+    o_comment        varchar(79) not null
+    )
+    DUPLICATE KEY(o_orderkey, o_custkey)
+    PARTITION BY RANGE(o_orderdate)(
+    FROM ('2023-10-17') TO ('2023-10-20') INTERVAL 1 DAY)
+    DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3
+    PROPERTIES ("replication_num" = "1");
+
+insert into orders values
+   (1, 1, 'ok', 99.5, '2023-10-17', 'a', 'b', 1, 'yy'),
+   (2, 2, 'ok', 109.2, '2023-10-18', 'c','d',2, 'mm'),
+   (3, 3, 'ok', 99.5, '2023-10-19', 'a', 'b', 1, 'yy');
+
+CREATE TABLE IF NOT EXISTS lineitem (
+    l_orderkey    integer not null,
+    l_partkey     integer not null,
+    l_suppkey     integer not null,
+    l_linenumber  integer not null,
+    l_quantity    decimalv3(15,2) not null,
+    l_extendedprice  decimalv3(15,2) not null,
+    l_discount    decimalv3(15,2) not null,
+    l_tax         decimalv3(15,2) not null,
+    l_returnflag  char(1) not null,
+    l_linestatus  char(1) not null,
+    l_shipdate    date not null,
+    l_commitdate  date not null,
+    l_receiptdate date not null,
+    l_shipinstruct char(25) not null,
+    l_shipmode     char(10) not null,
+    l_comment      varchar(44) not null
+    )
+    DUPLICATE KEY(l_orderkey, l_partkey, l_suppkey, l_linenumber)
+    PARTITION BY RANGE(l_shipdate)
+    (FROM ('2023-10-17') TO ('2023-10-20') INTERVAL 1 DAY)
+    DISTRIBUTED BY HASH(l_orderkey) BUCKETS 3
+    PROPERTIES ("replication_num" = "1");
+
+insert into lineitem values
+ (1, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'),
+ (2, 2, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'),
+ (3, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx');
+```
+创建物化视图
+```sql
+CREATE MATERIALIZED VIEW mv1 
+        BUILD DEFERRED REFRESH AUTO ON MANUAL
+        partition by(l_shipdate)
+        DISTRIBUTED BY RANDOM BUCKETS 2
+        PROPERTIES ('replication_num' = '1') 
+        AS 
+        select l_shipdate, o_orderdate, l_partkey, l_suppkey, sum(o_totalprice) as sum_total
+            from lineitem
+            left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate
+            group by
+            l_shipdate,
+            o_orderdate,
+            l_partkey,
+            l_suppkey;
+```
+
+具体的语法可查看[CREATE MATERIALIZED VIEW](../sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md)
+
+### 查看物化视图元信息
+
+```sql
+select * from mv_infos("database"="tpch") where Name="mv1";
+```
+
+物化视图独有的特性可以通过[mv_infos()](../sql-manual/sql-functions/table-functions/mv_infos.md)查看
+
+和table相关的属性，仍通过[SHOW TABLES](../sql-manual/sql-reference/Show-Statements/SHOW-TABLES.md)来查看
+
+### 刷新物化视图
+
+物化视图支持不同刷新策略，如定时刷新和手动刷新。也支持不同的刷新粒度，如全量刷新，分区粒度的增量刷新等。这里我们以手动刷新物化视图的部分分区为例。
+
+首先查看物化视图分区列表
+```sql
+SHOW PARTITIONS FROM mv1;
+```
+
+刷新名字为`p_20231017_20231018`的分区
+```sql
+REFRESH MATERIALIZED VIEW mv1 partitions(p_20231017_20231018);
+```
+
+具体的语法可查看[REFRESH MATERIALIZED VIEW](../sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md)
+
+### 任务管理
+
+每个物化视图都会默认有一个job负责刷新数据，job用来描述物化视图的刷新策略等信息，每次触发刷新，都会产生一个task，
+task用来描述具体的一次刷新信息，例如刷新用的时间，刷新了哪些分区等
+
+#### 查看物化视图的job
+
+```sql
+select * from jobs("type"="mv") order by CreateTime;
+```
+
+具体的语法可查看[jobs("type"="mv")](../sql-manual/sql-functions/table-functions/jobs.md)
+
+#### 暂停物化视图job定时调度
+
+```sql
+PAUSE MATERIALIZED VIEW JOB ON mv1;
+```
+
+可以暂停物化视图的定时调度
+
+具体的语法可查看[PAUSE MATERIALIZED VIEW JOB](../sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md)
+
+#### 恢复物化视图job定时调度
+
+```sql
+RESUME MATERIALIZED VIEW JOB ON mv1;
+```
+
+可以恢复物化视图的定时调度
+
+具体的语法可查看[RESUME MATERIALIZED VIEW JOB](../sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md)
+
+#### 查看物化视图的task
+
+```sql
+select * from tasks("type"="mv");
+```
+
+具体的语法可查看[tasks("type"="mv")](../sql-manual/sql-functions/table-functions/tasks.md)
+
+#### 取消物化视图的task
+
+```sql
+CANCEL MATERIALIZED VIEW TASK realTaskId on mv1;
+```
+
+可以取消本次task的运行
+
+具体的语法可查看[CANCEL MATERIALIZED VIEW TASK](../sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md)
+
+### 修改物化视图
+
+修改物化视图的属性
+```sql
+ALTER MATERIALIZED VIEW mv1 set("grace_period"="3333");
+```
+
+修改物化视图的名字，物化视图的刷新方式及物化视图特有的property可通过[ALTER MATERIALIZED VIEW](../sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md)来修改
+
+物化视图本身也是一个 Table，所以 Table 相关的属性，例如副本数，仍通过`ALTER TABLE`相关的语法来修改。
+
+### 删除物化视图
+
+```sql
+DROP MATERIALIZED VIEW mv1;
+```
+
+物化视图有专门的删除语法，不能通过drop table来删除，
+
+具体的语法可查看[DROP MATERIALIZED VIEW](../sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md)
+
+## 物化视图的使用
+
+### 直接查看物化视图的数据
+
+物化视图本身也是一个 Table，所以可以直接查询
+
+```sql
+select * FROM mv1;
+```
+
+### 透明改写
\ No newline at end of file
diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/jobs.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/jobs.md
new file mode 100644
index 00000000000000..cbbe080703e8d7
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/jobs.md
@@ -0,0 +1,90 @@
+---
+{
+    "title": "JOBS",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `jobs`
+
+### Name
+
+jobs
+
+### description
+
+表函数，生成任务临时表，可以查看某个任务类型中的job信息。
+
+该函数用于 from 子句中。
+
+#### syntax
+
+`jobs("type"="")`
+
+jobs("type"="mv")表结构：
+```sql
+mysql> desc function jobs("type"="mv");
++-------------------+------+------+-------+---------+-------+
+| Field             | Type | Null | Key   | Default | Extra |
++-------------------+------+------+-------+---------+-------+
+| Id                | TEXT | No   | false | NULL    | NONE  |
+| Name              | TEXT | No   | false | NULL    | NONE  |
+| MvId              | TEXT | No   | false | NULL    | NONE  |
+| MvName            | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseId      | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseName    | TEXT | No   | false | NULL    | NONE  |
+| ExecuteType       | TEXT | No   | false | NULL    | NONE  |
+| RecurringStrategy | TEXT | No   | false | NULL    | NONE  |
+| Status            | TEXT | No   | false | NULL    | NONE  |
+| CreateTime        | TEXT | No   | false | NULL    | NONE  |
++-------------------+------+------+-------+---------+-------+
+10 rows in set (0.00 sec)
+```
+
+* Id：job id.
+* Name：job名称.
+* MvId：物化视图id
+* MvName：物化视图名称
+* MvDatabaseId：物化视图所属db id
+* MvDatabaseName：物化视图所属db名称
+* ExecuteType：执行类型
+* RecurringStrategy：循环策略
+* Status：job状态
+* CreateTime：task创建时间
+
+### example
+
+1. 查看所有物化视图的job
+
+```sql
+mysql> select * from jobs("type"="mv");
+```
+
+2. 查看name为`inner_mtmv_75043`的job
+
+```sql
+mysql> select * from jobs("type"="mv") where Name="inner_mtmv_75043";
+```
+
+### keywords
+
+    jobs
diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/mv_infos.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/mv_infos.md
new file mode 100644
index 00000000000000..ee9064ded0d3a0
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/mv_infos.md
@@ -0,0 +1,100 @@
+---
+{
+    "title": "MV_INFOS",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `mv_infos`
+
+### Name
+
+mv_infos
+
+### description
+
+表函数，生成异步物化视图临时表，可以查看某个db中创建的异步物化视图信息。
+
+该函数用于 from 子句中。
+
+#### syntax
+
+`mv_infos("database"="")`
+
+mv_infos()表结构：
+```sql
+mysql> desc function mv_infos("database"="tpch100");
++--------------------+---------+------+-------+---------+-------+
+| Field              | Type    | Null | Key   | Default | Extra |
++--------------------+---------+------+-------+---------+-------+
+| Id                 | BIGINT  | No   | false | NULL    | NONE  |
+| Name               | TEXT    | No   | false | NULL    | NONE  |
+| JobName            | TEXT    | No   | false | NULL    | NONE  |
+| State              | TEXT    | No   | false | NULL    | NONE  |
+| SchemaChangeDetail | TEXT    | No   | false | NULL    | NONE  |
+| RefreshState       | TEXT    | No   | false | NULL    | NONE  |
+| RefreshInfo        | TEXT    | No   | false | NULL    | NONE  |
+| QuerySql           | TEXT    | No   | false | NULL    | NONE  |
+| EnvInfo            | TEXT    | No   | false | NULL    | NONE  |
+| MvProperties       | TEXT    | No   | false | NULL    | NONE  |
+| MvPartitionInfo    | TEXT    | No   | false | NULL    | NONE  |
+| SyncWithBaseTables | BOOLEAN | No   | false | NULL    | NONE  |
++--------------------+---------+------+-------+---------+-------+
+12 rows in set (0.01 sec)
+```
+
+* Id：物化视图id
+* Name：物化视图Name
+* JobName：物化视图对应的job名称
+* State：物化视图状态
+* SchemaChangeDetail：物化视图State变为SchemaChange的原因
+* RefreshState：物化视图刷新状态
+* RefreshInfo：物化视图定义的刷新策略信息
+* QuerySql：物化视图定义的查询语句
+* EnvInfo：物化视图创建时的环境信息
+* MvProperties：物化视属性
+* MvPartitionInfo：物化视图的分区信息
+* SyncWithBaseTables：是否和base表数据同步，如需查看哪个分区不同步，请使用[SHOW PARTITIONS](../sql-reference/Show-Statements/SHOW-PARTITIONS.md)
+
+### example
+
+1. 查看db1下的所有物化视图
+
+```sql
+mysql> select * from mv_infos("database"="db1");
+```
+
+2. 查看db1下的物化视图名称为mv1的物化视图
+
+```sql
+mysql> select * from mv_infos("database"="db1") where Name = "mv1";
+```
+
+3. 查看db1下的物化视图名称为mv1的状态
+
+```sql
+mysql> select State from mv_infos("database"="db1") where Name = "mv1";
+```
+
+### keywords
+
+    mv, infos
diff --git a/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/tasks.md b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/tasks.md
new file mode 100644
index 00000000000000..0a4ee24bbb6d24
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-functions/table-functions/tasks.md
@@ -0,0 +1,106 @@
+---
+{
+    "title": "TASKS",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## `tasks`
+
+### Name
+
+tasks
+
+### description
+
+表函数，生成任务临时表，可以查看某个任务类型中的task信息。
+
+该函数用于 from 子句中。
+
+#### syntax
+
+`tasks("type"="")`
+
+tasks("type"="mv")表结构：
+```sql
+mysql> desc function tasks("type"="mv");
++-----------------------+------+------+-------+---------+-------+
+| Field                 | Type | Null | Key   | Default | Extra |
++-----------------------+------+------+-------+---------+-------+
+| TaskId                | TEXT | No   | false | NULL    | NONE  |
+| JobId                 | TEXT | No   | false | NULL    | NONE  |
+| JobName               | TEXT | No   | false | NULL    | NONE  |
+| MvId                  | TEXT | No   | false | NULL    | NONE  |
+| MvName                | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseId          | TEXT | No   | false | NULL    | NONE  |
+| MvDatabaseName        | TEXT | No   | false | NULL    | NONE  |
+| Status                | TEXT | No   | false | NULL    | NONE  |
+| ErrorMsg              | TEXT | No   | false | NULL    | NONE  |
+| CreateTime            | TEXT | No   | false | NULL    | NONE  |
+| StartTime             | TEXT | No   | false | NULL    | NONE  |
+| FinishTime            | TEXT | No   | false | NULL    | NONE  |
+| DurationMs            | TEXT | No   | false | NULL    | NONE  |
+| TaskContext           | TEXT | No   | false | NULL    | NONE  |
+| RefreshMode           | TEXT | No   | false | NULL    | NONE  |
+| NeedRefreshPartitions | TEXT | No   | false | NULL    | NONE  |
+| CompletedPartitions   | TEXT | No   | false | NULL    | NONE  |
+| Progress              | TEXT | No   | false | NULL    | NONE  |
++-----------------------+------+------+-------+---------+-------+
+18 rows in set (0.00 sec)
+```
+
+* TaskId：task id
+* JobId：job id
+* JobName：job名称
+* MvId：物化视图id
+* MvName：物化视图名称
+* MvDatabaseId：物化视图所属db id
+* MvDatabaseName：物化视图所属db名称
+* Status：task状态
+* ErrorMsg：task失败信息
+* CreateTime：task创建时间
+* StartTime：task开始运行时间
+* FinishTime：task结束运行时间
+* DurationMs：task运行时间
+* TaskContext：task运行参数
+* RefreshMode：刷新模式
+* NeedRefreshPartitions：本次task需要刷新的分区信息
+* CompletedPartitions：本次task刷新完成的分区信息
+* Progress：task运行进度
+
+### example
+
+1. 查看所有物化视图的task
+
+```sql
+mysql> select * from tasks("type"="mv");
+```
+
+2. 查看jobName为`inner_mtmv_75043`的所有task
+
+```sql
+mysql> select * from tasks("type"="mv") where JobName="inner_mtmv_75043";
+```
+
+### keywords
+
+    tasks
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..0bb39ff32a9085
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Alter/ALTER-ASYNC-MATERIALIZED-VIEW.md
@@ -0,0 +1,75 @@
+---
+{
+    "title": "ALTER-ASYNC-MATERIALIZED-VIEW",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## ALTER-ASYNC-MATERIALIZED-VIEW
+
+### Name
+
+ALTER ASYNC MATERIALIZED VIEW
+
+### Description
+
+该语句用于修改异步物化视图。
+
+#### 语法
+
+```sql
+ALTER MATERIALIZED VIEW mvName=multipartIdentifier ((RENAME newName=identifier)
+       | (REFRESH (refreshMethod | refreshTrigger | refreshMethod refreshTrigger))
+       | (SET  LEFT_PAREN fileProperties=propertyItemList RIGHT_PAREN))
+```
+
+#### 说明
+
+##### RENAME
+
+用来更改物化视图的名字
+
+例如: 将mv1的名字改为mv2
+```sql
+ALTER MATERIALIZED VIEW mv1 rename mv2;
+```
+
+##### refreshMethod
+
+同[创建异步物化视图](../Create/CREATE-ASYNC-MATERIALIZED-VIEW.md)
+
+##### refreshTrigger
+
+同[创建异步物化视图](../Create/CREATE-ASYNC-MATERIALIZED-VIEW.md)
+
+##### SET
+修改物化视图特有的property
+
+例如修改mv1的grace_period为3000ms
+```sql
+ALTER MATERIALIZED VIEW mv1 set("grace_period"="3000");
+```
+
+### Keywords
+
+    ALTER, ASYNC, MATERIALIZED, VIEW
+
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..7be195589a8752
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-ASYNC-MATERIALIZED-VIEW.md
@@ -0,0 +1,215 @@
+---
+{
+    "title": "CREATE-ASYNC-MATERIALIZED-VIEW",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## CREATE-ASYNC-MATERIALIZED-VIEW
+
+### Name
+
+CREATE ASYNC MATERIALIZED VIEW
+
+### Description
+
+该语句用于创建异步物化视图。
+
+#### 语法
+
+```sql
+CREATE MATERIALIZED VIEW (IF NOT EXISTS)? mvName=multipartIdentifier
+        (LEFT_PAREN cols=simpleColumnDefs RIGHT_PAREN)? buildMode?
+        (REFRESH refreshMethod? refreshTrigger?)?
+        (KEY keys=identifierList)?
+        (COMMENT STRING_LITERAL)?
+        (PARTITION BY LEFT_PAREN partitionKey = identifier RIGHT_PAREN)?
+        (DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM) (BUCKETS (INTEGER_VALUE | AUTO))?)?
+        propertyClause?
+        AS query
+```
+
+#### 说明
+
+##### simpleColumnDefs
+
+用来定义物化视图column信息，如果不定义，将自动推导
+
+```sql
+simpleColumnDefs
+: cols+=simpleColumnDef (COMMA cols+=simpleColumnDef)*
+    ;
+
+simpleColumnDef
+: colName=identifier (COMMENT comment=STRING_LITERAL)?
+    ;
+```
+
+例如:定义两列aa和bb,其中aa的注释为"name"
+```sql
+CREATE MATERIALIZED VIEW mv1
+(aa comment "name",bb)
+```
+
+##### buildMode
+
+用来定义物化视图是否创建完成立即刷新,默认IMMEDIATE
+
+IMMEDIATE：立即刷新
+
+DEFERRED：延迟刷新
+
+```sql
+buildMode
+: BUILD (IMMEDIATE | DEFERRED)
+;
+```
+
+例如：指定物化视图立即刷新
+
+```sql
+CREATE MATERIALIZED VIEW mv1
+BUILD IMMEDIATE
+```
+
+##### refreshMethod
+
+用来定义物化视图刷新方式，默认AUTO
+
+COMPLETE：全量刷新
+
+AUTO：尽量增量刷新，如果不能增量刷新，就全量刷新
+
+```sql
+refreshMethod
+: COMPLETE | AUTO
+;
+```
+
+例如：指定物化视图全量刷新
+```sql
+CREATE MATERIALIZED VIEW mv1
+REFRESH COMPLETE
+```
+
+##### refreshTrigger
+
+物化视图刷新数据的触发方式，默认MANUAL
+
+MANUAL：手动刷新
+
+SCHEDULE：定时刷新
+
+```sql
+refreshTrigger
+: ON MANUAL
+| ON SCHEDULE refreshSchedule
+;
+    
+refreshSchedule
+: EVERY INTEGER_VALUE mvRefreshUnit (STARTS STRING_LITERAL)?
+;
+    
+mvRefreshUnit
+: MINUTE | HOUR | DAY | WEEK
+;    
+```
+
+例如：每2小时执行一次，从2023-12-13 21:07:09开始
+```sql
+CREATE MATERIALIZED VIEW mv1
+REFRESH ON SCHEDULE EVERY 2 HOUR STARTS "2023-12-13 21:07:09"
+```
+
+##### key
+物化视图为DUPLICATE KEY模型，因此指定的列为排序列
+
+```sql
+identifierList
+: LEFT_PAREN identifierSeq RIGHT_PAREN
+    ;
+
+identifierSeq
+: ident+=errorCapturingIdentifier (COMMA ident+=errorCapturingIdentifier)*
+;
+```
+
+例如：指定k1,k2为排序列
+```sql
+CREATE MATERIALIZED VIEW mv1
+KEY(k1,k2)
+```
+
+##### partition
+物化视图有两种分区方式，如果不指定分区，默认只有一个分区，如果指定分区字段，会自动推导出字段来自哪个基表并同步基表的所有分区（限制条件：基表只能有一个分区字段且不能允许空值）
+
+例如：基表是range分区，分区字段为`create_time`并按天分区，创建物化视图时指定`partition by(ct) as select create_time as ct from t1`
+那么物化视图也会是range分区，分区字段为`ct`,并且按天分区
+
+#### property
+物化视图既可以指定table的property，也可以指定物化视图特有的property。
+
+物化视图特有的property包括：
+
+`grace_period`：查询改写时允许物化视图数据的最大延迟时间
+
+`excluded_trigger_tables`：数据刷新时忽略的表名，逗号分割。例如`table1,table2`
+
+`refresh_partition_num`：单次insert语句刷新的分区数量，默认为1
+
+##### query
+
+创建物化视图的查询语句，其结果即为物化视图中的数据
+
+不支持随机函数，例如:
+```sql
+SELECT random() as dd,k3 FROM user
+```
+
+### Example
+
+1. 创建一个立即刷新，之后每周刷新一次的物化视图mv1,数据源为hive catalog
+
+   ```sql
+   CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 1 WEEK
+    DISTRIBUTED BY RANDOM BUCKETS 2
+    PROPERTIES (
+    "replication_num" = "1"
+    )
+    AS SELECT * FROM hive_catalog.db1.user;
+   ```
+
+2. 创建一个多表join的物化视图
+
+   ```sql
+   CREATE MATERIALIZED VIEW mv1 BUILD IMMEDIATE REFRESH COMPLETE ON SCHEDULE EVERY 1 WEEK
+    DISTRIBUTED BY RANDOM BUCKETS 2
+    PROPERTIES (
+    "replication_num" = "1"
+    )
+    AS select user.k1,user.k3,com.k4 from user join com on user.k1=com.k1;
+   ```
+
+### Keywords
+
+    CREATE, ASYNC, MATERIALIZED, VIEW
+
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..49694b7309ce77
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Drop/DROP-ASYNC-MATERIALIZED-VIEW.md
@@ -0,0 +1,68 @@
+---
+{
+    "title": "DROP-ASYNC-MATERIALIZED-VIEW",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## DROP-ASYNC-MATERIALIZED-VIEW
+
+### Name
+
+DROP ASYNC MATERIALIZED VIEW
+
+### Description
+
+该语句用于删除异步物化视图。
+
+语法：
+
+```sql
+DROP MATERIALIZED VIEW (IF EXISTS)? mvName=multipartIdentifier
+```
+
+
+1. IF EXISTS:
+        如果物化视图不存在，不要抛出错误。如果不声明此关键字，物化视图不存在则报错。
+
+2. mv_name:
+        待删除的物化视图的名称。必填项。
+
+### Example
+
+1. 删除表物化视图mv1
+
+```sql
+DROP MATERIALIZED VIEW mv1;
+```
+2.如果存在，删除指定 database 的物化视图
+
+```sql
+DROP MATERIALIZED VIEW IF EXISTS db1.mv1;
+```
+
+### Keywords
+
+    DROP, ASYNC, MATERIALIZED, VIEW
+
+### Best Practice
+
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md
new file mode 100644
index 00000000000000..d96b28f7912959
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/CANCEL-MATERIALIZED-VIEW-TASK.md
@@ -0,0 +1,56 @@
+---
+{
+    "title": "CANCEL-MATERIALIZED-VIEW-TASK",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## CANCEL-MATERIALIZED-VIEW-TASK
+
+### Name
+
+CANCEL MATERIALIZED VIEW TASK
+
+### Description
+
+该语句用于取消物化视图的task
+
+语法：
+
+```sql
+CANCEL MATERIALIZED VIEW TASK taskId=INTEGER_VALUE ON mvName=multipartIdentifier
+```
+
+### Example
+
+1. 取消物化视图mv1的id为1的task
+
+    ```sql
+    CANCEL MATERIALIZED VIEW TASK 1 on mv1;
+    ```
+   
+### Keywords
+
+    CANCEL, MATERIALIZED, VIEW, TASK
+
+### Best Practice
+
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..ae5bd7bd803aac
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/PAUSE-MATERIALIZED-VIEW.md
@@ -0,0 +1,56 @@
+---
+{
+    "title": "PAUSE-MATERIALIZED-VIEW",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## PAUSE-MATERIALIZED-VIEW
+
+### Name
+
+PAUSE MATERIALIZED VIEW
+
+### Description
+
+该语句用于暂停物化视图的定时调度
+
+语法：
+
+```sql
+PAUSE MATERIALIZED VIEW JOB ON mvName=multipartIdentifier
+```
+
+### Example
+
+1. 暂停物化视图mv1的定时调度
+
+    ```sql
+    PAUSE MATERIALIZED VIEW mv1;
+    ```
+   
+### Keywords
+
+    PAUSE, MATERIALIZED, VIEW
+
+### Best Practice
+
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..b719d4aefe6ffd
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/REFRESH-MATERIALIZED-VIEW.md
@@ -0,0 +1,72 @@
+---
+{
+    "title": "REFRESH-MATERIALIZED-VIEW",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## REFRESH-MATERIALIZED-VIEW
+
+### Name
+
+REFRESH MATERIALIZED VIEW
+
+### Description
+
+该语句用于手动刷新指定的异步物化视图
+
+语法：
+
+```sql
+REFRESH MATERIALIZED VIEW mvName=multipartIdentifier (partitionSpec | COMPLETE)? 
+```
+
+说明：
+
+异步刷新某个物化视图的数据
+
+### Example
+
+1. 刷新物化视图mv1(自动计算要刷新的分区)
+
+    ```sql
+    REFRESH MATERIALIZED VIEW mv1;
+    ```
+
+2. 刷新名字为p_19950801_19950901和p_19950901_19951001的分区
+
+    ```sql
+    REFRESH MATERIALIZED VIEW mv1 partitions(p_19950801_19950901,p_19950901_19951001);
+    ```
+ 
+3. 强制刷新物化视图全部数据
+
+    ```sql
+    REFRESH MATERIALIZED VIEW mv1 complete;
+    ```
+   
+### Keywords
+
+    REFRESH, MATERIALIZED, VIEW
+
+### Best Practice
+
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md
new file mode 100644
index 00000000000000..2d772578b67deb
--- /dev/null
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Utility-Statements/RESUME-MATERIALIZED-VIEW.md
@@ -0,0 +1,56 @@
+---
+{
+    "title": "RESUME-MATERIALIZED-VIEW",
+    "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+## RESUME-MATERIALIZED-VIEW
+
+### Name
+
+RESUME MATERIALIZED VIEW
+
+### Description
+
+该语句用于暂恢复物化视图的定时调度
+
+语法：
+
+```sql
+RESUME MATERIALIZED VIEW JOB ON mvName=multipartIdentifier
+```
+
+### Example
+
+1. 恢复物化视图mv1的定时调度
+
+    ```sql
+    RESUME MATERIALIZED VIEW mv1;
+    ```
+   
+### Keywords
+
+    RESUME, MATERIALIZED, VIEW
+
+### Best Practice
+

From 738abac9eda3e39bf4ac5c2f4996b3a6a67b2919 Mon Sep 17 00:00:00 2001
From: yiguolei <676222867@qq.com>
Date: Mon, 1 Jan 2024 22:08:23 +0800
Subject: [PATCH 235/299] [minor](context) duplicate query context in fragment
 ctx (#29364)

---------

Co-authored-by: yiguolei <yiguolei@gmail.com>
---
 be/src/pipeline/pipeline_fragment_context.h | 4 +---
 be/src/pipeline/pipeline_task.cpp           | 2 +-
 be/src/pipeline/task_scheduler.cpp          | 4 ++--
 be/src/runtime/exec_env_init.cpp            | 7 ++++---
 be/src/runtime/fragment_mgr.cpp             | 4 ++--
 be/src/vec/exec/scan/new_olap_scanner.cpp   | 1 -
 6 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/be/src/pipeline/pipeline_fragment_context.h b/be/src/pipeline/pipeline_fragment_context.h
index 5009bde9b20e48..2a3a11d59cc107 100644
--- a/be/src/pipeline/pipeline_fragment_context.h
+++ b/be/src/pipeline/pipeline_fragment_context.h
@@ -83,7 +83,7 @@ class PipelineFragmentContext : public TaskExecutionContext {
         return _runtime_state->runtime_filter_mgr();
     }
 
-    QueryContext* get_query_ctx() { return _runtime_state->get_query_ctx(); }
+    QueryContext* get_query_ctx() { return _query_ctx.get(); }
     // should be protected by lock?
     [[nodiscard]] bool is_canceled() const { return _runtime_state->is_cancelled(); }
 
@@ -108,8 +108,6 @@ class PipelineFragmentContext : public TaskExecutionContext {
 
     // TODO: Support pipeline runtime filter
 
-    QueryContext* get_query_context() { return _query_ctx.get(); }
-
     TUniqueId get_query_id() const { return _query_id; }
 
     [[nodiscard]] int get_fragment_id() const { return _fragment_id; }
diff --git a/be/src/pipeline/pipeline_task.cpp b/be/src/pipeline/pipeline_task.cpp
index 75694373a4ce47..5f5ff56aa4bfa7 100644
--- a/be/src/pipeline/pipeline_task.cpp
+++ b/be/src/pipeline/pipeline_task.cpp
@@ -357,7 +357,7 @@ Status PipelineTask::close(Status exec_status) {
 }
 
 QueryContext* PipelineTask::query_context() {
-    return _fragment_context->get_query_context();
+    return _fragment_context->get_query_ctx();
 }
 
 // The FSM see PipelineTaskState's comment
diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp
index e0e668acbd9455..e814e4cdf2d092 100644
--- a/be/src/pipeline/task_scheduler.cpp
+++ b/be/src/pipeline/task_scheduler.cpp
@@ -247,7 +247,7 @@ void TaskScheduler::_do_work(size_t index) {
         if (state == PipelineTaskState::PENDING_FINISH) {
             DCHECK(task->is_pipelineX() || !task->is_pending_finish())
                     << "must not pending close " << task->debug_string();
-            Status exec_status = fragment_ctx->get_query_context()->exec_status();
+            Status exec_status = fragment_ctx->get_query_ctx()->exec_status();
             _try_close_task(task,
                             canceled ? PipelineTaskState::CANCELED : PipelineTaskState::FINISHED,
                             exec_status);
@@ -264,7 +264,7 @@ void TaskScheduler::_do_work(size_t index) {
             // If pipeline is canceled, it will report after pipeline closed, and will propagate
             // errors to downstream through exchange. So, here we needn't send_report.
             // fragment_ctx->send_report(true);
-            Status cancel_status = fragment_ctx->get_query_context()->exec_status();
+            Status cancel_status = fragment_ctx->get_query_ctx()->exec_status();
             _try_close_task(task, PipelineTaskState::CANCELED, cancel_status);
             continue;
         }
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 508ea7c7921011..2a1233c9b5d6b1 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -588,9 +588,6 @@ void ExecEnv::destroy() {
     SAFE_DELETE(_scanner_scheduler);
     // _storage_page_cache must be destoried before _cache_manager
     SAFE_DELETE(_storage_page_cache);
-    // cache_manager must be destoried after _inverted_index_query_cache
-    // https://github.com/apache/doris/issues/24082#issuecomment-1712544039
-    SAFE_DELETE(_cache_manager);
 
     SAFE_DELETE(_small_file_mgr);
     SAFE_DELETE(_broker_mgr);
@@ -629,6 +626,10 @@ void ExecEnv::destroy() {
     SAFE_DELETE(_external_scan_context_mgr);
     SAFE_DELETE(_user_function_cache);
 
+    // cache_manager must be destoried after _inverted_index_query_cache
+    // https://github.com/apache/doris/issues/24082#issuecomment-1712544039
+    SAFE_DELETE(_cache_manager);
+
     // _heartbeat_flags must be destoried after staroge engine
     SAFE_DELETE(_heartbeat_flags);
 
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index 9256c3bccfddef..d265b5e35b5203 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -576,7 +576,7 @@ void FragmentMgr::remove_pipeline_context(
         std::shared_ptr<pipeline::PipelineFragmentContext> f_context) {
     std::lock_guard<std::mutex> lock(_lock);
     auto query_id = f_context->get_query_id();
-    auto* q_context = f_context->get_query_context();
+    auto* q_context = f_context->get_query_ctx();
     std::vector<TUniqueId> ins_ids;
     f_context->instance_ids(ins_ids);
     bool all_done = q_context->countdown(ins_ids.size());
@@ -1413,7 +1413,7 @@ Status FragmentMgr::apply_filterv2(const PPublishFilterRequestV2* request,
 
             DCHECK(pip_context != nullptr);
             runtime_filter_mgr = pip_context->get_query_ctx()->runtime_filter_mgr();
-            pool = &pip_context->get_query_context()->obj_pool;
+            pool = &pip_context->get_query_ctx()->obj_pool;
         } else {
             std::unique_lock<std::mutex> lock(_lock);
             auto iter = _fragment_instance_map.find(tfragment_instance_id);
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp b/be/src/vec/exec/scan/new_olap_scanner.cpp
index 5566abec3ab383..bc15cf7207fb80 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -524,7 +524,6 @@ Status NewOlapScanner::close(RuntimeState* state) {
     // so that it will core
     _tablet_reader_params.rs_splits.clear();
     _tablet_reader.reset();
-    LOG(INFO) << "close_tablet_id" << _tablet_reader_params.tablet->tablet_id();
     RETURN_IF_ERROR(VScanner::close(state));
     return Status::OK();
 }

From 4cbbd25d8c9f0a4ef8178fb38f49738c6ec45963 Mon Sep 17 00:00:00 2001
From: Mingyu Chen <morningman@163.com>
Date: Mon, 1 Jan 2024 23:32:28 +0800
Subject: [PATCH 236/299] [fix](ctx) manager the lifecycle of connection
 context (#29346)

In FrontendService, we may create some connection context and set it as a thread local varaible.
These context should be removed from thread local after call.
Otherwise, it may be reused by other thread incorrectly.
---
 be/src/runtime/group_commit_mgr.cpp           |  2 +
 .../external/FederationBackendPolicy.java     | 11 +++--
 .../doris/service/FrontendServiceImpl.java    | 41 ++++++++++---------
 3 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index acb40ae6c78661..00b3a559fbf6ef 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -235,6 +235,8 @@ Status GroupCommitTable::_create_group_commit_load(
     request.__set_token("group_commit"); // this is a fake, fe not check it now
     request.__set_max_filter_ratio(1.0);
     request.__set_strictMode(false);
+    // this is an internal interface, use admin to pass the auth check
+    request.__set_user("admin");
     if (_exec_env->master_info()->__isset.backend_id) {
         request.__set_backend_id(_exec_env->master_info()->backend_id);
     } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
index d1d7e90d35ad16..ade03291c30552 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
@@ -30,6 +30,7 @@
 import org.apache.doris.thrift.TScanRangeLocations;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
@@ -122,9 +123,13 @@ public void init(List<String> preLocations) throws UserException {
         Set<Tag> tags = Sets.newHashSet();
         if (ConnectContext.get() != null && ConnectContext.get().getCurrentUserIdentity() != null) {
             String qualifiedUser = ConnectContext.get().getCurrentUserIdentity().getQualifiedUser();
-            tags = Env.getCurrentEnv().getAuth().getResourceTags(qualifiedUser);
-            if (tags == UserProperty.INVALID_RESOURCE_TAGS) {
-                throw new UserException("No valid resource tag for user: " + qualifiedUser);
+            // Some request from stream load(eg, mysql load) may not set user info in ConnectContext
+            // just ignore it.
+            if (!Strings.isNullOrEmpty(qualifiedUser)) {
+                tags = Env.getCurrentEnv().getAuth().getResourceTags(qualifiedUser);
+                if (tags == UserProperty.INVALID_RESOURCE_TAGS) {
+                    throw new UserException("No valid resource tag for user: " + qualifiedUser);
+                }
             }
         } else {
             LOG.debug("user info in ExternalFileScanNode should not be null, add log to observer");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index e5b0f651016ce9..136084f5f1c012 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -1786,9 +1786,19 @@ public TStreamLoadPutResult streamLoadPut(TStreamLoadPutRequest request) {
         TStreamLoadPutResult result = new TStreamLoadPutResult();
         TStatus status = new TStatus(TStatusCode.OK);
         result.setStatus(status);
+
+        // create connect context
+        ConnectContext ctx = new ConnectContext();
+        ctx.setEnv(Env.getCurrentEnv());
+        ctx.setQueryId(request.getLoadId());
+        ctx.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp(request.getUser(), "%"));
+        ctx.setQualifiedUser(request.getUser());
+        ctx.setBackendId(request.getBackendId());
+        ctx.setThreadLocalInfo();
+
         try {
             if (!Strings.isNullOrEmpty(request.getLoadSql())) {
-                httpStreamPutImpl(request, result);
+                httpStreamPutImpl(request, result, ctx);
                 return result;
             } else {
                 if (Config.enable_pipeline_load) {
@@ -1806,6 +1816,8 @@ public TStreamLoadPutResult streamLoadPut(TStreamLoadPutRequest request) {
             status.setStatusCode(TStatusCode.INTERNAL_ERROR);
             status.addToErrorMsgs(e.getClass().getSimpleName() + ": " + Strings.nullToEmpty(e.getMessage()));
             return result;
+        } finally {
+            ConnectContext.remove();
         }
         return result;
     }
@@ -1917,12 +1929,12 @@ public TStreamLoadMultiTablePutResult streamLoadMultiTablePut(TStreamLoadPutRequ
         return result;
     }
 
-
-    private void httpStreamPutImpl(TStreamLoadPutRequest request, TStreamLoadPutResult result)
+    private void httpStreamPutImpl(TStreamLoadPutRequest request, TStreamLoadPutResult result, ConnectContext ctx)
             throws UserException {
-        LOG.info("receive http stream put request");
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("receive http stream put request: {}", request);
+        }
         String originStmt = request.getLoadSql();
-        ConnectContext ctx = new ConnectContext();
         if (request.isSetAuthCode()) {
             // TODO(cmy): find a way to check
         } else if (Strings.isNullOrEmpty(request.getToken())) {
@@ -1930,12 +1942,6 @@ private void httpStreamPutImpl(TStreamLoadPutRequest request, TStreamLoadPutResu
                     request.getTbl(),
                     request.getUserIp(), PrivPredicate.LOAD);
         }
-        ctx.setEnv(Env.getCurrentEnv());
-        ctx.setQueryId(request.getLoadId());
-        ctx.setCurrentUserIdentity(UserIdentity.ROOT);
-        ctx.setQualifiedUser(UserIdentity.ROOT.getQualifiedUser());
-        ctx.setBackendId(request.getBackendId());
-        ctx.setThreadLocalInfo();
         SqlScanner input = new SqlScanner(new StringReader(originStmt), ctx.getSessionVariable().getSqlMode());
         SqlParser parser = new SqlParser(input);
         try {
@@ -2859,16 +2865,11 @@ private TRestoreSnapshotResult restoreSnapshotImpl(TRestoreSnapshotRequest reque
         restoreStmt.setIsBeingSynced();
         LOG.trace("restore snapshot info, restoreStmt: {}", restoreStmt);
         try {
-            ConnectContext ctx = ConnectContext.get();
-            if (ctx == null) {
-                ctx = new ConnectContext();
-                ctx.setThreadLocalInfo();
-            }
+            ConnectContext ctx = new ConnectContext();
             ctx.setQualifiedUser(request.getUser());
             String fullUserName = ClusterNamespace.getNameFromFullName(request.getUser());
-            UserIdentity currentUserIdentity = new UserIdentity(fullUserName, "%");
-            ctx.setCurrentUserIdentity(currentUserIdentity);
-
+            ctx.setCurrentUserIdentity(UserIdentity.createAnalyzedUserIdentWithIp(fullUserName, "%"));
+            ctx.setThreadLocalInfo();
             Analyzer analyzer = new Analyzer(ctx.getEnv(), ctx);
             restoreStmt.analyze(analyzer);
             DdlExecutor.execute(Env.getCurrentEnv(), restoreStmt);
@@ -2880,6 +2881,8 @@ private TRestoreSnapshotResult restoreSnapshotImpl(TRestoreSnapshotRequest reque
             LOG.warn("catch unknown result.", e);
             status.setStatusCode(TStatusCode.INTERNAL_ERROR);
             status.addToErrorMsgs(Strings.nullToEmpty(e.getMessage()));
+        } finally {
+            ConnectContext.remove();
         }
 
         return result;

From cec69f4cbd3be1d4ce67faa820e3b4190b670a3c Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Tue, 2 Jan 2024 11:02:17 +0800
Subject: [PATCH 237/299] [Enhancement](wal) Add timout for wal memory back
 pressure (#29178)

---
 be/src/common/config.cpp                    |  2 +-
 be/src/common/config.h                      |  2 +-
 be/src/runtime/group_commit_mgr.cpp         | 22 ++++++++++++++++++---
 be/src/runtime/group_commit_mgr.h           |  6 +++++-
 be/src/vec/sink/group_commit_block_sink.cpp | 11 ++++++-----
 be/src/vec/sink/group_commit_block_sink.h   |  2 +-
 6 files changed, 33 insertions(+), 12 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index e94b4c6dadb315..9b30ae145c46bd 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1121,7 +1121,7 @@ DEFINE_Int32(group_commit_insert_threads, "10");
 DEFINE_Int32(group_commit_memory_rows_for_max_filter_ratio, "10000");
 DEFINE_Bool(wait_internal_group_commit_finish, "false");
 // Max size(bytes) of group commit queues, used for mem back pressure, defult 64M.
-DEFINE_Int32(group_commit_max_queue_size, "67108864");
+DEFINE_Int32(group_commit_queue_mem_limit, "67108864");
 // Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
 // group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
 DEFINE_String(group_commit_wal_max_disk_limit, "10%");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 2d257b49510c10..c340abe05f3f58 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1187,7 +1187,7 @@ DECLARE_mInt32(group_commit_insert_threads);
 DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio);
 DECLARE_Bool(wait_internal_group_commit_finish);
 // Max size(bytes) of group commit queues, used for mem back pressure.
-DECLARE_Int32(group_commit_max_queue_size);
+DECLARE_Int32(group_commit_queue_mem_limit);
 // Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
 // group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
 DECLARE_mString(group_commit_wal_max_disk_limit);
diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index 00b3a559fbf6ef..b965efadd70598 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -21,6 +21,7 @@
 #include <glog/logging.h>
 
 #include <atomic>
+#include <chrono>
 #include <cstddef>
 #include <cstdint>
 #include <memory>
@@ -40,14 +41,29 @@
 
 namespace doris {
 
-Status LoadBlockQueue::add_block(std::shared_ptr<vectorized::Block> block, bool write_wal) {
+Status LoadBlockQueue::add_block(RuntimeState* runtime_state,
+                                 std::shared_ptr<vectorized::Block> block, bool write_wal) {
     std::unique_lock l(mutex);
     RETURN_IF_ERROR(status);
-    while (_all_block_queues_bytes->load(std::memory_order_relaxed) >
-           config::group_commit_max_queue_size) {
+    auto start = std::chrono::steady_clock::now();
+    while (!runtime_state->is_cancelled() && status.ok() &&
+           _all_block_queues_bytes->load(std::memory_order_relaxed) >
+                   config::group_commit_queue_mem_limit) {
         _put_cond.wait_for(
                 l, std::chrono::milliseconds(LoadBlockQueue::MAX_BLOCK_QUEUE_ADD_WAIT_TIME));
+        auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
+                std::chrono::steady_clock::now() - start);
+        if (duration.count() > LoadBlockQueue::WAL_MEM_BACK_PRESSURE_TIME_OUT) {
+            return Status::TimedOut(
+                    "Wal memory back pressure wait too much time! Load block queue txn id: {}, "
+                    "label: {}, instance id: {}",
+                    txn_id, label, load_instance_id.to_string());
+        }
     }
+    if (runtime_state->is_cancelled()) {
+        return Status::Cancelled(runtime_state->cancel_reason());
+    }
+    RETURN_IF_ERROR(status);
     if (block->rows() > 0) {
         _block_queue.push_back(block);
         if (write_wal) {
diff --git a/be/src/runtime/group_commit_mgr.h b/be/src/runtime/group_commit_mgr.h
index b0553b448769eb..03c917f8fcc48e 100644
--- a/be/src/runtime/group_commit_mgr.h
+++ b/be/src/runtime/group_commit_mgr.h
@@ -55,7 +55,8 @@ class LoadBlockQueue {
               _all_block_queues_bytes(all_block_queues_bytes),
               _group_commit_interval_ms(group_commit_interval_ms) {};
 
-    Status add_block(std::shared_ptr<vectorized::Block> block, bool write_wal);
+    Status add_block(RuntimeState* runtime_state, std::shared_ptr<vectorized::Block> block,
+                     bool write_wal);
     Status get_block(RuntimeState* runtime_state, vectorized::Block* block, bool* find_block,
                      bool* eos);
     Status add_load_id(const UniqueId& load_id);
@@ -68,7 +69,10 @@ class LoadBlockQueue {
     bool has_enough_wal_disk_space(const std::vector<std::shared_ptr<vectorized::Block>>& blocks,
                                    const TUniqueId& load_id, bool is_blocks_contain_all_load_data);
 
+    // 1s
     static constexpr size_t MAX_BLOCK_QUEUE_ADD_WAIT_TIME = 1000;
+    // 120s
+    static constexpr size_t WAL_MEM_BACK_PRESSURE_TIME_OUT = 120000;
     UniqueId load_instance_id;
     std::string label;
     int64_t txn_id;
diff --git a/be/src/vec/sink/group_commit_block_sink.cpp b/be/src/vec/sink/group_commit_block_sink.cpp
index 56869665878cc2..277b9859bd528e 100644
--- a/be/src/vec/sink/group_commit_block_sink.cpp
+++ b/be/src/vec/sink/group_commit_block_sink.cpp
@@ -114,7 +114,7 @@ Status GroupCommitBlockSink::close(RuntimeState* state, Status close_status) {
             (double)state->num_rows_load_filtered() / num_selected_rows > _max_filter_ratio) {
             return Status::DataQualityError("too many filtered rows");
         }
-        RETURN_IF_ERROR(_add_blocks(true));
+        RETURN_IF_ERROR(_add_blocks(state, true));
     }
     if (_load_block_queue) {
         _load_block_queue->remove_load_id(_load_id);
@@ -220,15 +220,16 @@ Status GroupCommitBlockSink::_add_block(RuntimeState* state,
         _blocks.emplace_back(output_block);
     } else {
         if (!_is_block_appended) {
-            RETURN_IF_ERROR(_add_blocks(false));
+            RETURN_IF_ERROR(_add_blocks(state, false));
         }
         RETURN_IF_ERROR(_load_block_queue->add_block(
-                output_block, _group_commit_mode == TGroupCommitMode::ASYNC_MODE));
+                state, output_block, _group_commit_mode == TGroupCommitMode::ASYNC_MODE));
     }
     return Status::OK();
 }
 
-Status GroupCommitBlockSink::_add_blocks(bool is_blocks_contain_all_load_data) {
+Status GroupCommitBlockSink::_add_blocks(RuntimeState* state,
+                                         bool is_blocks_contain_all_load_data) {
     DCHECK(_is_block_appended == false);
     TUniqueId load_id;
     load_id.__set_hi(_load_id.hi);
@@ -257,7 +258,7 @@ Status GroupCommitBlockSink::_add_blocks(bool is_blocks_contain_all_load_data) {
     }
     for (auto it = _blocks.begin(); it != _blocks.end(); ++it) {
         RETURN_IF_ERROR(_load_block_queue->add_block(
-                *it, _group_commit_mode == TGroupCommitMode::ASYNC_MODE));
+                state, *it, _group_commit_mode == TGroupCommitMode::ASYNC_MODE));
     }
     _is_block_appended = true;
     _blocks.clear();
diff --git a/be/src/vec/sink/group_commit_block_sink.h b/be/src/vec/sink/group_commit_block_sink.h
index 84ffebf8fe1346..3db4bdd31f8d48 100644
--- a/be/src/vec/sink/group_commit_block_sink.h
+++ b/be/src/vec/sink/group_commit_block_sink.h
@@ -47,7 +47,7 @@ class GroupCommitBlockSink : public DataSink {
 
 private:
     Status _add_block(RuntimeState* state, std::shared_ptr<vectorized::Block> block);
-    Status _add_blocks(bool is_blocks_contain_all_load_data);
+    Status _add_blocks(RuntimeState* state, bool is_blocks_contain_all_load_data);
 
     vectorized::VExprContextSPtrs _output_vexpr_ctxs;
 

From 7cb323959949dd256a18517135574386d5630853 Mon Sep 17 00:00:00 2001
From: Siyang Tang <82279870+TangSiyang2001@users.noreply.github.com>
Date: Tue, 2 Jan 2024 11:16:06 +0800
Subject: [PATCH 238/299] [fix](sink) add writting restriction for
 OlapTableSinkV2Operator (#29340)

---
 be/src/pipeline/exec/olap_table_sink_v2_operator.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/be/src/pipeline/exec/olap_table_sink_v2_operator.h b/be/src/pipeline/exec/olap_table_sink_v2_operator.h
index 72b6a15e733a93..1e68f9e98d3b21 100644
--- a/be/src/pipeline/exec/olap_table_sink_v2_operator.h
+++ b/be/src/pipeline/exec/olap_table_sink_v2_operator.h
@@ -39,7 +39,7 @@ class OlapTableSinkV2Operator final : public DataSinkOperator<vectorized::VOlapT
     OlapTableSinkV2Operator(OperatorBuilderBase* operator_builder, DataSink* sink)
             : DataSinkOperator(operator_builder, sink) {}
 
-    bool can_write() override { return true; } // TODO: need use mem_limit
+    bool can_write() override { return _sink->can_write(); }
 };
 
 class OlapTableSinkV2OperatorX;

From b7487430da1386e7fa73e14179a43acc0d646767 Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Tue, 2 Jan 2024 11:32:14 +0800
Subject: [PATCH 239/299] Revert "[improve](move-memtable) cancel load rapidly
 when stream close wait (#29322)" (#29371)

This reverts commit bbf58c5aa42d40e66bc6ccc9ed91a4fcb4bdfff7.
---
 be/src/vec/sink/load_stream_stub.cpp          |  11 +-
 be/src/vec/sink/load_stream_stub.h            |  54 +--------
 be/src/vec/sink/load_stream_stub_pool.cpp     |  13 +--
 be/src/vec/sink/load_stream_stub_pool.h       |   4 +-
 be/src/vec/sink/writer/vtablet_writer_v2.cpp  |   5 +-
 .../io/fs/stream_sink_file_writer_test.cpp    |   6 +-
 .../vec/exec/load_stream_stub_pool_test.cpp   |   7 +-
 ...eam_stub_close_wait_fault_injection.groovy | 106 ------------------
 8 files changed, 21 insertions(+), 185 deletions(-)
 delete mode 100644 regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy

diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index a5f4f59dda48f3..4b557cae93d06e 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -90,22 +90,19 @@ int LoadStreamStub::LoadStreamReplyHandler::on_received_messages(brpc::StreamId
 void LoadStreamStub::LoadStreamReplyHandler::on_closed(brpc::StreamId id) {
     LOG(INFO) << "on_closed, load_id=" << _load_id << ", stream_id=" << id;
     std::lock_guard<bthread::Mutex> lock(_mutex);
-    DBUG_EXECUTE_IF("LoadStreamStub::LoadStreamReplyHandler::on_closed.close_wait", { return; });
     _is_closed.store(true);
     _close_cv.notify_all();
 }
 
-LoadStreamStub::LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use, RuntimeState* state)
-        : _state(state),
-          _use_cnt(num_use),
+LoadStreamStub::LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use)
+        : _use_cnt(num_use),
           _load_id(load_id),
           _src_id(src_id),
           _tablet_schema_for_index(std::make_shared<IndexToTabletSchema>()),
           _enable_unique_mow_for_index(std::make_shared<IndexToEnableMoW>()) {};
 
-LoadStreamStub::LoadStreamStub(LoadStreamStub& stub, RuntimeState* state)
-        : _state(state),
-          _use_cnt(stub._use_cnt.load()),
+LoadStreamStub::LoadStreamStub(LoadStreamStub& stub)
+        : _use_cnt(stub._use_cnt.load()),
           _load_id(stub._load_id),
           _src_id(stub._src_id),
           _tablet_schema_for_index(stub._tablet_schema_for_index),
diff --git a/be/src/vec/sink/load_stream_stub.h b/be/src/vec/sink/load_stream_stub.h
index 9c45926d3cc1f5..edbbbda1e64cbf 100644
--- a/be/src/vec/sink/load_stream_stub.h
+++ b/be/src/vec/sink/load_stream_stub.h
@@ -57,7 +57,6 @@
 #include "gutil/ref_counted.h"
 #include "runtime/exec_env.h"
 #include "runtime/memory/mem_tracker.h"
-#include "runtime/runtime_state.h"
 #include "runtime/thread_context.h"
 #include "runtime/types.h"
 #include "util/countdown_latch.h"
@@ -102,29 +101,11 @@ class LoadStreamStub {
         Status close_wait(int64_t timeout_ms) {
             DCHECK(timeout_ms > 0) << "timeout_ms should be greator than 0";
             std::unique_lock<bthread::Mutex> lock(_mutex);
-            int ret = 0;
-            MonotonicStopWatch watch;
-            watch.start();
-            while (true) {
-                if (_is_closed) {
-                    return Status::OK();
-                }
-                if (_stub->is_cancelled()) {
-                    return _stub->cancelled_state();
-                }
-                if (_stub->runtime_state()->is_cancelled()) {
-                    return Status::Cancelled(_stub->runtime_state()->cancel_reason());
-                }
-                // wait 1s once time.
-                ret = _close_cv.wait_for(lock, 1);
-                if (ret == 0) {
-                    return Status::OK();
-                }
-                if (watch.elapsed_time() / 1000 / 1000 >= timeout_ms) {
-                    return Status::InternalError("stream close wait timeout, result: {}", ret);
-                }
+            if (_is_closed) {
+                return Status::OK();
             }
-            return Status::OK();
+            int ret = _close_cv.wait_for(lock, timeout_ms * 1000);
+            return ret == 0 ? Status::OK() : Status::Error<true>(ret, "stream close_wait timeout");
         };
 
         std::vector<int64_t> success_tablets() {
@@ -157,28 +138,10 @@ class LoadStreamStub {
 
 public:
     // construct new stub
-    LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use, RuntimeState* state);
+    LoadStreamStub(PUniqueId load_id, int64_t src_id, int num_use);
 
     // copy constructor, shared_ptr members are shared
-    LoadStreamStub(LoadStreamStub& stub, RuntimeState* state);
-
-    void cancel(Status status) {
-        _cancel = true;
-        _cancel_status = status;
-    }
-
-    RuntimeState* runtime_state() const { return _state; }
-
-    bool is_cancelled() const { return _cancel; }
-
-    Status cancelled_state() const { return _cancel_status; }
-
-    std::string cancel_reason() const {
-        if (_state == nullptr) {
-            return "";
-        }
-        return _state->cancel_reason();
-    }
+    LoadStreamStub(LoadStreamStub& stub);
 
 // for mock this class in UT
 #ifdef BE_TEST
@@ -268,11 +231,6 @@ class LoadStreamStub {
     Status _send_with_retry(butil::IOBuf& buf);
 
 protected:
-    RuntimeState* _state = nullptr;
-
-    bool _cancel = false;
-    Status _cancel_status;
-
     std::atomic<bool> _is_init;
     bthread::Mutex _mutex;
 
diff --git a/be/src/vec/sink/load_stream_stub_pool.cpp b/be/src/vec/sink/load_stream_stub_pool.cpp
index 94838acdc08b20..c867756d8fb2be 100644
--- a/be/src/vec/sink/load_stream_stub_pool.cpp
+++ b/be/src/vec/sink/load_stream_stub_pool.cpp
@@ -54,19 +54,13 @@ void LoadStreams::release() {
     }
 }
 
-void LoadStreams::cancel(Status status) {
-    for (auto& stream : _streams) {
-        stream->cancel(status);
-    }
-}
-
 LoadStreamStubPool::LoadStreamStubPool() = default;
 
 LoadStreamStubPool::~LoadStreamStubPool() = default;
 
 std::shared_ptr<LoadStreams> LoadStreamStubPool::get_or_create(PUniqueId load_id, int64_t src_id,
                                                                int64_t dst_id, int num_streams,
-                                                               int num_sink, RuntimeState* state) {
+                                                               int num_sink) {
     auto key = std::make_pair(UniqueId(load_id), dst_id);
     std::lock_guard<std::mutex> lock(_mutex);
     std::shared_ptr<LoadStreams> streams = _pool[key];
@@ -75,12 +69,11 @@ std::shared_ptr<LoadStreams> LoadStreamStubPool::get_or_create(PUniqueId load_id
     }
     DCHECK(num_streams > 0) << "stream num should be greater than 0";
     DCHECK(num_sink > 0) << "sink num should be greater than 0";
-    auto [it, _] =
-            _template_stubs.emplace(load_id, new LoadStreamStub {load_id, src_id, num_sink, state});
+    auto [it, _] = _template_stubs.emplace(load_id, new LoadStreamStub {load_id, src_id, num_sink});
     streams = std::make_shared<LoadStreams>(load_id, dst_id, num_sink, this);
     for (int32_t i = 0; i < num_streams; i++) {
         // copy construct, internal tablet schema map will be shared among all stubs
-        streams->streams().emplace_back(new LoadStreamStub {*it->second, state});
+        streams->streams().emplace_back(new LoadStreamStub {*it->second});
     }
     _pool[key] = streams;
     return streams;
diff --git a/be/src/vec/sink/load_stream_stub_pool.h b/be/src/vec/sink/load_stream_stub_pool.h
index b548d31a14ff56..e41083825be07e 100644
--- a/be/src/vec/sink/load_stream_stub_pool.h
+++ b/be/src/vec/sink/load_stream_stub_pool.h
@@ -78,8 +78,6 @@ class LoadStreams {
 
     void release();
 
-    void cancel(Status status);
-
     Streams& streams() { return _streams; }
 
 private:
@@ -97,7 +95,7 @@ class LoadStreamStubPool {
     ~LoadStreamStubPool();
 
     std::shared_ptr<LoadStreams> get_or_create(PUniqueId load_id, int64_t src_id, int64_t dst_id,
-                                               int num_streams, int num_sink, RuntimeState* state);
+                                               int num_streams, int num_sink);
 
     void erase(UniqueId load_id, int64_t dst_id);
 
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 2550fb172c57c1..7a83f670fbe667 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -113,7 +113,7 @@ Status VTabletWriterV2::_incremental_open_streams(
     }
     for (int64_t node_id : new_backends) {
         auto load_streams = ExecEnv::GetInstance()->load_stream_stub_pool()->get_or_create(
-                _load_id, _backend_id, node_id, _stream_per_node, _num_local_sink, _state);
+                _load_id, _backend_id, node_id, _stream_per_node, _num_local_sink);
         RETURN_IF_ERROR(_open_streams_to_backend(node_id, *load_streams));
         _streams_for_node[node_id] = load_streams;
     }
@@ -261,7 +261,7 @@ Status VTabletWriterV2::open(RuntimeState* state, RuntimeProfile* profile) {
 Status VTabletWriterV2::_open_streams(int64_t src_id) {
     for (auto& [dst_id, _] : _tablets_for_node) {
         auto streams = ExecEnv::GetInstance()->load_stream_stub_pool()->get_or_create(
-                _load_id, src_id, dst_id, _stream_per_node, _num_local_sink, _state);
+                _load_id, src_id, dst_id, _stream_per_node, _num_local_sink);
         RETURN_IF_ERROR(_open_streams_to_backend(dst_id, *streams));
         _streams_for_node[dst_id] = streams;
     }
@@ -457,7 +457,6 @@ Status VTabletWriterV2::_cancel(Status status) {
     }
     for (const auto& [_, streams] : _streams_for_node) {
         streams->release();
-        streams->cancel(status);
     }
     return Status::OK();
 }
diff --git a/be/test/io/fs/stream_sink_file_writer_test.cpp b/be/test/io/fs/stream_sink_file_writer_test.cpp
index a0daa15916e0a2..7e5bdd350f5b37 100644
--- a/be/test/io/fs/stream_sink_file_writer_test.cpp
+++ b/be/test/io/fs/stream_sink_file_writer_test.cpp
@@ -51,8 +51,7 @@ static std::atomic<int64_t> g_num_request;
 class StreamSinkFileWriterTest : public testing::Test {
     class MockStreamStub : public LoadStreamStub {
     public:
-        MockStreamStub(PUniqueId load_id, int64_t src_id, RuntimeState* state)
-                : LoadStreamStub(load_id, src_id, 1, state) {};
+        MockStreamStub(PUniqueId load_id, int64_t src_id) : LoadStreamStub(load_id, src_id, 1) {};
 
         virtual ~MockStreamStub() = default;
 
@@ -86,9 +85,8 @@ class StreamSinkFileWriterTest : public testing::Test {
     virtual void SetUp() {
         _load_id.set_hi(LOAD_ID_HI);
         _load_id.set_lo(LOAD_ID_LO);
-        RuntimeState state;
         for (int src_id = 0; src_id < NUM_STREAM; src_id++) {
-            _streams.emplace_back(new MockStreamStub(_load_id, src_id, &state));
+            _streams.emplace_back(new MockStreamStub(_load_id, src_id));
         }
     }
 
diff --git a/be/test/vec/exec/load_stream_stub_pool_test.cpp b/be/test/vec/exec/load_stream_stub_pool_test.cpp
index 7faa5fb2e5dd23..24da3bb6999f36 100644
--- a/be/test/vec/exec/load_stream_stub_pool_test.cpp
+++ b/be/test/vec/exec/load_stream_stub_pool_test.cpp
@@ -30,14 +30,13 @@ class LoadStreamStubPoolTest : public testing::Test {
 
 TEST_F(LoadStreamStubPoolTest, test) {
     LoadStreamStubPool pool;
-    RuntimeState state;
     int64_t src_id = 100;
     PUniqueId load_id;
     load_id.set_hi(1);
     load_id.set_hi(2);
-    auto streams1 = pool.get_or_create(load_id, src_id, 101, 5, 1, &state);
-    auto streams2 = pool.get_or_create(load_id, src_id, 102, 5, 1, &state);
-    auto streams3 = pool.get_or_create(load_id, src_id, 101, 5, 1, &state);
+    auto streams1 = pool.get_or_create(load_id, src_id, 101, 5, 1);
+    auto streams2 = pool.get_or_create(load_id, src_id, 102, 5, 1);
+    auto streams3 = pool.get_or_create(load_id, src_id, 101, 5, 1);
     EXPECT_EQ(2, pool.size());
     EXPECT_EQ(1, pool.templates_size());
     EXPECT_EQ(streams1, streams3);
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
deleted file mode 100644
index 85ad5f7bc9d682..00000000000000
--- a/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
+++ /dev/null
@@ -1,106 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import org.codehaus.groovy.runtime.IOGroovyMethods
-import org.apache.doris.regression.util.Http
-
-suite("test_load_stream_stub_close_wait_fault_injection", "nonConcurrent") {
-    sql """ set enable_memtable_on_sink_node=true """
-    sql """ DROP TABLE IF EXISTS `baseall` """
-    sql """ DROP TABLE IF EXISTS `test` """
-    sql """
-        CREATE TABLE IF NOT EXISTS `baseall` (
-            `k0` boolean null comment "",
-            `k1` tinyint(4) null comment "",
-            `k2` smallint(6) null comment "",
-            `k3` int(11) null comment "",
-            `k4` bigint(20) null comment "",
-            `k5` decimal(9, 3) null comment "",
-            `k6` char(5) null comment "",
-            `k10` date null comment "",
-            `k11` datetime null comment "",
-            `k7` varchar(20) null comment "",
-            `k8` double max null comment "",
-            `k9` float sum null comment "",
-            `k12` string replace null comment "",
-            `k13` largeint(40) replace null comment ""
-        ) engine=olap
-        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
-        """
-    sql """
-        CREATE TABLE IF NOT EXISTS `test` (
-            `k0` boolean null comment "",
-            `k1` tinyint(4) null comment "",
-            `k2` smallint(6) null comment "",
-            `k3` int(11) null comment "",
-            `k4` bigint(20) null comment "",
-            `k5` decimal(9, 3) null comment "",
-            `k6` char(5) null comment "",
-            `k10` date null comment "",
-            `k11` datetime null comment "",
-            `k7` varchar(20) null comment "",
-            `k8` double max null comment "",
-            `k9` float sum null comment "",
-            `k12` string replace_if_not_null null comment "",
-            `k13` largeint(40) replace null comment ""
-        ) engine=olap
-        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
-        """
-
-    GetDebugPoint().clearDebugPointsForAllBEs()
-    streamLoad {
-        table "baseall"
-        db "regression_test_fault_injection_p0"
-        set 'column_separator', ','
-        file "baseall.txt"
-    }
-
-    try {
-        GetDebugPoint().enableDebugPointForAllBEs("LoadStreamStub::LoadStreamReplyHandler::on_closed.close_wait")
-        def thread1 = new Thread({
-            try {
-                def res = sql "insert into test select * from baseall where k1 <= 3"
-                logger.info(res.toString())
-            } catch(Exception e) {
-                logger.info(e.getMessage())
-                assertTrue(e.getMessage().contains("Communications link failure"))
-            } finally {
-                GetDebugPoint().disableDebugPointForAllBEs("LoadStreamStub::LoadStreamReplyHandler::on_closed.close_wait")
-            }
-        })
-        thread1.start()
-
-        sleep(1000)
-
-        def processList = sql "show processlist"
-        logger.info(processList.toString())
-        processList.each { item ->
-            logger.info(item[1].toString())
-            logger.info(item[11].toString())
-            if (item[11].toString() == "insert into test select * from baseall where k1 <= 3".toString()){
-                def res = sql "kill ${item[1]}"
-                logger.info(res.toString())
-            }
-        }
-    } catch(Exception e) {
-        logger.info(e.getMessage())
-    }
-
-    sql """ DROP TABLE IF EXISTS `baseall` """
-    sql """ DROP TABLE IF EXISTS `test` """
-    sql """ set enable_memtable_on_sink_node=false """
-}
\ No newline at end of file

From 90b2ee90b22c0c4a1b42f78ab66972cf790ca449 Mon Sep 17 00:00:00 2001
From: xzj7019 <131111794+xzj7019@users.noreply.github.com>
Date: Tue, 2 Jan 2024 13:51:11 +0800
Subject: [PATCH 240/299] [nereids] consider numNulls in filter estimation
 (#29184)

consider numNulls in filter estimation
---
 .../doris/nereids/stats/FilterEstimation.java |  59 +++--
 .../nereids/stats/FilterEstimationTest.java   | 203 +++++++++++++++++-
 2 files changed, 243 insertions(+), 19 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
index b716b350f24b84..c086aaef5c81bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/FilterEstimation.java
@@ -117,6 +117,8 @@ public Statistics visitCompoundPredicate(CompoundPredicate predicate, Estimation
                     colBuilder.setMinValue(union.getLow()).setMinExpr(union.getLowExpr())
                             .setMaxValue(union.getHigh()).setMaxExpr(union.getHighExpr())
                             .setNdv(union.getDistinctValues());
+                    double maxNumNulls = Math.max(leftColStats.numNulls, rightColStats.numNulls);
+                    colBuilder.setNumNulls(Math.min(colBuilder.getCount(), maxNumNulls));
                     orStats.addColumnStats(slot, colBuilder.build());
                 }
             }
@@ -175,7 +177,7 @@ public Statistics visitComparisonPredicate(ComparisonPredicate cp, EstimationCon
     }
 
     private Statistics updateLessThanLiteral(Expression leftExpr, ColumnStatistic statsForLeft,
-            ColumnStatistic statsForRight, EstimationContext context, boolean contains) {
+            ColumnStatistic statsForRight, EstimationContext context) {
         StatisticRange rightRange = new StatisticRange(statsForLeft.minValue, statsForLeft.minExpr,
                 statsForRight.maxValue, statsForRight.maxExpr,
                 statsForLeft.ndv, leftExpr.getDataType());
@@ -185,7 +187,7 @@ private Statistics updateLessThanLiteral(Expression leftExpr, ColumnStatistic st
     }
 
     private Statistics updateGreaterThanLiteral(Expression leftExpr, ColumnStatistic statsForLeft,
-            ColumnStatistic statsForRight, EstimationContext context, boolean contains) {
+            ColumnStatistic statsForRight, EstimationContext context) {
         StatisticRange rightRange = new StatisticRange(statsForRight.minValue, statsForRight.minExpr,
                 statsForLeft.maxValue, statsForLeft.maxExpr,
                 statsForLeft.ndv, leftExpr.getDataType());
@@ -202,12 +204,9 @@ private Statistics calculateWhenLiteralRight(ComparisonPredicate cp,
             return estimateEqualTo(cp, statsForLeft, statsForRight, context);
         } else {
             if (cp instanceof LessThan || cp instanceof LessThanEqual) {
-                return updateLessThanLiteral(cp.left(), statsForLeft, statsForRight,
-                        context, cp instanceof LessThanEqual);
+                return updateLessThanLiteral(cp.left(), statsForLeft, statsForRight, context);
             } else if (cp instanceof GreaterThan || cp instanceof GreaterThanEqual) {
-
-                return updateGreaterThanLiteral(cp.left(), statsForLeft, statsForRight, context,
-                        cp instanceof GreaterThanEqual);
+                return updateGreaterThanLiteral(cp.left(), statsForLeft, statsForRight, context);
             } else {
                 throw new RuntimeException(String.format("Unexpected expression : %s", cp.toSql()));
             }
@@ -225,6 +224,7 @@ private Statistics estimateEqualTo(ComparisonPredicate cp, ColumnStatistic stats
         } else {
             selectivity = StatsMathUtil.minNonNaN(1.0, 1.0 / ndv);
         }
+        selectivity = getNotNullSelectivity(statsForLeft, selectivity);
         Statistics equalStats = context.statistics.withSel(selectivity);
         Expression left = cp.left();
         equalStats.addColumnStats(left, statsForRight);
@@ -331,10 +331,12 @@ A not in (1, 2, 3, 100):
                 selectivity = 1.0;
             }
         }
+        compareExprStatsBuilder.setNumNulls(0);
         Statistics estimated = new Statistics(context.statistics);
+        ColumnStatistic stats = compareExprStatsBuilder.build();
+        selectivity = getNotNullSelectivity(stats, selectivity);
         estimated = estimated.withSel(selectivity);
-        estimated.addColumnStats(compareExpr,
-                compareExprStatsBuilder.build());
+        estimated.addColumnStats(compareExpr, stats);
         context.addKeyIfSlot(compareExpr);
         return estimated;
     }
@@ -394,6 +396,11 @@ public Statistics visitNot(Not not, EstimationContext context) {
                             .setMaxValue(originColStats.maxValue)
                             .setMaxExpr(originColStats.maxExpr);
                 }
+                if (not.child().getInputSlots().size() == 1 && !(child instanceof IsNull)) {
+                    // only consider the single column numNull, otherwise, ignore
+                    rowCount = Math.max(rowCount - originColStats.numNulls, 1);
+                    statisticsBuilder.setRowCount(rowCount);
+                }
                 statisticsBuilder.putColumnStatistics(slot, colBuilder.build());
             }
         }
@@ -460,15 +467,18 @@ private Statistics estimateBinaryComparisonFilter(Expression leftExpr, ColumnSta
                     .setMaxValue(Double.POSITIVE_INFINITY)
                     .setMaxExpr(null)
                     .setNdv(0)
-                    .setCount(0);
+                    .setCount(0)
+                    .setNumNulls(0);
         } else {
             leftColumnStatisticBuilder = new ColumnStatisticBuilder(leftStats)
                     .setMinValue(intersectRange.getLow())
                     .setMinExpr(intersectRange.getLowExpr())
                     .setMaxValue(intersectRange.getHigh())
                     .setMaxExpr(intersectRange.getHighExpr())
-                    .setNdv(intersectRange.getDistinctValues());
+                    .setNdv(intersectRange.getDistinctValues())
+                    .setNumNulls(0);
             double sel = leftRange.overlapPercentWith(rightRange);
+            sel = getNotNullSelectivity(leftStats, sel);
             updatedStatistics = context.statistics.withSel(sel);
             leftColumnStatisticBuilder.setCount(updatedStatistics.getRowCount());
         }
@@ -488,6 +498,7 @@ private Statistics estimateColumnEqualToColumn(Expression leftExpr, ColumnStatis
         intersectBuilder.setNdv(intersect.getDistinctValues());
         intersectBuilder.setMinValue(intersect.getLow());
         intersectBuilder.setMaxValue(intersect.getHigh());
+        intersectBuilder.setNumNulls(0);
         double sel = 1 / StatsMathUtil.nonZeroDivisor(Math.max(leftStats.ndv, rightStats.ndv));
         Statistics updatedStatistics = context.statistics.withSel(sel);
         updatedStatistics.addColumnStats(leftExpr, intersectBuilder.build());
@@ -568,10 +579,34 @@ public Statistics visitLike(Like like, EstimationContext context) {
                     "col stats not found. slot=%s in %s",
                     like.left().toSql(), like.toSql());
             ColumnStatisticBuilder colBuilder = new ColumnStatisticBuilder(origin);
-            colBuilder.setNdv(origin.ndv * DEFAULT_LIKE_COMPARISON_SELECTIVITY).setNumNulls(0);
+            double selectivity = StatsMathUtil.divide(DEFAULT_LIKE_COMPARISON_SELECTIVITY, origin.ndv);
+            double notNullSel = getNotNullSelectivity(origin, selectivity);
+            colBuilder.setNdv(origin.ndv * DEFAULT_LIKE_COMPARISON_SELECTIVITY)
+                    .setCount(notNullSel * context.statistics.getRowCount()).setNumNulls(0);
             statsBuilder.putColumnStatistics(like.left(), colBuilder.build());
             context.addKeyIfSlot(like.left());
         }
         return statsBuilder.build();
     }
+
+    private double getNotNullSelectivity(ColumnStatistic stats, double origSel) {
+        double rowCount = stats.count;
+        double numNulls = stats.numNulls;
+
+        // comment following check since current rowCount and ndv may be inconsistant
+        // e.g, rowCount has been reduced by one filter but another filter column's
+        // ndv and numNull remains originally, which will unexpectedly go into the following
+        // normalization.
+
+        //if (numNulls > rowCount - ndv) {
+        //    numNulls = rowCount - ndv > 0 ? rowCount - ndv : 0;
+        //}
+        double notNullSel = rowCount <= 1.0 ? 1.0 : 1 - getValidSelectivity(numNulls / rowCount);
+        double validSel = origSel * notNullSel;
+        return getValidSelectivity(validSel);
+    }
+
+    private static double getValidSelectivity(double nullSel) {
+        return nullSel < 0 ? 0 : (nullSel > 1 ? 1 : nullSel);
+    }
 }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
index f5491c633313e0..b476cc563be32b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/FilterEstimationTest.java
@@ -64,10 +64,10 @@ public void testOrNaN() {
         Or or = new Or(greaterThan1, lessThan);
         Map<Expression, ColumnStatistic> columnStat = new HashMap<>();
         ColumnStatistic aStats = new ColumnStatisticBuilder().setCount(500).setNdv(500).setAvgSizeByte(4)
-                .setNumNulls(500).setDataSize(0)
+                .setNumNulls(0).setDataSize(0)
                 .setMinValue(0).setMaxValue(1000).setMinExpr(null).build();
         ColumnStatistic bStats = new ColumnStatisticBuilder().setCount(500).setNdv(500).setAvgSizeByte(4)
-                .setNumNulls(500).setDataSize(0)
+                .setNumNulls(0).setDataSize(0)
                 .setMinValue(0).setMaxValue(1000).setMinExpr(null).setIsUnknown(true).build();
         columnStat.put(a, aStats);
         columnStat.put(b, bStats);
@@ -93,10 +93,10 @@ public void testAndNaN() {
         And and = new And(greaterThan1, lessThan);
         Map<Expression, ColumnStatistic> columnStat = new HashMap<>();
         ColumnStatistic aStats = new ColumnStatisticBuilder().setCount(500).setNdv(500)
-                .setAvgSizeByte(4).setNumNulls(500).setDataSize(0)
+                .setAvgSizeByte(4).setNumNulls(0).setDataSize(0)
                 .setMinValue(0).setMaxValue(1000).setMinExpr(null).build();
         ColumnStatistic bStats = new ColumnStatisticBuilder().setCount(500).setNdv(500)
-                .setAvgSizeByte(4).setNumNulls(500).setDataSize(0)
+                .setAvgSizeByte(4).setNumNulls(0).setDataSize(0)
                 .setMinValue(0).setMaxValue(1000).setMinExpr(null).setIsUnknown(true).build();
         columnStat.put(a, aStats);
         columnStat.put(b, bStats);
@@ -185,13 +185,13 @@ public void test1() {
         Or or = new Or(and, equalTo);
         Map<Expression, ColumnStatistic> slotToColumnStat = new HashMap<>();
         ColumnStatistic aStats = new ColumnStatisticBuilder().setCount(500).setNdv(500)
-                .setAvgSizeByte(4).setNumNulls(500).setDataSize(0)
+                .setAvgSizeByte(4).setNumNulls(0).setDataSize(0)
                 .setMinValue(0).setMaxValue(1000).setMinExpr(null).build();
         ColumnStatistic bStats = new ColumnStatisticBuilder().setCount(500).setNdv(500)
-                .setAvgSizeByte(4).setNumNulls(500).setDataSize(0)
+                .setAvgSizeByte(4).setNumNulls(0).setDataSize(0)
                 .setMinValue(0).setMaxValue(1000).setMinExpr(null).build();
         ColumnStatistic cStats = new ColumnStatisticBuilder().setCount(500).setNdv(500)
-                .setAvgSizeByte(4).setNumNulls(500).setDataSize(0)
+                .setAvgSizeByte(4).setNumNulls(0).setDataSize(0)
                 .setMinValue(0).setMaxValue(1000).setMinExpr(null).build();
         slotToColumnStat.put(a, aStats);
         slotToColumnStat.put(b, bStats);
@@ -910,4 +910,193 @@ public void testIsNotNull() {
         Statistics result = filterEstimation.estimate(not, stats);
         Assertions.assertEquals(result.getRowCount(), 90);
     }
+
+    /**
+     * a = 1
+     */
+    @Test
+    public void testNumNullsEqualTo() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        EqualTo equalTo = new EqualTo(a, int1);
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(equalTo, stats);
+        Assertions.assertEquals(result.getRowCount(), 1.0, 0.01);
+    }
+
+    /**
+     * a > 1
+     */
+    @Test
+    public void testNumNullsComparable() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        GreaterThan greaterThan = new GreaterThan(a, int1);
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(greaterThan, stats);
+        Assertions.assertEquals(result.getRowCount(), 2.0, 0.01);
+    }
+
+    /**
+     * a in (1, 2)
+     */
+    @Test
+    public void testNumNullsIn() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        IntegerLiteral int2 = new IntegerLiteral(2);
+        InPredicate in = new InPredicate(a, Lists.newArrayList(int1, int2));
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(in, stats);
+        Assertions.assertEquals(result.getRowCount(), 10.0, 0.01);
+    }
+
+    /**
+     * not a = 1
+     */
+    @Test
+    public void testNumNullsNotEqualTo() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        EqualTo equalTo = new EqualTo(a, int1);
+        Not not = new Not(equalTo);
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(not, stats);
+        Assertions.assertEquals(result.getRowCount(), 1.0, 0.01);
+    }
+
+    /**
+     * a not in (1, 2)
+     */
+    @Test
+    public void testNumNullsNotIn() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        IntegerLiteral int2 = new IntegerLiteral(2);
+        InPredicate in = new InPredicate(a, Lists.newArrayList(int1, int2));
+        Not not = new Not(in);
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(not, stats);
+        Assertions.assertEquals(result.getRowCount(), 1.0, 0.01);
+    }
+
+    /**
+     * a >= 1 and a <= 2
+     */
+    @Test
+    public void testNumNullsAnd() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        IntegerLiteral int2 = new IntegerLiteral(2);
+        GreaterThanEqual greaterThanEqual = new GreaterThanEqual(a, int1);
+        LessThanEqual lessThanEqual = new LessThanEqual(a, int2);
+        And and = new And(greaterThanEqual, lessThanEqual);
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(and, stats);
+        Assertions.assertEquals(result.getRowCount(), 2.0, 0.01);
+    }
+
+    /**
+     * a >= 1 or a <= 2
+     */
+    @Test
+    public void testNumNullsOr() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        IntegerLiteral int2 = new IntegerLiteral(2);
+        GreaterThanEqual greaterThanEqual = new GreaterThanEqual(a, int2);
+        LessThanEqual lessThanEqual = new LessThanEqual(a, int1);
+        Or or = new Or(greaterThanEqual, lessThanEqual);
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(or, stats);
+        Assertions.assertEquals(result.getRowCount(), 2.0, 0.01);
+    }
+
+    /**
+     * a >= 1 or a is null
+     */
+    @Test
+    public void testNumNullsOrIsNull() {
+        SlotReference a = new SlotReference("a", IntegerType.INSTANCE);
+        ColumnStatisticBuilder builder = new ColumnStatisticBuilder()
+                .setNdv(2)
+                .setAvgSizeByte(4)
+                .setNumNulls(8)
+                .setMaxValue(2)
+                .setMinValue(1)
+                .setCount(10);
+        IntegerLiteral int1 = new IntegerLiteral(1);
+        GreaterThanEqual greaterThanEqual = new GreaterThanEqual(a, int1);
+        IsNull isNull = new IsNull(a);
+        Or or = new Or(greaterThanEqual, isNull);
+        Statistics stats = new Statistics(10, new HashMap<>());
+        stats.addColumnStats(a, builder.build());
+        FilterEstimation filterEstimation = new FilterEstimation();
+        Statistics result = filterEstimation.estimate(or, stats);
+        Assertions.assertEquals(result.getRowCount(), 10.0, 0.01);
+    }
+
 }

From bf6488e5189f51f5fefbc4da3cc75a3cd653454f Mon Sep 17 00:00:00 2001
From: Rohit Satardekar <rohitrs1983@yahoo.com>
Date: Tue, 2 Jan 2024 11:35:05 +0530
Subject: [PATCH 241/299] [fix](doc) show catalog example output fix (#29150)

Co-authored-by: Rohit Satardekar <rohitrs1983@gmail.com>
---
 .../lakehouse/multi-catalog/multi-catalog.md  | 14 ++++++------
 .../Show-Statements/SHOW-CATALOGS.md          | 22 +++++++++----------
 .../lakehouse/multi-catalog/multi-catalog.md  | 12 +++++-----
 .../Show-Statements/SHOW-CATALOGS.md          | 22 +++++++++----------
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md b/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md
index e79ca063d1adb2..200f5cd1fb222c 100644
--- a/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md
+++ b/docs/en/docs/lakehouse/multi-catalog/multi-catalog.md
@@ -98,12 +98,12 @@ For more information about Hive, please see [Hive](./hive.md).
 	
 	```
 	mysql> SHOW CATALOGS;
-	+-----------+-------------+----------+
-	| CatalogId | CatalogName | Type     |
-	+-----------+-------------+----------+
-	|     10024 | hive        | hms      |
-	|         0 | internal    | internal |
-	+-----------+-------------+----------+
+	+-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+	| CatalogId | CatalogName | Type     | IsCurrent | CreateTime              | LastUpdateTime      | Comment                |
+	+-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+	|     10024 | hive        | hms      | yes       | 2023-12-25 16:11:41.687 | 2023-12-25 20:43:18 | NULL                   |
+	|         0 | internal    | internal |           | UNRECORDED              | NULL                | Doris internal catalog |
+	+-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
 	```
 	
 	> Syntax Help: [SHOW CATALOGS](https://doris.apache.org/docs/dev/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS/)
@@ -321,4 +321,4 @@ CREATE CATALOG es PROPERTIES (
 
 ### Auto Refresh
 
-Auto-refresh currently only supports [Hive Catalog](./hive.md).
\ No newline at end of file
+Auto-refresh currently only supports [Hive Catalog](./hive.md).
diff --git a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md
index 5e29098d934edf..e9648748e256a4 100644
--- a/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md
+++ b/docs/en/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md
@@ -58,23 +58,23 @@ Return result:
 
    ```sql
    SHOW CATALOGS;
-    +-----------+-------------+----------+-----------+
-    | CatalogId | CatalogName | Type     | IsCurrent |
-    +-----------+-------------+----------+-----------+
-    |    130100 | hive        | hms      |           |
-    |         0 | internal    | internal | yes       |
-    +-----------+-------------+----------+-----------+
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    | CatalogId | CatalogName | Type     | IsCurrent | CreateTime              | LastUpdateTime      | Comment                |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    |    130100 | hive        | hms      |           | 2023-12-25 16:11:41.687 | 2023-12-25 20:43:18 | NULL                   |
+    |         0 | internal    | internal | yes       | UNRECORDED              | NULL                | Doris internal catalog |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
    	```
 
 2. Fuzzy query by catalog name
 
    ```sql
    SHOW CATALOGS LIKE 'hi%';
-    +-----------+-------------+----------+-----------+
-    | CatalogId | CatalogName | Type     | IsCurrent |
-    +-----------+-------------+----------+-----------+
-    |    130100 | hive        | hms      |           |
-    +-----------+-------------+----------+-----------+
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    | CatalogId | CatalogName | Type     | IsCurrent | CreateTime              | LastUpdateTime      | Comment                |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    |    130100 | hive        | hms      |           | 2023-12-25 16:11:41.687 | 2023-12-25 20:43:18 | NULL                   |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
        ```
    
 ### Keywords
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md b/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md
index b868a3ca3559fd..a42e984c3fc5a0 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/multi-catalog.md
@@ -98,12 +98,12 @@ under the License.
 	
 	```
 	mysql> SHOW CATALOGS;
-	+-----------+-------------+----------+
-	| CatalogId | CatalogName | Type     |
-	+-----------+-------------+----------+
-	|     10024 | hive        | hms      |
-	|         0 | internal    | internal |
-	+-----------+-------------+----------+
+	+-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+	| CatalogId | CatalogName | Type     | IsCurrent | CreateTime              | LastUpdateTime      | Comment                |
+	+-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+	|     10024 | hive        | hms      | yes       | 2023-12-25 16:11:41.687 | 2023-12-25 20:43:18 | NULL                   |
+	|         0 | internal    | internal |           | UNRECORDED              | NULL                | Doris internal catalog |
+	+-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
 	```
 	
 	> [SHOW CATALOGS 语法帮助](../../sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md)
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md
index ddc30d1e052508..10061fcb6119ed 100644
--- a/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Show-Statements/SHOW-CATALOGS.md
@@ -61,23 +61,23 @@ LIKE：可按照CATALOG名进行模糊查询
 
    ```sql
    SHOW CATALOGS;
-    +-----------+-------------+----------+-----------+
-    | CatalogId | CatalogName | Type     | IsCurrent |
-    +-----------+-------------+----------+-----------+
-    |    130100 | hive        | hms      |           |
-    |         0 | internal    | internal | yes       |
-    +-----------+-------------+----------+-----------+
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    | CatalogId | CatalogName | Type     | IsCurrent | CreateTime              | LastUpdateTime      | Comment                |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    |    130100 | hive        | hms      |           | 2023-12-25 16:11:41.687 | 2023-12-25 20:43:18 | NULL                   |
+    |         0 | internal    | internal | yes       | UNRECORDED              | NULL                | Doris internal catalog |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
        ```
    
 2. 按照目录名进行模糊搜索
 
    ```sql
    SHOW CATALOGS LIKE 'hi%';
-    +-----------+-------------+----------+-----------+
-    | CatalogId | CatalogName | Type     | IsCurrent |
-    +-----------+-------------+----------+-----------+
-    |    130100 | hive        | hms      |           |
-    +-----------+-------------+----------+-----------+
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    | CatalogId | CatalogName | Type     | IsCurrent | CreateTime              | LastUpdateTime      | Comment                |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
+    |    130100 | hive        | hms      |           | 2023-12-25 16:11:41.687 | 2023-12-25 20:43:18 | NULL                   |
+    +-----------+-------------+----------+-----------+-------------------------+---------------------+------------------------+
        ```
 
 ### Keywords

From f6443efe1af08d490ca108ede17e6df680492d42 Mon Sep 17 00:00:00 2001
From: Liqf <109049295+LemonLiTree@users.noreply.github.com>
Date: Tue, 2 Jan 2024 14:05:19 +0800
Subject: [PATCH 242/299] [doc](fix)fix kill doc error and add example (#29313)

---
 docs/en/docs/ecosystem/audit-plugin.md        |  6 +++
 .../KILL.md                                   | 39 ++++++++++++++++++-
 .../KILL.md                                   | 37 +++++++++++++++++-
 3 files changed, 80 insertions(+), 2 deletions(-)

diff --git a/docs/en/docs/ecosystem/audit-plugin.md b/docs/en/docs/ecosystem/audit-plugin.md
index 5c045b71781859..401891c8aaf6fb 100644
--- a/docs/en/docs/ecosystem/audit-plugin.md
+++ b/docs/en/docs/ecosystem/audit-plugin.md
@@ -70,6 +70,12 @@ After executing install, the AuditLoader directory will be automatically generat
     * user: Cluster username. The user must have INSERT permission on the corresponding table.
     * password: Cluster user password.
 
+4. Repackaging the Audit Loader plugin
+
+    ```shell
+    zip -r -q -m auditloader.zip auditloader.jar plugin.properties plugin.conf
+    ```
+
 ### 创建库表
 
 In Doris, you need to create the library and table of the audit log. The table structure is as follows:
diff --git a/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md b/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md
index 67f948c964078b..3405938101407b 100644
--- a/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md
+++ b/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md
@@ -34,7 +34,7 @@ KILL
 
 Each Doris connection runs in a separate thread. You can kill a thread with the KILL processlist_id statement.
 
-The thread process list identifier can be determined from the ID column of the INFORMATION_SCHEMA PROCESSLIST table, the Id column of the SHOW PROCESSLIST output, and the PROCESSLIST_ID column of the Performance Schema thread table.
+The thread process list identifier can be queried from the Id column output by SHOW PROCESSLIST or SELECT CONNECTION_ID() to query the current connection id.
 
 grammar:
 
@@ -53,6 +53,43 @@ KILL QUERY processlist_id | query_id
 
 ### Example
 
+
+1. View the connection id of the current connection.
+
+```sql
+mysql> select connection_id();
++-----------------+
+| connection_id() |
++-----------------+
+| 48              |
++-----------------+
+1 row in set (0.00 sec)
+```
+
+2. View the connection ids of all connections.
+
+```sql
+mysql> SHOW PROCESSLIST;
++------------------+------+------+--------------------+---------------------+----------+---------+---------+------+-------+-----------------------------------+---------------------------------------------------------------------------------------+
+| CurrentConnected | Id   | User | Host               | LoginTime           | Catalog  | Db      | Command | Time | State | QueryId                           | Info                                                                                  |
++------------------+------+------+--------------------+---------------------+----------+---------+---------+------+-------+-----------------------------------+---------------------------------------------------------------------------------------+
+| Yes              |   48 | root | 10.16.xx.xx:44834   | 2023-12-29 16:49:47 | internal | test | Query   |    0 | OK    | e6e4ce9567b04859-8eeab8d6b5513e38 | SHOW PROCESSLIST                                                                      |
+|                  |   50 | root | 192.168.xx.xx:52837 | 2023-12-29 16:51:34 | internal |      | Sleep   | 1837 | EOF   | deaf13c52b3b4a3b-b25e8254b50ff8cb | SELECT @@session.transaction_isolation                                                |
+|                  |   51 | root | 192.168.xx.xx:52843 | 2023-12-29 16:51:35 | internal |      | Sleep   |  907 | EOF   | 437f219addc0404f-9befe7f6acf9a700 | /* ApplicationName=DBeaver Ultimate 23.1.3 - Metadata */ SHOW STATUS                  |
+|                  |   55 | root | 192.168.xx.xx:55533 | 2023-12-29 17:09:32 | internal | test | Sleep   |  271 | EOF   | f02603dc163a4da3-beebbb5d1ced760c | /* ApplicationName=DBeaver Ultimate 23.1.3 - SQLEditor <Console> */ SELECT DATABASE() |
+|                  |   47 | root | 10.16.xx.xx:35678   | 2023-12-29 16:21:56 | internal | test | Sleep   | 3528 | EOF   | f4944c543dc34a99-b0d0f3986c8f1c98 | select * from test                                                                    |
++------------------+------+------+--------------------+---------------------+----------+---------+---------+------+-------+-----------------------------------+---------------------------------------------------------------------------------------+
+5 rows in set (0.00 sec)
+```
+
+3. Terminate the running query. The running query will appear canceled.
+
+```sql
+mysql> kill query 55;
+Query OK, 0 rows affected (0.01 sec)
+```
+
+
 ### Keywords
 
     KILL
diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md b/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md
index d35cdfa3145839..56b820957be650 100644
--- a/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md
+++ b/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/KILL.md
@@ -34,7 +34,7 @@ KILL
 
 每个 Doris 的连接都在一个单独的线程中运行。 您可以使用 KILL processlist_id 语句终止线程。
 
-线程进程列表标识符可以从 INFORMATION_SCHEMA PROCESSLIST 表的 ID 列、SHOW PROCESSLIST 输出的 Id 列和性能模式线程表的 PROCESSLIST_ID 列确定。 
+线程进程列表标识符可以从 SHOW PROCESSLIST 输出的 Id 列查询 或者 SELECT CONNECTION_ID() 来查询当前connection id。 
 
 语法：
 
@@ -54,6 +54,41 @@ KILL QUERY processlist_id | query_id
 
 ### Example
 
+1. 查看当前连接的connection id。
+
+```sql
+mysql> select connection_id();
++-----------------+
+| connection_id() |
++-----------------+
+| 48              |
++-----------------+
+1 row in set (0.00 sec)
+```
+
+2. 查看所有连接的connection id。
+
+```sql
+mysql> SHOW PROCESSLIST;
++------------------+------+------+--------------------+---------------------+----------+---------+---------+------+-------+-----------------------------------+---------------------------------------------------------------------------------------+
+| CurrentConnected | Id   | User | Host               | LoginTime           | Catalog  | Db      | Command | Time | State | QueryId                           | Info                                                                                  |
++------------------+------+------+--------------------+---------------------+----------+---------+---------+------+-------+-----------------------------------+---------------------------------------------------------------------------------------+
+| Yes              |   48 | root | 10.16.xx.xx:44834   | 2023-12-29 16:49:47 | internal | test | Query   |    0 | OK    | e6e4ce9567b04859-8eeab8d6b5513e38 | SHOW PROCESSLIST                                                                      |
+|                  |   50 | root | 192.168.xx.xx:52837 | 2023-12-29 16:51:34 | internal |      | Sleep   | 1837 | EOF   | deaf13c52b3b4a3b-b25e8254b50ff8cb | SELECT @@session.transaction_isolation                                                |
+|                  |   51 | root | 192.168.xx.xx:52843 | 2023-12-29 16:51:35 | internal |      | Sleep   |  907 | EOF   | 437f219addc0404f-9befe7f6acf9a700 | /* ApplicationName=DBeaver Ultimate 23.1.3 - Metadata */ SHOW STATUS                  |
+|                  |   55 | root | 192.168.xx.xx:55533 | 2023-12-29 17:09:32 | internal | test | Sleep   |  271 | EOF   | f02603dc163a4da3-beebbb5d1ced760c | /* ApplicationName=DBeaver Ultimate 23.1.3 - SQLEditor <Console> */ SELECT DATABASE() |
+|                  |   47 | root | 10.16.xx.xx:35678   | 2023-12-29 16:21:56 | internal | test | Sleep   | 3528 | EOF   | f4944c543dc34a99-b0d0f3986c8f1c98 | select * from test                                                                    |
++------------------+------+------+--------------------+---------------------+----------+---------+---------+------+-------+-----------------------------------+---------------------------------------------------------------------------------------+
+5 rows in set (0.00 sec)
+```
+
+3. 终止正在运行的查询, 正在运行的查询会显示被取消。
+
+```sql
+mysql> kill query 55;
+Query OK, 0 rows affected (0.01 sec)
+```
+
 ### Keywords
 
     KILL

From eac96005b3350240c77321cd237935a2aa6c8a85 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Tue, 2 Jan 2024 14:06:16 +0800
Subject: [PATCH 243/299] [Fix](Job)Concurrency may result in event loss
 (#29385)

---
 .../main/java/org/apache/doris/job/disruptor/TaskDisruptor.java | 2 +-
 .../org/apache/doris/scheduler/disruptor/TaskDisruptor.java     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
index 0994a1d9211e0c..45564e99b17829 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/disruptor/TaskDisruptor.java
@@ -50,7 +50,7 @@ public TaskDisruptor(EventFactory<T> eventFactory, int ringBufferSize, ThreadFac
                          WaitStrategy waitStrategy, WorkHandler<T>[] workHandlers,
                          EventTranslatorVararg<T> eventTranslator) {
         disruptor = new Disruptor<>(eventFactory, ringBufferSize, threadFactory,
-                ProducerType.SINGLE, waitStrategy);
+                ProducerType.MULTI, waitStrategy);
         disruptor.handleEventsWithWorkerPool(workHandlers);
         this.eventTranslator = eventTranslator;
         disruptor.start();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java
index a20f772edaa1e4..dc065360d09f62 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/scheduler/disruptor/TaskDisruptor.java
@@ -81,7 +81,7 @@ public TaskDisruptor(TransientTaskManager transientTaskManager) {
     public void start() {
         ThreadFactory producerThreadFactory = DaemonThreadFactory.INSTANCE;
         disruptor = new Disruptor<>(TaskEvent.FACTORY, DEFAULT_RING_BUFFER_SIZE, producerThreadFactory,
-                ProducerType.SINGLE, new BlockingWaitStrategy());
+                ProducerType.MULTI, new BlockingWaitStrategy());
         WorkHandler<TaskEvent>[] workers = new TaskHandler[consumerThreadCount];
         for (int i = 0; i < consumerThreadCount; i++) {
             workers[i] = new TaskHandler(transientTaskManager);

From 4581618b09ec5e0e6633f919f2ad20a4ec83495e Mon Sep 17 00:00:00 2001
From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com>
Date: Tue, 2 Jan 2024 14:08:09 +0800
Subject: [PATCH 244/299] [improvement](disk) pick disk randomly when usage is
 less than 0.7 (#29368)

---
 be/src/olap/data_dir.cpp       |  3 +--
 be/src/olap/data_dir.h         |  7 +++++++
 be/src/olap/storage_engine.cpp | 38 +++++++++++++++++++++-------------
 3 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 154673dfb4ff95..d19e77f6e8e096 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -870,8 +870,7 @@ size_t DataDir::tablet_size() const {
 }
 
 bool DataDir::reach_capacity_limit(int64_t incoming_data_size) {
-    double used_pct = (_disk_capacity_bytes - _available_bytes + incoming_data_size) /
-                      (double)_disk_capacity_bytes;
+    double used_pct = get_usage(incoming_data_size);
     int64_t left_bytes = _available_bytes - incoming_data_size;
     if (used_pct >= config::storage_flood_stage_usage_percent / 100.0 &&
         left_bytes <= config::storage_flood_stage_left_capacity_bytes) {
diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h
index 84b79a8b2876b6..f8a405d4ffd4fd 100644
--- a/be/src/olap/data_dir.h
+++ b/be/src/olap/data_dir.h
@@ -133,6 +133,13 @@ class DataDir {
 
     void disks_compaction_num_increment(int64_t delta);
 
+    double get_usage(int64_t incoming_data_size) const {
+        return _disk_capacity_bytes == 0
+                       ? 0
+                       : (_disk_capacity_bytes - _available_bytes + incoming_data_size) /
+                                 (double)_disk_capacity_bytes;
+    }
+
     // Move tablet to trash.
     Status move_to_trash(const std::string& tablet_path);
 
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 4b3827daf29394..33844e899290d4 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -455,28 +455,38 @@ std::vector<DataDir*> StorageEngine::get_stores_for_create_tablet(
         std::lock_guard<std::mutex> l(_store_lock);
         for (auto& it : _store_map) {
             if (it.second->is_used()) {
-                if (_available_storage_medium_type_count == 1 ||
-                    it.second->storage_medium() == storage_medium) {
+                if ((_available_storage_medium_type_count == 1 ||
+                     it.second->storage_medium() == storage_medium) &&
+                    !it.second->reach_capacity_limit(0)) {
                     stores.push_back(it.second);
                 }
             }
         }
     }
-    std::random_device rd;
-    std::mt19937 g(rd());
-    std::shuffle(stores.begin(), stores.end(), g);
-    // Two random choices
-    for (int i = 0; i < stores.size(); i++) {
-        int j = i + 1;
-        if (j < stores.size()) {
-            if (stores[i]->tablet_size() > stores[j]->tablet_size()) {
-                std::swap(stores[i], stores[j]);
-            }
-            std::shuffle(stores.begin() + j, stores.end(), g);
-        } else {
+
+    std::sort(stores.begin(), stores.end(),
+              [](DataDir* a, DataDir* b) { return a->get_usage(0) < b->get_usage(0); });
+
+    size_t seventy_percent_index = stores.size();
+    size_t eighty_five_percent_index = stores.size();
+    for (size_t index = 0; index < stores.size(); index++) {
+        // If the usage of the store is less than 70%, we choose disk randomly.
+        if (stores[index]->get_usage(0) > 0.7 && seventy_percent_index == stores.size()) {
+            seventy_percent_index = index;
+        }
+        if (stores[index]->get_usage(0) > 0.85 && eighty_five_percent_index == stores.size()) {
+            eighty_five_percent_index = index;
             break;
         }
     }
+
+    std::random_device rd;
+    std::mt19937 g(rd());
+    std::shuffle(stores.begin(), stores.begin() + seventy_percent_index, g);
+    std::shuffle(stores.begin() + seventy_percent_index, stores.begin() + eighty_five_percent_index,
+                 g);
+    std::shuffle(stores.begin() + eighty_five_percent_index, stores.end(), g);
+
     return stores;
 }
 

From 2ed122b787f673797a4e9e20b4af4e4ece149fcd Mon Sep 17 00:00:00 2001
From: yiguolei <676222867@qq.com>
Date: Tue, 2 Jan 2024 15:28:27 +0800
Subject: [PATCH 245/299] [improvement](task exec context) add parent class
 HasTaskExecutionCtx to own the task ctx (#29388)

---------

Co-authored-by: yiguolei <yiguolei@gmail.com>
---
 be/src/pipeline/task_scheduler.cpp         |  3 +--
 be/src/runtime/task_execution_context.h    | 26 +++++++++++++++++++++-
 be/src/vec/exec/scan/scanner_context.cpp   | 10 ++++-----
 be/src/vec/exec/scan/scanner_context.h     |  6 ++---
 be/src/vec/exec/scan/scanner_scheduler.cpp |  4 ++--
 5 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp
index e814e4cdf2d092..91980aeec7a8d8 100644
--- a/be/src/pipeline/task_scheduler.cpp
+++ b/be/src/pipeline/task_scheduler.cpp
@@ -345,8 +345,7 @@ void TaskScheduler::_try_close_task(PipelineTask* task, PipelineTaskState state,
                                     Status exec_status) {
     // close_a_pipeline may delete fragment context and will core in some defer
     // code, because the defer code will access fragment context it self.
-    std::shared_ptr<TaskExecutionContext> lock_for_context =
-            task->fragment_context()->shared_from_this();
+    auto lock_for_context = task->fragment_context()->shared_from_this();
     auto status = task->try_close(exec_status);
     auto cancel = [&]() {
         task->query_context()->cancel(true, status.to_string(),
diff --git a/be/src/runtime/task_execution_context.h b/be/src/runtime/task_execution_context.h
index 08564840f59c46..c876ed5cb0d7b9 100644
--- a/be/src/runtime/task_execution_context.h
+++ b/be/src/runtime/task_execution_context.h
@@ -21,11 +21,35 @@
 
 namespace doris {
 
-// This class act as a super class of all context like things
+// This class act as a super class of all context like things such as
+// plan fragment executor or pipelinefragmentcontext or pipelinexfragmentcontext
 class TaskExecutionContext : public std::enable_shared_from_this<TaskExecutionContext> {
 public:
     TaskExecutionContext() = default;
     virtual ~TaskExecutionContext() = default;
 };
 
+using TaskExecutionContextSPtr = std::shared_ptr<TaskExecutionContext>;
+
+// Task Execution Context maybe plan fragment executor or pipelinefragmentcontext or pipelinexfragmentcontext
+// In multi thread scenario, the object is created in main thread (such as FragmentExecThread), but the object
+// maybe used in other thread(such as scanner thread, brpc->sender queue). If the main thread stopped and destroy
+// the object, then the other thread may core. So the other thread must lock the context to ensure the object exists.
+struct HasTaskExecutionCtx {
+    using Weak = typename TaskExecutionContextSPtr::weak_type;
+
+    HasTaskExecutionCtx(TaskExecutionContextSPtr task_exec_ctx) : task_exec_ctx_(task_exec_ctx) {}
+
+    // Init task ctx from state, the state has to own a method named get_task_execution_context()
+    // like runtime state
+    template <typename T>
+    HasTaskExecutionCtx(T* state) : task_exec_ctx_(state->get_task_execution_context()) {}
+
+public:
+    inline TaskExecutionContextSPtr task_exec_ctx() const { return task_exec_ctx_.lock(); }
+
+private:
+    Weak task_exec_ctx_;
+};
+
 } // namespace doris
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index 16bb1ce8487f4a..9c87967f505568 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -52,7 +52,8 @@ ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* outpu
                                pipeline::ScanLocalStateBase* local_state,
                                std::shared_ptr<pipeline::ScanDependency> dependency,
                                std::shared_ptr<pipeline::Dependency> finish_dependency)
-        : _state(state),
+        : HasTaskExecutionCtx(state),
+          _state(state),
           _parent(nullptr),
           _local_state(local_state),
           _output_tuple_desc(output_row_descriptor
@@ -72,8 +73,6 @@ ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* outpu
           _finish_dependency(finish_dependency) {
     DCHECK(_output_row_descriptor == nullptr ||
            _output_row_descriptor->tuple_descriptors().size() == 1);
-    // Use the task exec context as a lock between scanner threads and fragment exection threads
-    _task_exec_ctx = _state->get_task_execution_context();
     _query_id = _state->get_query_ctx()->query_id();
     ctx_id = UniqueId::gen_uid().to_string();
     if (_scanners.empty()) {
@@ -102,7 +101,8 @@ ScannerContext::ScannerContext(doris::RuntimeState* state, doris::vectorized::VS
                                const std::list<VScannerSPtr>& scanners, int64_t limit_,
                                int64_t max_bytes_in_blocks_queue, const int num_parallel_instances,
                                pipeline::ScanLocalStateBase* local_state)
-        : _state(state),
+        : HasTaskExecutionCtx(state),
+          _state(state),
           _parent(parent),
           _local_state(local_state),
           _output_tuple_desc(output_row_descriptor
@@ -120,8 +120,6 @@ ScannerContext::ScannerContext(doris::RuntimeState* state, doris::vectorized::VS
           _num_parallel_instances(num_parallel_instances) {
     DCHECK(_output_row_descriptor == nullptr ||
            _output_row_descriptor->tuple_descriptors().size() == 1);
-    // Use the task exec context as a lock between scanner threads and fragment exection threads
-    _task_exec_ctx = _state->get_task_execution_context();
     _query_id = _state->get_query_ctx()->query_id();
     ctx_id = UniqueId::gen_uid().to_string();
     if (_scanners.empty()) {
diff --git a/be/src/vec/exec/scan/scanner_context.h b/be/src/vec/exec/scan/scanner_context.h
index 035d396bf65660..8e840a47465882 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -65,7 +65,8 @@ class SimplifiedScanScheduler;
 // ScannerContext is also the scheduling unit of ScannerScheduler.
 // ScannerScheduler schedules a ScannerContext at a time,
 // and submits the Scanners to the scanner thread pool for data scanning.
-class ScannerContext : public std::enable_shared_from_this<ScannerContext> {
+class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
+                       public HasTaskExecutionCtx {
     ENABLE_FACTORY_CREATOR(ScannerContext);
 
 public:
@@ -180,8 +181,6 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext> {
 
     bool _should_reset_thread_name = true;
 
-    std::weak_ptr<TaskExecutionContext> get_task_execution_context() { return _task_exec_ctx; }
-
 private:
     template <typename Parent>
     Status _close_and_clear_scanners(Parent* parent, RuntimeState* state);
@@ -199,7 +198,6 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext> {
     void _set_scanner_done();
 
     RuntimeState* _state = nullptr;
-    std::weak_ptr<TaskExecutionContext> _task_exec_ctx;
     VScanNode* _parent = nullptr;
     pipeline::ScanLocalStateBase* _local_state = nullptr;
 
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 6ec83e8bd6aec8..29b53b39353bba 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -172,7 +172,7 @@ void ScannerScheduler::_schedule_thread(int queue_id) {
 }
 
 void ScannerScheduler::_schedule_scanners(std::shared_ptr<ScannerContext> ctx) {
-    auto task_lock = ctx->get_task_execution_context().lock();
+    auto task_lock = ctx->task_exec_ctx();
     if (task_lock == nullptr) {
         // LOG(WARNING) << "could not lock task execution context, query " << print_id(_query_id)
         //             << " maybe finished";
@@ -266,7 +266,7 @@ void ScannerScheduler::_schedule_scanners(std::shared_ptr<ScannerContext> ctx) {
 
 void ScannerScheduler::_scanner_scan(ScannerScheduler* scheduler,
                                      std::shared_ptr<ScannerContext> ctx, VScannerSPtr scanner) {
-    auto task_lock = ctx->get_task_execution_context().lock();
+    auto task_lock = ctx->task_exec_ctx();
     if (task_lock == nullptr) {
         // LOG(WARNING) << "could not lock task execution context, query " << print_id(_query_id)
         //             << " maybe finished";

From c07e3b7b47d98ab77569258d6bea01e3fa26ba3b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= <jianxie0@gmail.com>
Date: Tue, 2 Jan 2024 15:50:49 +0800
Subject: [PATCH 246/299] [feature](Nereids): eliminate left outer join by
 unique (#28853)

---
 .../doris/nereids/jobs/executor/Rewriter.java |  7 +-
 .../apache/doris/nereids/rules/RuleType.java  |  1 +
 .../rules/rewrite/EliminateJoinByUnique.java  | 48 +++++++++
 .../rewrite/EliminateJoinByUniqueTest.java    | 97 +++++++++++++++++++
 4 files changed, 152 insertions(+), 1 deletion(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java
 create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 4fe9df664c39b5..a97bd61e5c611a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -57,6 +57,7 @@
 import org.apache.doris.nereids.rules.rewrite.EliminateFilter;
 import org.apache.doris.nereids.rules.rewrite.EliminateGroupBy;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinByFK;
+import org.apache.doris.nereids.rules.rewrite.EliminateJoinByUnique;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinCondition;
 import org.apache.doris.nereids.rules.rewrite.EliminateLimit;
 import org.apache.doris.nereids.rules.rewrite.EliminateNotNull;
@@ -295,7 +296,11 @@ public class Rewriter extends AbstractBatchJobExecutor {
             ),
 
             // this rule should invoke after infer predicate and push down distinct, and before push down limit
-            custom(RuleType.ELIMINATE_JOIN_BY_FOREIGN_KEY, EliminateJoinByFK::new),
+            topic("eliminate join according unique or foreign key",
+                custom(RuleType.ELIMINATE_JOIN_BY_FOREIGN_KEY, EliminateJoinByFK::new),
+                topDown(new EliminateJoinByUnique())
+            ),
+
             // this rule should be after topic "Column pruning and infer predicate"
             topic("Join pull up",
                     topDown(
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 32794946dda19b..57e62380137c54 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -208,6 +208,7 @@ public enum RuleType {
     ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE),
     ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE),
     ELIMINATE_GROUP_BY(RuleTypeClass.REWRITE),
+    ELIMINATE_JOIN_BY_UK(RuleTypeClass.REWRITE),
     ELIMINATE_DEDUP_JOIN_CONDITION(RuleTypeClass.REWRITE),
     ELIMINATE_NULL_AWARE_LEFT_ANTI_JOIN(RuleTypeClass.REWRITE),
     ELIMINATE_ASSERT_NUM_ROWS(RuleTypeClass.REWRITE),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java
new file mode 100644
index 00000000000000..6dba90572d550e
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUnique.java
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+
+/**
+ * Eliminate outer join.
+ */
+public class EliminateJoinByUnique extends OneRewriteRuleFactory {
+    @Override
+    public Rule build() {
+        return logicalProject(
+                logicalJoin().when(join -> join.getJoinType().isLeftOuterJoin())
+        ).then(project -> {
+            LogicalJoin<?, ?> join = project.child();
+            if (!join.left().getOutputSet().containsAll(project.getInputSlots())) {
+                return project;
+            }
+            if (join.getHashJoinConjuncts().stream().anyMatch(NullSafeEqual.class::isInstance)) {
+                // TODO: support null safe equals in fd and this
+                return project;
+            }
+            if (!project.getLogicalProperties().getFunctionalDependencies().isUnique(project.getOutputSet())) {
+                return project;
+            }
+            return project.withChildren(join.left());
+        }).toRule(RuleType.ELIMINATE_JOIN_BY_UK);
+    }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java
new file mode 100644
index 00000000000000..c3b9c9de0056e6
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByUniqueTest.java
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.utframe.TestWithFeService;
+
+import org.junit.jupiter.api.Test;
+
+class EliminateJoinByUniqueTest extends TestWithFeService implements MemoPatternMatchSupported {
+    @Override
+    protected void runBeforeAll() throws Exception {
+        createDatabase("test");
+        connectContext.setDatabase("default_cluster:test");
+        createTables(
+                "CREATE TABLE IF NOT EXISTS t1 (\n"
+                        + "    id1 int not null,\n"
+                        + "    id_null int\n"
+                        + ")\n"
+                        + "DUPLICATE KEY(id1)\n"
+                        + "DISTRIBUTED BY HASH(id1) BUCKETS 10\n"
+                        + "PROPERTIES (\"replication_num\" = \"1\")\n",
+                "CREATE TABLE IF NOT EXISTS t2 (\n"
+                        + "    id2 int not null\n"
+                        + ")\n"
+                        + "DUPLICATE KEY(id2)\n"
+                        + "DISTRIBUTED BY HASH(id2) BUCKETS 10\n"
+                        + "PROPERTIES (\"replication_num\" = \"1\")\n");
+        addConstraint("Alter table t1 add constraint uk_t1 unique (id1)");
+        addConstraint("Alter table t1 add constraint id_null unique (id_null)");
+        addConstraint("Alter table t2 add constraint uk_t2 unique (id2)");
+    }
+
+    @Test
+    void testNotNull() throws Exception {
+        String sql = "select t1.id1 from t1 left outer join t2 on t1.id1 = t2.id2";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .nonMatch(logicalJoin())
+                .printlnTree();
+
+        sql = "select t1.id1 from t1 left outer join t2 on t1.id1 <=> t2.id2";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(logicalJoin())
+                .printlnTree();
+
+        sql = "select t2.id2 from t1 left outer join t2 on t1.id1 = t2.id2";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(logicalJoin())
+                .printlnTree();
+    }
+
+    @Test
+    void testNull() throws Exception {
+        String sql = "select t1.id1 from t1 left outer join t2 on t1.id_null = t2.id2";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .nonMatch(logicalJoin())
+                .printlnTree();
+
+        sql = "select t1.id1 from t1 left outer join t2 on t1.id_null <=> t2.id2";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(logicalJoin())
+                .printlnTree();
+
+        sql = "select t2.id2 from t1 left outer join t2 on t1.id_null = t2.id2";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matches(logicalJoin())
+                .printlnTree();
+    }
+}

From 706463781c3fc6891d9bd22f3acc7b87e1123b32 Mon Sep 17 00:00:00 2001
From: meiyi <myimeiyi@gmail.com>
Date: Tue, 2 Jan 2024 15:52:03 +0800
Subject: [PATCH 247/299] [refactor](group commit) refactor group commit wal
 code (#29375)

---
 be/src/http/action/http_stream.cpp            |  1 -
 be/src/http/action/stream_load.cpp            |  1 -
 be/src/http/utils.cpp                         |  2 +-
 be/src/olap/{ => wal}/wal_dirs_info.cpp       | 42 ++-------
 be/src/olap/{ => wal}/wal_dirs_info.h         | 11 +--
 be/src/olap/{ => wal}/wal_info.cpp            |  9 +-
 be/src/olap/{ => wal}/wal_info.h              |  2 +-
 be/src/olap/{ => wal}/wal_manager.cpp         | 48 ++++------
 be/src/olap/{ => wal}/wal_manager.h           | 92 +++++++++++--------
 be/src/olap/{ => wal}/wal_reader.cpp          |  2 +-
 be/src/olap/{ => wal}/wal_reader.h            |  0
 be/src/olap/{ => wal}/wal_table.cpp           | 16 ++--
 be/src/olap/{ => wal}/wal_table.h             | 39 ++++----
 be/src/olap/{ => wal}/wal_writer.cpp          |  8 +-
 be/src/olap/{ => wal}/wal_writer.h            |  6 --
 be/src/runtime/exec_env_init.cpp              |  2 +-
 be/src/runtime/group_commit_mgr.cpp           | 13 ---
 be/src/runtime/group_commit_mgr.h             |  2 +-
 be/src/service/internal_service.cpp           |  2 +-
 .../format/arrow/arrow_pip_input_stream.cpp   |  1 -
 .../exec/format/arrow/arrow_stream_reader.cpp |  1 -
 be/src/vec/exec/format/wal/wal_reader.cpp     |  6 +-
 be/src/vec/exec/format/wal/wal_reader.h       |  9 +-
 be/src/vec/sink/volap_table_sink.h            |  1 -
 be/src/vec/sink/writer/vtablet_writer.cpp     |  1 -
 be/src/vec/sink/writer/vwal_writer.cpp        | 18 ----
 be/src/vec/sink/writer/vwal_writer.h          | 61 +-----------
 be/test/http/stream_load_test.cpp             |  2 +-
 be/test/olap/wal_manager_test.cpp             |  2 +-
 be/test/olap/wal_reader_writer_test.cpp       |  4 +-
 be/test/vec/exec/vtablet_sink_test.cpp        |  2 +-
 be/test/vec/exec/vwal_scanner_test.cpp        |  2 +-
 .../doris/planner/GroupCommitPlanner.java     |  2 +
 33 files changed, 156 insertions(+), 254 deletions(-)
 rename be/src/olap/{ => wal}/wal_dirs_info.cpp (88%)
 rename be/src/olap/{ => wal}/wal_dirs_info.h (92%)
 rename be/src/olap/{ => wal}/wal_info.cpp (96%)
 rename be/src/olap/{ => wal}/wal_info.h (97%)
 rename be/src/olap/{ => wal}/wal_manager.cpp (95%)
 rename be/src/olap/{ => wal}/wal_manager.h (85%)
 rename be/src/olap/{ => wal}/wal_reader.cpp (99%)
 rename be/src/olap/{ => wal}/wal_reader.h (100%)
 rename be/src/olap/{ => wal}/wal_table.cpp (98%)
 rename be/src/olap/{ => wal}/wal_table.h (91%)
 rename be/src/olap/{ => wal}/wal_writer.cpp (96%)
 rename be/src/olap/{ => wal}/wal_writer.h (91%)

diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp
index 15d2a1a18d1bed..a9652707834312 100644
--- a/be/src/http/action/http_stream.cpp
+++ b/be/src/http/action/http_stream.cpp
@@ -46,7 +46,6 @@
 #include "http/utils.h"
 #include "io/fs/stream_load_pipe.h"
 #include "olap/storage_engine.h"
-#include "olap/wal_manager.h"
 #include "runtime/client_cache.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index a702be65d05523..6f6b7ad6b03af6 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -51,7 +51,6 @@
 #include "http/utils.h"
 #include "io/fs/stream_load_pipe.h"
 #include "olap/storage_engine.h"
-#include "olap/wal_manager.h"
 #include "runtime/client_cache.h"
 #include "runtime/exec_env.h"
 #include "runtime/group_commit_mgr.h"
diff --git a/be/src/http/utils.cpp b/be/src/http/utils.cpp
index 9fbf4c79300dcd..72b80f875e208c 100644
--- a/be/src/http/utils.cpp
+++ b/be/src/http/utils.cpp
@@ -39,7 +39,7 @@
 #include "http/http_status.h"
 #include "io/fs/file_system.h"
 #include "io/fs/local_file_system.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/exec_env.h"
 #include "util/path_util.h"
 #include "util/url_coding.h"
diff --git a/be/src/olap/wal_dirs_info.cpp b/be/src/olap/wal/wal_dirs_info.cpp
similarity index 88%
rename from be/src/olap/wal_dirs_info.cpp
rename to be/src/olap/wal/wal_dirs_info.cpp
index 1b7216b0724598..19ad2562778047 100644
--- a/be/src/olap/wal_dirs_info.cpp
+++ b/be/src/olap/wal/wal_dirs_info.cpp
@@ -15,11 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/wal_dirs_info.h"
-
-#include <memory>
-#include <mutex>
-#include <shared_mutex>
+#include "olap/wal/wal_dirs_info.h"
 
 #include "common/config.h"
 #include "common/status.h"
@@ -37,36 +33,23 @@ size_t WalDirInfo::get_limit() {
     return _limit;
 }
 
-size_t WalDirInfo::get_used() {
-    std::shared_lock rlock(_lock);
-    return _used;
-}
-
-size_t WalDirInfo::get_pre_allocated() {
-    std::shared_lock rlock(_lock);
-    return _pre_allocated;
-}
-
-Status WalDirInfo::set_limit(size_t limit) {
+void WalDirInfo::set_limit(size_t limit) {
     std::unique_lock wlock(_lock);
     _limit = limit;
-    return Status::OK();
 }
 
-Status WalDirInfo::set_used(size_t used) {
+void WalDirInfo::set_used(size_t used) {
     std::unique_lock wlock(_lock);
     _used = used;
-    return Status::OK();
 }
 
-Status WalDirInfo::set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated) {
+void WalDirInfo::set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated) {
     std::unique_lock wlock(_lock);
     if (is_add_pre_allocated) {
         _pre_allocated += pre_allocated;
     } else {
         _pre_allocated -= pre_allocated;
     }
-    return Status::OK();
 }
 
 size_t WalDirInfo::available() {
@@ -77,7 +60,7 @@ size_t WalDirInfo::available() {
 
 Status WalDirInfo::update_wal_dir_limit(size_t limit) {
     if (limit != static_cast<size_t>(-1)) {
-        RETURN_IF_ERROR(set_limit(limit));
+        set_limit(limit);
     } else {
         size_t available_bytes;
         size_t disk_capacity_bytes;
@@ -91,24 +74,25 @@ Status WalDirInfo::update_wal_dir_limit(size_t limit) {
         }
         size_t wal_dir_size = 0;
         RETURN_IF_ERROR(io::global_local_filesystem()->directory_size(_wal_dir, &wal_dir_size));
-        RETURN_IF_ERROR(set_limit(wal_disk_limit));
+        // TODO should be wal_disk_limit + wal_dir_size
+        set_limit(wal_disk_limit);
     }
     return Status::OK();
 }
 
 Status WalDirInfo::update_wal_dir_used(size_t used) {
     if (used != static_cast<size_t>(-1)) {
-        RETURN_IF_ERROR(set_used(used));
+        set_used(used);
     } else {
         size_t wal_dir_size = 0;
         RETURN_IF_ERROR(io::global_local_filesystem()->directory_size(_wal_dir, &wal_dir_size));
-        RETURN_IF_ERROR(set_used(wal_dir_size));
+        set_used(wal_dir_size);
     }
     return Status::OK();
 }
 
 Status WalDirInfo::update_wal_dir_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated) {
-    RETURN_IF_ERROR(set_pre_allocated(pre_allocated, is_add_pre_allocated));
+    set_pre_allocated(pre_allocated, is_add_pre_allocated);
     return Status::OK();
 }
 
@@ -128,12 +112,6 @@ Status WalDirsInfo::add(const std::string& wal_dir, size_t limit, size_t used,
     return Status::OK();
 }
 
-Status WalDirsInfo::clear() {
-    std::unique_lock wlock(_lock);
-    _wal_dirs_info_vec.clear();
-    return Status::OK();
-}
-
 std::string WalDirsInfo::get_available_random_wal_dir() {
     if (_wal_dirs_info_vec.size() == 1) {
         return (*_wal_dirs_info_vec.begin())->get_wal_dir();
diff --git a/be/src/olap/wal_dirs_info.h b/be/src/olap/wal/wal_dirs_info.h
similarity index 92%
rename from be/src/olap/wal_dirs_info.h
rename to be/src/olap/wal/wal_dirs_info.h
index 048cd8f9564362..91a26af2b5d02a 100644
--- a/be/src/olap/wal_dirs_info.h
+++ b/be/src/olap/wal/wal_dirs_info.h
@@ -1,4 +1,3 @@
-
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
@@ -41,11 +40,10 @@ class WalDirInfo {
               _pre_allocated(pre_allocated) {}
     std::string get_wal_dir();
     size_t get_limit();
-    size_t get_used();
-    size_t get_pre_allocated();
-    Status set_limit(size_t limit);
-    Status set_used(size_t used);
-    Status set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated);
+    void set_limit(size_t limit);
+    void set_used(size_t used);
+    // TODO increase_pre_allocated and decrease_pre_allocated
+    void set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated);
     size_t available();
     Status update_wal_dir_limit(size_t limit = -1);
     Status update_wal_dir_used(size_t used = -1);
@@ -66,7 +64,6 @@ class WalDirsInfo {
     WalDirsInfo() = default;
     ~WalDirsInfo() = default;
     Status add(const std::string& wal_dir, size_t limit, size_t used, size_t pre_allocated);
-    Status clear();
     std::string get_available_random_wal_dir();
     size_t get_max_available_size();
     Status update_wal_dir_limit(std::string wal_dir, size_t limit = -1);
diff --git a/be/src/olap/wal_info.cpp b/be/src/olap/wal/wal_info.cpp
similarity index 96%
rename from be/src/olap/wal_info.cpp
rename to be/src/olap/wal/wal_info.cpp
index d93593cfaf026d..3c6fc4190b4c07 100644
--- a/be/src/olap/wal_info.cpp
+++ b/be/src/olap/wal/wal_info.cpp
@@ -15,26 +15,31 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/wal_info.h"
+#include "olap/wal/wal_info.h"
+
 namespace doris {
 WalInfo::WalInfo(int64_t wal_id, std::string wal_path, int64_t retry_num, int64_t start_time_ms)
         : _wal_id(wal_id),
           _wal_path(wal_path),
           _retry_num(retry_num),
           _start_time_ms(start_time_ms) {}
-WalInfo::~WalInfo() {}
+
 int64_t WalInfo::get_wal_id() {
     return _wal_id;
 }
+
 std::string WalInfo::get_wal_path() {
     return _wal_path;
 }
+
 int64_t WalInfo::get_retry_num() {
     return _retry_num;
 }
+
 int64_t WalInfo::get_start_time_ms() {
     return _start_time_ms;
 }
+
 void WalInfo::add_retry_num() {
     _retry_num++;
 }
diff --git a/be/src/olap/wal_info.h b/be/src/olap/wal/wal_info.h
similarity index 97%
rename from be/src/olap/wal_info.h
rename to be/src/olap/wal/wal_info.h
index 0383ac68f2ce36..0365e23abd49a8 100644
--- a/be/src/olap/wal_info.h
+++ b/be/src/olap/wal/wal_info.h
@@ -21,7 +21,7 @@ namespace doris {
 class WalInfo {
 public:
     WalInfo(int64_t wal_id, std::string wal_path, int64_t retry_num, int64_t start_time_ms);
-    ~WalInfo();
+    ~WalInfo() = default;
     int64_t get_wal_id();
     int64_t get_retry_num();
     int64_t get_start_time_ms();
diff --git a/be/src/olap/wal_manager.cpp b/be/src/olap/wal/wal_manager.cpp
similarity index 95%
rename from be/src/olap/wal_manager.cpp
rename to be/src/olap/wal/wal_manager.cpp
index a9c35794698698..9af91cf7b55348 100644
--- a/be/src/olap/wal_manager.cpp
+++ b/be/src/olap/wal/wal_manager.cpp
@@ -15,41 +15,31 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 
 #include <glog/logging.h>
-#include <thrift/protocol/TDebugProtocol.h>
 
-#include <atomic>
 #include <chrono>
-#include <cstdint>
 #include <filesystem>
-#include <memory>
-#include <mutex>
 #include <shared_mutex>
 #include <thread>
 #include <unordered_map>
-#include <utility>
 #include <vector>
 
 #include "common/config.h"
 #include "common/status.h"
 #include "io/fs/local_file_system.h"
-#include "olap/wal_dirs_info.h"
-#include "olap/wal_writer.h"
-#include "runtime/client_cache.h"
+#include "olap/wal/wal_dirs_info.h"
+#include "olap/wal/wal_writer.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/plan_fragment_executor.h"
-#include "runtime/stream_load/stream_load_context.h"
 #include "util/parse_util.h"
-#include "util/path_util.h"
-#include "util/thrift_rpc_helper.h"
 #include "vec/exec/format/wal/wal_reader.h"
 
 namespace doris {
 WalManager::WalManager(ExecEnv* exec_env, const std::string& wal_dir_list)
-        : _exec_env(exec_env), _stop_background_threads_latch(1), _stop(false) {
+        : _exec_env(exec_env), _stop(false), _stop_background_threads_latch(1) {
     doris::vectorized::WalReader::string_split(wal_dir_list, ";", _wal_dirs);
     static_cast<void>(ThreadPoolBuilder("GroupCommitReplayWalThreadPool")
                               .set_min_threads(1)
@@ -62,10 +52,14 @@ WalManager::~WalManager() {
     LOG(INFO) << "WalManager is destoried";
 }
 
+bool WalManager::is_running() {
+    return !_stop.load();
+}
+
 void WalManager::stop() {
     if (!this->_stop.load()) {
         this->_stop.store(true);
-        stop_relay_wal();
+        _stop_relay_wal();
         _stop_background_threads_latch.count_down();
         if (_replay_thread) {
             _replay_thread->join();
@@ -83,10 +77,10 @@ Status WalManager::init() {
     RETURN_IF_ERROR(_init_wal_dirs());
     RETURN_IF_ERROR(_init_wal_dirs_info());
     for (auto wal_dir : _wal_dirs) {
-        RETURN_IF_ERROR(scan_wals(wal_dir));
+        RETURN_IF_ERROR(_scan_wals(wal_dir));
     }
     return Thread::create(
-            "WalMgr", "replay_wal", [this]() { static_cast<void>(this->replay()); },
+            "WalMgr", "replay_wal", [this]() { static_cast<void>(this->_replay()); },
             &_replay_thread);
 }
 
@@ -276,6 +270,7 @@ Status WalManager::create_wal_reader(const std::string& wal_path,
 Status WalManager::create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>& wal_writer) {
     std::string wal_path;
     RETURN_IF_ERROR(get_wal_path(wal_id, wal_path));
+    // TODO move the create_dir into wal_writer::init
     std::vector<std::string> path_element;
     doris::vectorized::WalReader::string_split(wal_path, "/", path_element);
     std::stringstream ss;
@@ -292,13 +287,14 @@ Status WalManager::create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>&
     wal_writer = std::make_shared<WalWriter>(wal_path);
     RETURN_IF_ERROR(wal_writer->init());
     {
+        // TODO no use, should remove it
         std::lock_guard<std::shared_mutex> wrlock(_wal_lock);
         _wal_id_to_writer_map.emplace(wal_id, wal_writer);
     }
     return Status::OK();
 }
 
-Status WalManager::scan_wals(const std::string& wal_path) {
+Status WalManager::_scan_wals(const std::string& wal_path) {
     size_t count = 0;
     bool exists = true;
     std::vector<io::FileInfo> dbs;
@@ -360,7 +356,7 @@ Status WalManager::scan_wals(const std::string& wal_path) {
     return Status::OK();
 }
 
-Status WalManager::replay() {
+Status WalManager::_replay() {
     do {
         if (_stop.load()) {
             break;
@@ -372,7 +368,7 @@ Status WalManager::replay() {
         }
         std::vector<int64_t> replay_tables;
         {
-            std::lock_guard<std::shared_mutex> wrlock(_lock);
+            std::lock_guard<std::shared_mutex> wrlock(_table_lock);
             auto it = _table_map.begin();
             while (it != _table_map.end()) {
                 if (it->second->size() > 0) {
@@ -396,7 +392,7 @@ Status WalManager::replay() {
 
 Status WalManager::add_recover_wal(int64_t db_id, int64_t table_id, int64_t wal_id,
                                    std::string wal) {
-    std::lock_guard<std::shared_mutex> wrlock(_lock);
+    std::lock_guard<std::shared_mutex> wrlock(_table_lock);
     std::shared_ptr<WalTable> table_ptr;
     auto it = _table_map.find(table_id);
     if (it == _table_map.end()) {
@@ -414,7 +410,7 @@ Status WalManager::add_recover_wal(int64_t db_id, int64_t table_id, int64_t wal_
 }
 
 size_t WalManager::get_wal_table_size(int64_t table_id) {
-    std::shared_lock rdlock(_lock);
+    std::shared_lock rdlock(_table_lock);
     auto it = _table_map.find(table_id);
     if (it != _table_map.end()) {
         return it->second->size();
@@ -440,12 +436,8 @@ Status WalManager::delete_wal(int64_t wal_id, size_t block_queue_pre_allocated)
     return Status::OK();
 }
 
-bool WalManager::is_running() {
-    return !_stop.load();
-}
-
-void WalManager::stop_relay_wal() {
-    std::lock_guard<std::shared_mutex> wrlock(_lock);
+void WalManager::_stop_relay_wal() {
+    std::lock_guard<std::shared_mutex> wrlock(_table_lock);
     for (auto it = _table_map.begin(); it != _table_map.end(); it++) {
         it->second->stop();
     }
diff --git a/be/src/olap/wal_manager.h b/be/src/olap/wal/wal_manager.h
similarity index 85%
rename from be/src/olap/wal_manager.h
rename to be/src/olap/wal/wal_manager.h
index 4b42beaf45551e..f4e28445f8f786 100644
--- a/be/src/olap/wal_manager.h
+++ b/be/src/olap/wal/wal_manager.h
@@ -33,10 +33,10 @@
 #include "gen_cpp/FrontendService_types.h"
 #include "gen_cpp/HeartbeatService_types.h"
 #include "gutil/ref_counted.h"
-#include "olap/wal_dirs_info.h"
-#include "olap/wal_reader.h"
-#include "olap/wal_table.h"
-#include "olap/wal_writer.h"
+#include "olap/wal/wal_dirs_info.h"
+#include "olap/wal/wal_reader.h"
+#include "olap/wal/wal_table.h"
+#include "olap/wal/wal_writer.h"
 #include "runtime/exec_env.h"
 #include "runtime/stream_load/stream_load_context.h"
 #include "util/thread.h"
@@ -49,55 +49,62 @@ class WalManager {
 public:
     enum WalStatus {
         PREPARE = 0,
-        REPLAY,
-        CREATE,
+        REPLAY = 1,
+        CREATE = 2,
     };
 
 public:
     WalManager(ExecEnv* exec_env, const std::string& wal_dir);
     ~WalManager();
-    // used for wal
-    Status delete_wal(int64_t wal_id, size_t block_queue_pre_allocated = 0);
     Status init();
-    Status scan_wals(const std::string& wal_path);
-    Status replay();
-    Status create_wal_reader(const std::string& wal_path, std::shared_ptr<WalReader>& wal_reader);
-    Status create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>& wal_writer);
-    Status scan();
-    size_t get_wal_table_size(int64_t table_id);
-    Status add_recover_wal(int64_t db_id, int64_t table_id, int64_t wal_id, std::string wal);
+    bool is_running();
+    void stop();
+
+    // wal back pressure
+    Status update_wal_dir_limit(const std::string& wal_dir, size_t limit = -1);
+    Status update_wal_dir_used(const std::string& wal_dir, size_t used = -1);
+    Status update_wal_dir_pre_allocated(const std::string& wal_dir, size_t pre_allocated,
+                                        bool is_add_pre_allocated);
+    Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
+    size_t get_max_available_size();
+
+    // replay wal
     Status add_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id, const std::string& label,
                         std::string& base_path);
     Status get_wal_path(int64_t wal_id, std::string& wal_path);
+    Status delete_wal(int64_t wal_id, size_t block_queue_pre_allocated = 0);
+    Status add_recover_wal(int64_t db_id, int64_t table_id, int64_t wal_id, std::string wal);
+    // used for ut
+    size_t get_wal_table_size(int64_t table_id);
+    // TODO util function, should remove
+    Status create_wal_reader(const std::string& wal_path, std::shared_ptr<WalReader>& wal_reader);
+    Status create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>& wal_writer);
+
+    // for wal status, can be removed
     Status get_wal_status_queue_size(const PGetWalQueueSizeRequest* request,
                                      PGetWalQueueSizeResponse* response);
     void add_wal_status_queue(int64_t table_id, int64_t wal_id, WalStatus wal_status);
     Status erase_wal_status_queue(int64_t table_id, int64_t wal_id);
     void print_wal_status_queue();
-    void stop();
-    bool is_running();
-    void stop_relay_wal();
+
+    // for _wal_column_id_map, can be removed
     void add_wal_column_index(int64_t wal_id, std::vector<size_t>& column_index);
     void erase_wal_column_index(int64_t wal_id);
     Status get_wal_column_index(int64_t wal_id, std::vector<size_t>& column_index);
 
-    // used for limit
-    Status update_wal_dir_limit(const std::string& wal_dir, size_t limit = -1);
-    Status update_wal_dir_used(const std::string& wal_dir, size_t used = -1);
-    Status update_wal_dir_pre_allocated(const std::string& wal_dir, size_t pre_allocated,
-                                        bool is_add_pre_allocated);
-    Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
-    size_t get_max_available_size();
-
 private:
-    // used for limit
+    // wal back pressure
     Status _init_wal_dirs_conf();
     Status _init_wal_dirs();
     Status _init_wal_dirs_info();
     std::string _get_base_wal_path(const std::string& wal_path_str);
-    const std::string& _get_available_random_wal_dir();
     Status _update_wal_dir_info_thread();
 
+    // replay wal
+    Status _scan_wals(const std::string& wal_path);
+    Status _replay();
+    void _stop_relay_wal();
+
 public:
     // used for be ut
     size_t wal_limit_test_bytes;
@@ -105,24 +112,33 @@ class WalManager {
     const std::string tmp = "tmp";
 
 private:
-    //used for wal
     ExecEnv* _exec_env = nullptr;
-    std::shared_mutex _lock;
-    scoped_refptr<Thread> _replay_thread;
+    std::atomic<bool> _stop;
     CountDownLatch _stop_background_threads_latch;
-    std::map<int64_t, std::shared_ptr<WalTable>> _table_map;
+
+    // wal back pressure
     std::vector<std::string> _wal_dirs;
+    scoped_refptr<Thread> _update_wal_dirs_info_thread;
+    std::unique_ptr<WalDirsInfo> _wal_dirs_info;
+
+    // replay wal
+    scoped_refptr<Thread> _replay_thread;
+    std::unique_ptr<doris::ThreadPool> _thread_pool;
+
+    std::shared_mutex _table_lock;
+    std::map<int64_t, std::shared_ptr<WalTable>> _table_map;
+
     std::shared_mutex _wal_lock;
-    std::shared_mutex _wal_status_lock;
     std::unordered_map<int64_t, std::string> _wal_path_map;
+    // TODO no use? need remove it. And the map dose not clear
     std::unordered_map<int64_t, std::shared_ptr<WalWriter>> _wal_id_to_writer_map;
+
+    // TODO Now only used for debug wal status, consider remove it
+    std::shared_mutex _wal_status_lock;
     std::unordered_map<int64_t, std::unordered_map<int64_t, WalStatus>> _wal_status_queues;
-    std::atomic<bool> _stop;
+
+    // TODO should remove
     std::shared_mutex _wal_column_id_map_lock;
     std::unordered_map<int64_t, std::vector<size_t>&> _wal_column_id_map;
-    std::unique_ptr<doris::ThreadPool> _thread_pool;
-    // used for limit
-    scoped_refptr<Thread> _update_wal_dirs_info_thread;
-    std::unique_ptr<WalDirsInfo> _wal_dirs_info;
 };
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_reader.cpp b/be/src/olap/wal/wal_reader.cpp
similarity index 99%
rename from be/src/olap/wal_reader.cpp
rename to be/src/olap/wal/wal_reader.cpp
index c3a9b225dd7f20..75bbaf10947ef5 100644
--- a/be/src/olap/wal_reader.cpp
+++ b/be/src/olap/wal/wal_reader.cpp
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/wal_reader.h"
+#include "olap/wal/wal_reader.h"
 
 #include "common/status.h"
 #include "io/fs/file_reader.h"
diff --git a/be/src/olap/wal_reader.h b/be/src/olap/wal/wal_reader.h
similarity index 100%
rename from be/src/olap/wal_reader.h
rename to be/src/olap/wal/wal_reader.h
diff --git a/be/src/olap/wal_table.cpp b/be/src/olap/wal/wal_table.cpp
similarity index 98%
rename from be/src/olap/wal_table.cpp
rename to be/src/olap/wal/wal_table.cpp
index a1c9a12f3e1750..f89794a4586961 100644
--- a/be/src/olap/wal_table.cpp
+++ b/be/src/olap/wal/wal_table.cpp
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/wal_table.h"
+#include "olap/wal/wal_table.h"
 
 #include <thrift/protocol/TDebugProtocol.h>
 
@@ -27,7 +27,7 @@
 #include "http/utils.h"
 #include "io/fs/local_file_system.h"
 #include "io/fs/stream_load_pipe.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/client_cache.h"
 #include "runtime/fragment_mgr.h"
 #include "runtime/plan_fragment_executor.h"
@@ -53,7 +53,8 @@ void WalTable::add_wal(int64_t wal_id, std::string wal) {
     auto wal_info = std::make_shared<WalInfo>(wal_id, wal, 0, UnixMillis());
     _replay_wal_map.emplace(wal, wal_info);
 }
-void WalTable::pick_relay_wals() {
+
+void WalTable::_pick_relay_wals() {
     std::lock_guard<std::mutex> lock(_replay_wal_lock);
     std::vector<std::string> need_replay_wals;
     std::vector<std::string> need_erase_wals;
@@ -84,7 +85,7 @@ void WalTable::pick_relay_wals() {
     }
 }
 
-Status WalTable::relay_wal_one_by_one() {
+Status WalTable::_relay_wal_one_by_one() {
     std::vector<std::shared_ptr<WalInfo>> need_retry_wals;
     std::vector<std::shared_ptr<WalInfo>> need_delete_wals;
     while (!_replaying_queue.empty()) {
@@ -124,6 +125,7 @@ Status WalTable::relay_wal_one_by_one() {
     }
     return Status::OK();
 }
+
 Status WalTable::replay_wals() {
     {
         std::lock_guard<std::mutex> lock(_replay_wal_lock);
@@ -138,8 +140,8 @@ Status WalTable::replay_wals() {
     }
     VLOG_DEBUG << "Start replay wals for db=" << _db_id << ", table=" << _table_id
                << ", wal size=" << _replay_wal_map.size();
-    pick_relay_wals();
-    RETURN_IF_ERROR(relay_wal_one_by_one());
+    _pick_relay_wals();
+    RETURN_IF_ERROR(_relay_wal_one_by_one());
     return Status::OK();
 }
 
@@ -184,6 +186,7 @@ Status WalTable::_try_abort_txn(int64_t db_id, int64_t wal_id) {
     TLoadTxnRollbackRequest request;
     request.__set_auth_code(0); // this is a fake, fe not check it now
     request.__set_db_id(db_id);
+    // TODO should we use label, because the replay wal use the same label and different wal_id
     request.__set_txnId(wal_id);
     std::string reason = "relay wal " + std::to_string(wal_id);
     request.__set_reason(reason);
@@ -265,6 +268,7 @@ Status WalTable::_construct_sql_str(const std::string& wal, const std::string& l
     sql_str = ss.str().data();
     return Status::OK();
 }
+
 Status WalTable::_handle_stream_load(int64_t wal_id, const std::string& wal,
                                      const std::string& label) {
     std::string sql_str;
diff --git a/be/src/olap/wal_table.h b/be/src/olap/wal/wal_table.h
similarity index 91%
rename from be/src/olap/wal_table.h
rename to be/src/olap/wal/wal_table.h
index 251e8d51a61244..66ee4fd37219e7 100644
--- a/be/src/olap/wal_table.h
+++ b/be/src/olap/wal/wal_table.h
@@ -25,9 +25,10 @@
 #include "gen_cpp/FrontendService_types.h"
 #include "gen_cpp/HeartbeatService_types.h"
 #include "http/action/http_stream.h"
-#include "olap/wal_info.h"
+#include "olap/wal/wal_info.h"
 #include "runtime/exec_env.h"
 #include "runtime/stream_load/stream_load_context.h"
+
 namespace doris {
 class WalTable {
 public:
@@ -35,44 +36,46 @@ class WalTable {
     ~WalTable();
     // used when be start and there are wals need to do recovery
     void add_wal(int64_t wal_id, std::string wal);
-    void pick_relay_wals();
-    Status relay_wal_one_by_one();
     Status replay_wals();
     size_t size();
     void stop();
 
-public:
+private:
     // <column_name, column_index>
     using ColumnInfo = std::pair<std::string, int64_t>;
 
-private:
+    void _pick_relay_wals();
+    bool _need_replay(std::shared_ptr<WalInfo>);
+    Status _relay_wal_one_by_one();
+    Status _delete_wal(int64_t wal_id);
+    Status _rename_to_tmp_path(const std::string wal);
+
+    Status _replay_wal_internal(const std::string& wal);
+    // TODO change the param: (wal, int64_t* wal_id, std::string* label)
     Status _parse_wal_path(const std::string& wal,
                            std::shared_ptr<std::pair<int64_t, std::string>>&);
-    Status _rename_to_tmp_path(const std::string wal);
-    Status _replay_one_txn_with_stremaload(int64_t wal_id, const std::string& wal,
-                                           const std::string& label);
     Status _try_abort_txn(int64_t db_id, int64_t wal_id);
     Status _get_column_info(int64_t db_id, int64_t tb_id);
-    Status _read_wal_header(const std::string& wal, std::string& columns);
-    bool _need_replay(std::shared_ptr<WalInfo>);
-    Status _replay_wal_internal(const std::string& wal);
-    Status _delete_wal(int64_t wal_id);
+
+    Status _replay_one_txn_with_stremaload(int64_t wal_id, const std::string& wal,
+                                           const std::string& label);
+    Status _handle_stream_load(int64_t wal_id, const std::string& wal, const std::string& label);
     Status _construct_sql_str(const std::string& wal, const std::string& label,
                               std::string& sql_str, std::vector<size_t>& index_vector);
-    Status _handle_stream_load(int64_t wal_id, const std::string& wal, const std::string& label);
+    Status _read_wal_header(const std::string& wal, std::string& columns);
 
 private:
     ExecEnv* _exec_env;
     int64_t _db_id;
     int64_t _table_id;
-    std::string _relay = "relay";
-    std::string _split = "_";
+    // TODO the stop is not used?
+    std::atomic<bool> _stop;
+    std::shared_ptr<HttpStreamAction> _http_stream_action;
     mutable std::mutex _replay_wal_lock;
-    // key is wal_id
+    // key is wal_path
     std::map<std::string, std::shared_ptr<WalInfo>> _replay_wal_map;
     std::list<std::shared_ptr<WalInfo>> _replaying_queue;
-    std::atomic<bool> _stop;
+    // TODO should not use this map
     std::map<int64_t, std::shared_ptr<ColumnInfo>> _column_id_info_map;
-    std::shared_ptr<HttpStreamAction> _http_stream_action;
 };
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/olap/wal_writer.cpp b/be/src/olap/wal/wal_writer.cpp
similarity index 96%
rename from be/src/olap/wal_writer.cpp
rename to be/src/olap/wal/wal_writer.cpp
index beae6ec80b6c23..25f7cc0870b6eb 100644
--- a/be/src/olap/wal_writer.cpp
+++ b/be/src/olap/wal/wal_writer.cpp
@@ -15,18 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "olap/wal_writer.h"
-
-#include <atomic>
-#include <condition_variable>
-#include <memory>
+#include "olap/wal/wal_writer.h"
 
 #include "common/config.h"
 #include "io/fs/file_writer.h"
 #include "io/fs/local_file_system.h"
 #include "io/fs/path.h"
 #include "olap/storage_engine.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "util/crc32c.h"
 
 namespace doris {
diff --git a/be/src/olap/wal_writer.h b/be/src/olap/wal/wal_writer.h
similarity index 91%
rename from be/src/olap/wal_writer.h
rename to be/src/olap/wal/wal_writer.h
index ea8bba4f021e54..08d2a4eb710208 100644
--- a/be/src/olap/wal_writer.h
+++ b/be/src/olap/wal/wal_writer.h
@@ -17,11 +17,6 @@
 
 #pragma once
 
-#include <atomic>
-#include <condition_variable>
-#include <memory>
-#include <mutex>
-
 #include "common/status.h"
 #include "gen_cpp/internal_service.pb.h"
 #include "io/fs/file_reader_writer_fwd.h"
@@ -51,7 +46,6 @@ class WalWriter {
     static const int64_t VERSION_SIZE = 4;
 
 private:
-    static constexpr size_t MAX_WAL_WRITE_WAIT_TIME = 1000;
     std::string _file_name;
     io::FileWriterPtr _file_writer;
 };
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 2a1233c9b5d6b1..eae038aaa466a8 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -49,7 +49,7 @@
 #include "olap/segment_loader.h"
 #include "olap/storage_engine.h"
 #include "olap/tablet_schema_cache.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "pipeline/task_queue.h"
 #include "pipeline/task_scheduler.h"
 #include "runtime/block_spill_manager.h"
diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index b965efadd70598..69f150b6c6055a 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -20,24 +20,11 @@
 #include <gen_cpp/Types_types.h>
 #include <glog/logging.h>
 
-#include <atomic>
-#include <chrono>
-#include <cstddef>
-#include <cstdint>
-#include <memory>
-#include <mutex>
-#include <shared_mutex>
-#include <vector>
-
 #include "client_cache.h"
 #include "common/config.h"
-#include "common/status.h"
-#include "olap/wal_manager.h"
 #include "runtime/exec_env.h"
 #include "runtime/fragment_mgr.h"
-#include "runtime/runtime_state.h"
 #include "util/thrift_rpc_helper.h"
-#include "vec/core/block.h"
 
 namespace doris {
 
diff --git a/be/src/runtime/group_commit_mgr.h b/be/src/runtime/group_commit_mgr.h
index 03c917f8fcc48e..bf89aa2aa50a43 100644
--- a/be/src/runtime/group_commit_mgr.h
+++ b/be/src/runtime/group_commit_mgr.h
@@ -29,7 +29,7 @@
 #include <utility>
 
 #include "common/status.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/exec_env.h"
 #include "util/threadpool.h"
 #include "vec/core/block.h"
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index ba2686c0fd76c3..a92775a6f6e19e 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -83,7 +83,7 @@
 #include "olap/tablet_schema.h"
 #include "olap/txn_manager.h"
 #include "olap/utils.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/buffer_control_block.h"
 #include "runtime/cache/result_cache.h"
 #include "runtime/define_primitive_type.h"
diff --git a/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp b/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp
index 1c9edb84a4be54..7bc32c7ab4bb91 100644
--- a/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp
+++ b/be/src/vec/exec/format/arrow/arrow_pip_input_stream.cpp
@@ -27,7 +27,6 @@
 #include "arrow/result.h"
 #include "common/logging.h"
 #include "io/fs/stream_load_pipe.h"
-#include "olap/wal_manager.h"
 #include "runtime/runtime_state.h"
 
 namespace doris::vectorized {
diff --git a/be/src/vec/exec/format/arrow/arrow_stream_reader.cpp b/be/src/vec/exec/format/arrow/arrow_stream_reader.cpp
index 33ac46a8279443..d1a1a9d35c8861 100644
--- a/be/src/vec/exec/format/arrow/arrow_stream_reader.cpp
+++ b/be/src/vec/exec/format/arrow/arrow_stream_reader.cpp
@@ -27,7 +27,6 @@
 #include "arrow_pip_input_stream.h"
 #include "common/logging.h"
 #include "io/fs/stream_load_pipe.h"
-#include "olap/wal_manager.h"
 #include "runtime/descriptors.h"
 #include "runtime/runtime_state.h"
 #include "vec/utils/arrow_column_to_doris_column.h"
diff --git a/be/src/vec/exec/format/wal/wal_reader.cpp b/be/src/vec/exec/format/wal/wal_reader.cpp
index 32fd66cc309629..7344b4cbd9bc57 100644
--- a/be/src/vec/exec/format/wal/wal_reader.cpp
+++ b/be/src/vec/exec/format/wal/wal_reader.cpp
@@ -18,23 +18,27 @@
 #include "wal_reader.h"
 
 #include "common/logging.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/runtime_state.h"
 #include "vec/data_types/data_type_string.h"
+
 namespace doris::vectorized {
 WalReader::WalReader(RuntimeState* state) : _state(state) {
     _wal_id = state->wal_id();
 }
+
 WalReader::~WalReader() {
     if (_wal_reader.get() != nullptr) {
         static_cast<void>(_wal_reader->finalize());
     }
 }
+
 Status WalReader::init_reader() {
     RETURN_IF_ERROR(_state->exec_env()->wal_mgr()->get_wal_path(_wal_id, _wal_path));
     RETURN_IF_ERROR(_state->exec_env()->wal_mgr()->create_wal_reader(_wal_path, _wal_reader));
     return Status::OK();
 }
+
 Status WalReader::get_next_block(Block* block, size_t* read_rows, bool* eof) {
     //read src block
     PBlock pblock;
diff --git a/be/src/vec/exec/format/wal/wal_reader.h b/be/src/vec/exec/format/wal/wal_reader.h
index ed55d6e166096b..78f2c31d925d79 100644
--- a/be/src/vec/exec/format/wal/wal_reader.h
+++ b/be/src/vec/exec/format/wal/wal_reader.h
@@ -16,8 +16,9 @@
 // under the License.
 
 #pragma once
-#include "olap/wal_reader.h"
+#include "olap/wal/wal_reader.h"
 #include "vec/exec/format/generic_reader.h"
+
 namespace doris {
 namespace vectorized {
 struct ScannerCounter;
@@ -29,16 +30,16 @@ class WalReader : public GenericReader {
     Status get_next_block(Block* block, size_t* read_rows, bool* eof) override;
     Status get_columns(std::unordered_map<std::string, TypeDescriptor>* name_to_type,
                        std::unordered_set<std::string>* missing_cols) override;
+    // TODO move it
     static void string_split(const std::string& str, const std::string& splits,
                              std::vector<std::string>& res);
-    std::vector<size_t> get_index() { return _column_index; }
 
 private:
     RuntimeState* _state = nullptr;
-    std::string _wal_path;
-    std::string _path_split = "/";
     int64_t _wal_id;
+    std::string _wal_path;
     std::shared_ptr<doris::WalReader> _wal_reader;
+    // TODO version should in olap/wal_reader
     uint32_t _version = 0;
     std::string _col_ids;
     std::vector<size_t> _column_index;
diff --git a/be/src/vec/sink/volap_table_sink.h b/be/src/vec/sink/volap_table_sink.h
index 68315eb5a9b82d..add285cdfdd7df 100644
--- a/be/src/vec/sink/volap_table_sink.h
+++ b/be/src/vec/sink/volap_table_sink.h
@@ -57,7 +57,6 @@
 #include "exec/data_sink.h"
 #include "exec/tablet_info.h"
 #include "gutil/ref_counted.h"
-#include "olap/wal_writer.h"
 #include "runtime/decimalv2_value.h"
 #include "runtime/exec_env.h"
 #include "runtime/memory/mem_tracker.h"
diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index 40b1c544e237cd..89519ad6ca0ff8 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -47,7 +47,6 @@
 #include <vector>
 
 #include "common/config.h"
-#include "olap/wal_manager.h"
 #include "util/runtime_profile.h"
 #include "vec/data_types/data_type.h"
 #include "vec/exprs/vexpr_fwd.h"
diff --git a/be/src/vec/sink/writer/vwal_writer.cpp b/be/src/vec/sink/writer/vwal_writer.cpp
index 2ab37c340a0512..c697d373f1cd21 100644
--- a/be/src/vec/sink/writer/vwal_writer.cpp
+++ b/be/src/vec/sink/writer/vwal_writer.cpp
@@ -19,25 +19,7 @@
 
 #include <gen_cpp/data.pb.h>
 
-#include <mutex>
 #include <sstream>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "common/compiler_util.h"
-#include "common/status.h"
-#include "runtime/client_cache.h"
-#include "runtime/descriptors.h"
-#include "runtime/runtime_state.h"
-#include "util/doris_metrics.h"
-#include "util/network_util.h"
-#include "util/proto_util.h"
-#include "util/thrift_util.h"
-#include "vec/common/assert_cast.h"
-#include "vec/core/block.h"
-#include "vec/sink/vtablet_block_convertor.h"
-#include "vec/sink/vtablet_finder.h"
 
 namespace doris {
 namespace vectorized {
diff --git a/be/src/vec/sink/writer/vwal_writer.h b/be/src/vec/sink/writer/vwal_writer.h
index 324409e9d469a1..659383737333bf 100644
--- a/be/src/vec/sink/writer/vwal_writer.h
+++ b/be/src/vec/sink/writer/vwal_writer.h
@@ -16,68 +16,14 @@
 // under the License.
 
 #pragma once
-#include <brpc/controller.h>
-#include <bthread/types.h>
-#include <butil/errno.h>
-#include <fmt/format.h>
-#include <gen_cpp/Exprs_types.h>
-#include <gen_cpp/FrontendService.h>
-#include <gen_cpp/FrontendService_types.h>
-#include <gen_cpp/PaloInternalService_types.h>
-#include <gen_cpp/Types_types.h>
-#include <gen_cpp/internal_service.pb.h>
-#include <gen_cpp/types.pb.h>
-#include <glog/logging.h>
-#include <google/protobuf/stubs/callback.h>
-#include <stddef.h>
-#include <stdint.h>
-
-#include <atomic>
-#include <chrono>
-#include <cstdint>
-#include <functional>
-#include <initializer_list>
-#include <map>
-#include <memory>
-#include <mutex>
-#include <ostream>
-#include <queue>
-#include <set>
-#include <sstream>
 #include <string>
-#include <thread>
-#include <unordered_map>
-#include <unordered_set>
-#include <utility>
 #include <vector>
 
-#include "common/config.h"
 #include "common/status.h"
-#include "exec/data_sink.h"
-#include "exec/tablet_info.h"
-#include "gutil/ref_counted.h"
-#include "olap/wal_manager.h"
-#include "olap/wal_writer.h"
-#include "runtime/decimalv2_value.h"
-#include "runtime/exec_env.h"
-#include "runtime/memory/mem_tracker.h"
-#include "runtime/thread_context.h"
-#include "runtime/types.h"
-#include "util/countdown_latch.h"
-#include "util/ref_count_closure.h"
-#include "util/runtime_profile.h"
-#include "util/spinlock.h"
-#include "util/stopwatch.hpp"
-#include "vec/columns/column.h"
-#include "vec/common/allocator.h"
+#include "olap/wal/wal_manager.h"
+#include "olap/wal/wal_writer.h"
 #include "vec/core/block.h"
-#include "vec/data_types/data_type.h"
-#include "vec/exprs/vexpr_fwd.h"
-#include "vec/runtime/vfile_format_transformer.h"
-#include "vec/sink/vrow_distribution.h"
-#include "vec/sink/vtablet_block_convertor.h"
-#include "vec/sink/vtablet_finder.h"
-#include "vec/sink/writer/async_result_writer.h"
+
 namespace doris {
 namespace vectorized {
 
@@ -94,6 +40,7 @@ class VWalWriter {
 private:
     int64_t _tb_id;
     int64_t _wal_id;
+    // TODO version should in olap/wal_writer
     uint32_t _version = 0;
     std::string _label;
     WalManager* _wal_manager;
diff --git a/be/test/http/stream_load_test.cpp b/be/test/http/stream_load_test.cpp
index 74dcf9f1f3ff67..881154e857b231 100644
--- a/be/test/http/stream_load_test.cpp
+++ b/be/test/http/stream_load_test.cpp
@@ -35,7 +35,7 @@
 #include "http/http_headers.h"
 #include "http/http_request.h"
 #include "http/utils.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/exec_env.h"
 
 namespace doris {
diff --git a/be/test/olap/wal_manager_test.cpp b/be/test/olap/wal_manager_test.cpp
index c5c216f12a795a..d5db9bc75034ff 100644
--- a/be/test/olap/wal_manager_test.cpp
+++ b/be/test/olap/wal_manager_test.cpp
@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 
 #include <gtest/gtest.h>
 
diff --git a/be/test/olap/wal_reader_writer_test.cpp b/be/test/olap/wal_reader_writer_test.cpp
index d24db7286802c7..47a93fa7f25834 100644
--- a/be/test/olap/wal_reader_writer_test.cpp
+++ b/be/test/olap/wal_reader_writer_test.cpp
@@ -24,8 +24,8 @@
 #include "gen_cpp/internal_service.pb.h"
 #include "gmock/gmock.h"
 #include "io/fs/local_file_system.h"
-#include "olap/wal_reader.h"
-#include "olap/wal_writer.h"
+#include "olap/wal/wal_reader.h"
+#include "olap/wal/wal_writer.h"
 #include "runtime/exec_env.h"
 #include "service/brpc.h"
 #include "testutil/test_util.h"
diff --git a/be/test/vec/exec/vtablet_sink_test.cpp b/be/test/vec/exec/vtablet_sink_test.cpp
index f0b8e8c96433ef..51ad8660ea8dc2 100644
--- a/be/test/vec/exec/vtablet_sink_test.cpp
+++ b/be/test/vec/exec/vtablet_sink_test.cpp
@@ -33,7 +33,7 @@
 #include "gtest/gtest_pred_impl.h"
 #include "io/fs/local_file_system.h"
 #include "olap/olap_define.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/decimalv2_value.h"
 #include "runtime/define_primitive_type.h"
 #include "runtime/descriptor_helper.h"
diff --git a/be/test/vec/exec/vwal_scanner_test.cpp b/be/test/vec/exec/vwal_scanner_test.cpp
index ac066aceb98955..2d786679d4e6cd 100644
--- a/be/test/vec/exec/vwal_scanner_test.cpp
+++ b/be/test/vec/exec/vwal_scanner_test.cpp
@@ -23,7 +23,7 @@
 #include "gen_cpp/Descriptors_types.h"
 #include "gen_cpp/PlanNodes_types.h"
 #include "io/fs/local_file_system.h"
-#include "olap/wal_manager.h"
+#include "olap/wal/wal_manager.h"
 #include "runtime/descriptors.h"
 #include "runtime/memory/mem_tracker.h"
 #include "runtime/runtime_state.h"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
index 659314263099fb..8b9f6b18331971 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/GroupCommitPlanner.java
@@ -107,6 +107,7 @@ public GroupCommitPlanner(Database db, OlapTable table, List<String> targetColum
         StreamLoadTask streamLoadTask = StreamLoadTask.fromTStreamLoadPutRequest(streamLoadPutRequest);
         StreamLoadPlanner planner = new StreamLoadPlanner(db, table, streamLoadTask);
         // Will using load id as query id in fragment
+        // TODO support pipeline
         TExecPlanFragmentParams tRequest = planner.plan(streamLoadTask.getId());
         for (Map.Entry<Integer, List<TScanRangeParams>> entry : tRequest.params.per_node_scan_ranges.entrySet()) {
             for (TScanRangeParams scanRangeParams : entry.getValue()) {
@@ -116,6 +117,7 @@ public GroupCommitPlanner(Database db, OlapTable table, List<String> targetColum
                         TFileCompressType.PLAIN);
             }
         }
+        tRequest.query_options.setEnablePipelineEngine(false);
         List<TScanRangeParams> scanRangeParams = tRequest.params.per_node_scan_ranges.values().stream()
                 .flatMap(Collection::stream).collect(Collectors.toList());
         Preconditions.checkState(scanRangeParams.size() == 1);

From 4692a62e47616e7d19bf0132d76a45c7c490982d Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Tue, 2 Jan 2024 15:53:28 +0800
Subject: [PATCH 248/299] [fix](planner)nullable info of agg node's
 intermediate and output slots should be same (#29315)

---
 .../apache/doris/planner/AggregationNode.java |  1 +
 .../correctness_p0/test_agg_materialize.out   |  3 +
 .../test_agg_materialize.groovy               | 55 +++++++++++++++++++
 3 files changed, 59 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
index f0e31157d3f957..23522625bb8da2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AggregationNode.java
@@ -374,6 +374,7 @@ public void finalize(Analyzer analyzer) throws UserException {
         List<Expr> groupingExprs = aggInfo.getGroupingExprs();
         for (int i = 0; i < groupingExprs.size(); i++) {
             aggInfo.getOutputTupleDesc().getSlots().get(i).setIsNullable(groupingExprs.get(i).isNullable());
+            aggInfo.getIntermediateTupleDesc().getSlots().get(i).setIsNullable(groupingExprs.get(i).isNullable());
             aggInfo.getOutputTupleDesc().computeMemLayout();
         }
     }
diff --git a/regression-test/data/correctness_p0/test_agg_materialize.out b/regression-test/data/correctness_p0/test_agg_materialize.out
index 26c36a0045e46b..d74ca4811b48a2 100644
--- a/regression-test/data/correctness_p0/test_agg_materialize.out
+++ b/regression-test/data/correctness_p0/test_agg_materialize.out
@@ -6,3 +6,6 @@
 1
 qycs
 
+-- !selectx --
+1	1
+
diff --git a/regression-test/suites/correctness_p0/test_agg_materialize.groovy b/regression-test/suites/correctness_p0/test_agg_materialize.groovy
index eb44b85fafc522..06c0e26b8c581b 100644
--- a/regression-test/suites/correctness_p0/test_agg_materialize.groovy
+++ b/regression-test/suites/correctness_p0/test_agg_materialize.groovy
@@ -100,4 +100,59 @@ suite("test_agg_materialize") {
                     ORDER BY SPJ.org_code;
                     """
     sql """drop table if exists c5749_bug_t;"""
+
+    sql """drop table if exists table_test_count_distinct;"""
+    sql """CREATE TABLE `table_test_count_distinct` (
+        `special_zone_name` VARCHAR(65532) NULL,
+        `day` VARCHAR(65532) NOT NULL 
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`special_zone_name`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`special_zone_name`) BUCKETS 1
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1"
+        );"""
+    sql """insert into
+            table_test_count_distinct
+            values
+            (
+                'a',
+                "2023-12-01"
+            );"""
+    qt_selectx"""SELECT
+                (COUNT(DISTINCT descript)) as c475859316,
+                (COUNT(DISTINCT day)) as c125327027
+                FROM
+                (
+                    SELECT
+                    descript,
+                    day
+                    FROM
+                    (
+                        with tmp2 as (
+                        select
+                            special_zone_name as descript,
+                            cast(day as datev2) as day
+                        from
+                            table_test_count_distinct
+                        union
+                        all
+                        select
+                            special_zone_name as descript,
+                            
+                            cast(day as datev2) as day
+                        from
+                            table_test_count_distinct
+                        )
+                        select
+                        t1.descript,
+                        t1.day
+                        from
+                        tmp2 t1
+
+                    ) t665446993
+                ) t1600566476
+                GROUP BY
+                day;"""
+    sql """drop table if exists table_test_count_distinct;"""
 }

From b3f69211b7313208b74fd00f664b1a08a2bea6f7 Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Tue, 2 Jan 2024 15:53:38 +0800
Subject: [PATCH 249/299] [refactor](wal) move group commit load content length
 to runtime state (#29188)

---
 be/src/http/action/http_stream.cpp          |  3 +-
 be/src/http/action/stream_load.cpp          |  3 +-
 be/src/runtime/group_commit_mgr.cpp         | 48 ++-------------------
 be/src/runtime/group_commit_mgr.h           |  8 +---
 be/src/runtime/plan_fragment_executor.cpp   |  3 ++
 be/src/runtime/runtime_state.h              |  7 ++-
 be/src/vec/sink/group_commit_block_sink.cpp | 17 ++++++--
 be/src/vec/sink/group_commit_block_sink.h   |  1 +
 gensrc/thrift/PaloInternalService.thrift    |  2 +
 9 files changed, 31 insertions(+), 61 deletions(-)

diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp
index a9652707834312..b97ce2976eb8c8 100644
--- a/be/src/http/action/http_stream.cpp
+++ b/be/src/http/action/http_stream.cpp
@@ -345,8 +345,7 @@ Status HttpStreamAction::process_put(HttpRequest* http_req,
                 ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
                 content_length *= 3;
             }
-            RETURN_IF_ERROR(ExecEnv::GetInstance()->group_commit_mgr()->update_load_info(
-                    ctx->id.to_thrift(), content_length));
+            ctx->put_result.params.__set_content_length(content_length);
         }
     }
 
diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index 6f6b7ad6b03af6..88e12e19dca449 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -644,8 +644,7 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
                 ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
                 content_length *= 3;
             }
-            RETURN_IF_ERROR(ExecEnv::GetInstance()->group_commit_mgr()->update_load_info(
-                    ctx->id.to_thrift(), content_length));
+            ctx->put_result.params.__set_content_length(content_length);
         }
     }
 
diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index 69f150b6c6055a..0c3589edbe0379 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -498,33 +498,17 @@ Status LoadBlockQueue::close_wal() {
     return Status::OK();
 }
 
-bool LoadBlockQueue::has_enough_wal_disk_space(
-        const std::vector<std::shared_ptr<vectorized::Block>>& blocks, const TUniqueId& load_id,
-        bool is_blocks_contain_all_load_data) {
-    size_t blocks_size = 0;
-    for (auto block : blocks) {
-        blocks_size += block->bytes();
-    }
-    size_t content_length = 0;
-    Status st = ExecEnv::GetInstance()->group_commit_mgr()->get_load_info(load_id, &content_length);
-    if (st.ok()) {
-        RETURN_IF_ERROR(ExecEnv::GetInstance()->group_commit_mgr()->remove_load_info(load_id));
-    } else {
-        return Status::InternalError("can not find load id.");
-    }
-    size_t pre_allocated = is_blocks_contain_all_load_data
-                                   ? blocks_size
-                                   : (blocks_size > content_length ? blocks_size : content_length);
+bool LoadBlockQueue::has_enough_wal_disk_space(size_t pre_allocated) {
     auto* wal_mgr = ExecEnv::GetInstance()->wal_mgr();
     size_t available_bytes = 0;
     {
-        st = wal_mgr->get_wal_dir_available_size(wal_base_path, &available_bytes);
+        Status st = wal_mgr->get_wal_dir_available_size(wal_base_path, &available_bytes);
         if (!st.ok()) {
             LOG(WARNING) << "get wal disk available size filed!";
         }
     }
     if (pre_allocated < available_bytes) {
-        st = wal_mgr->update_wal_dir_pre_allocated(wal_base_path, pre_allocated, true);
+        Status st = wal_mgr->update_wal_dir_pre_allocated(wal_base_path, pre_allocated, true);
         if (!st.ok()) {
             LOG(WARNING) << "update wal dir pre_allocated failed, reason: " << st.to_string();
         }
@@ -534,30 +518,4 @@ bool LoadBlockQueue::has_enough_wal_disk_space(
         return false;
     }
 }
-
-Status GroupCommitMgr::update_load_info(TUniqueId load_id, size_t content_length) {
-    std::unique_lock l(_load_info_lock);
-    if (_load_id_to_content_length_map.find(load_id) == _load_id_to_content_length_map.end()) {
-        _load_id_to_content_length_map.insert(std::make_pair(load_id, content_length));
-    }
-    return Status::OK();
-}
-
-Status GroupCommitMgr::get_load_info(TUniqueId load_id, size_t* content_length) {
-    std::shared_lock l(_load_info_lock);
-    if (_load_id_to_content_length_map.find(load_id) != _load_id_to_content_length_map.end()) {
-        *content_length = _load_id_to_content_length_map[load_id];
-        return Status::OK();
-    }
-    return Status::InternalError("can not find load id!");
-}
-
-Status GroupCommitMgr::remove_load_info(TUniqueId load_id) {
-    std::unique_lock l(_load_info_lock);
-    if (_load_id_to_content_length_map.find(load_id) == _load_id_to_content_length_map.end()) {
-        return Status::InternalError("can not remove load id!");
-    }
-    _load_id_to_content_length_map.erase(load_id);
-    return Status::OK();
-}
 } // namespace doris
diff --git a/be/src/runtime/group_commit_mgr.h b/be/src/runtime/group_commit_mgr.h
index bf89aa2aa50a43..125256535fec30 100644
--- a/be/src/runtime/group_commit_mgr.h
+++ b/be/src/runtime/group_commit_mgr.h
@@ -66,8 +66,7 @@ class LoadBlockQueue {
                       WalManager* wal_manager, std::vector<TSlotDescriptor>& slot_desc,
                       int be_exe_version);
     Status close_wal();
-    bool has_enough_wal_disk_space(const std::vector<std::shared_ptr<vectorized::Block>>& blocks,
-                                   const TUniqueId& load_id, bool is_blocks_contain_all_load_data);
+    bool has_enough_wal_disk_space(size_t pre_allocated);
 
     // 1s
     static constexpr size_t MAX_BLOCK_QUEUE_ADD_WAIT_TIME = 1000;
@@ -157,9 +156,6 @@ class GroupCommitMgr {
                                       const UniqueId& load_id,
                                       std::shared_ptr<LoadBlockQueue>& load_block_queue,
                                       int be_exe_version);
-    Status update_load_info(TUniqueId load_id, size_t content_length);
-    Status get_load_info(TUniqueId load_id, size_t* content_length);
-    Status remove_load_info(TUniqueId load_id);
 
 private:
     ExecEnv* _exec_env = nullptr;
@@ -170,8 +166,6 @@ class GroupCommitMgr {
     std::unique_ptr<doris::ThreadPool> _thread_pool;
     // memory consumption of all tables' load block queues, used for back pressure.
     std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
-    std::shared_mutex _load_info_lock;
-    std::unordered_map<TUniqueId, size_t> _load_id_to_content_length_map;
 };
 
 } // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/plan_fragment_executor.cpp b/be/src/runtime/plan_fragment_executor.cpp
index dcbf6346fddf85..896178946e3a68 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -149,6 +149,9 @@ Status PlanFragmentExecutor::prepare(const TExecPlanFragmentParams& request) {
     if (request.__isset.wal_id) {
         _runtime_state->set_wal_id(request.wal_id);
     }
+    if (request.__isset.content_length) {
+        _runtime_state->set_content_length(request.content_length);
+    }
 
     if (request.query_options.__isset.is_report_success) {
         _is_report_success = request.query_options.is_report_success;
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index ee515007eb604c..1470ec89776b9b 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -255,7 +255,11 @@ class RuntimeState {
 
     void set_wal_id(int64_t wal_id) { _wal_id = wal_id; }
 
-    int64_t wal_id() { return _wal_id; }
+    int64_t wal_id() const { return _wal_id; }
+
+    void set_content_length(size_t content_length) { _content_length = content_length; }
+
+    size_t content_length() const { return _content_length; }
 
     const std::string& import_label() { return _import_label; }
 
@@ -659,6 +663,7 @@ class RuntimeState {
     std::string _load_dir;
     int64_t _load_job_id;
     int64_t _wal_id = -1;
+    size_t _content_length = 0;
 
     // mini load
     int64_t _normal_row_number;
diff --git a/be/src/vec/sink/group_commit_block_sink.cpp b/be/src/vec/sink/group_commit_block_sink.cpp
index 277b9859bd528e..75ffa13233e9e1 100644
--- a/be/src/vec/sink/group_commit_block_sink.cpp
+++ b/be/src/vec/sink/group_commit_block_sink.cpp
@@ -66,8 +66,6 @@ Status GroupCommitBlockSink::init(const TDataSink& t_sink) {
 
 Status GroupCommitBlockSink::prepare(RuntimeState* state) {
     RETURN_IF_ERROR(DataSink::prepare(state));
-    RETURN_IF_ERROR(
-            ExecEnv::GetInstance()->group_commit_mgr()->update_load_info(_load_id.to_thrift(), 0));
     _state = state;
 
     // profile must add to state's object pool
@@ -240,8 +238,8 @@ Status GroupCommitBlockSink::_add_blocks(RuntimeState* state,
                     _db_id, _table_id, _base_schema_version, load_id, _load_block_queue,
                     _state->be_exec_version()));
             if (_group_commit_mode == TGroupCommitMode::ASYNC_MODE) {
-                _group_commit_mode = _load_block_queue->has_enough_wal_disk_space(
-                                             _blocks, load_id, is_blocks_contain_all_load_data)
+                size_t pre_allocated = _pre_allocated(is_blocks_contain_all_load_data);
+                _group_commit_mode = _load_block_queue->has_enough_wal_disk_space(pre_allocated)
                                              ? TGroupCommitMode::ASYNC_MODE
                                              : TGroupCommitMode::SYNC_MODE;
                 if (_group_commit_mode == TGroupCommitMode::SYNC_MODE) {
@@ -265,5 +263,16 @@ Status GroupCommitBlockSink::_add_blocks(RuntimeState* state,
     return Status::OK();
 }
 
+size_t GroupCommitBlockSink::_pre_allocated(bool is_blocks_contain_all_load_data) {
+    size_t blocks_size = 0;
+    for (auto block : _blocks) {
+        blocks_size += block->bytes();
+    }
+    return is_blocks_contain_all_load_data
+                   ? blocks_size
+                   : (blocks_size > _state->content_length() ? blocks_size
+                                                             : _state->content_length());
+}
+
 } // namespace vectorized
 } // namespace doris
diff --git a/be/src/vec/sink/group_commit_block_sink.h b/be/src/vec/sink/group_commit_block_sink.h
index 3db4bdd31f8d48..9a57cb594e7df9 100644
--- a/be/src/vec/sink/group_commit_block_sink.h
+++ b/be/src/vec/sink/group_commit_block_sink.h
@@ -48,6 +48,7 @@ class GroupCommitBlockSink : public DataSink {
 private:
     Status _add_block(RuntimeState* state, std::shared_ptr<vectorized::Block> block);
     Status _add_blocks(RuntimeState* state, bool is_blocks_contain_all_load_data);
+    size_t _pre_allocated(bool is_blocks_contain_all_load_data);
 
     vectorized::VExprContextSPtrs _output_vexpr_ctxs;
 
diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift
index e07797b593d06c..8700a50790f809 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -477,6 +477,8 @@ struct TExecPlanFragmentParams {
   27: optional i32 total_load_streams
 
   28: optional i32 num_local_sink
+
+  29: optional i64 content_length
 }
 
 struct TExecPlanFragmentParamsList {

From d474d4e701c366822e87fe21b0946f34a3111416 Mon Sep 17 00:00:00 2001
From: Pxl <pxl290@qq.com>
Date: Tue, 2 Jan 2024 17:54:56 +0800
Subject: [PATCH 250/299] [Bug](rollup) fallback and disable tulpe when rollup
 select failed on old planner (#29226)

* fallback and disable tulpe when rollup select failed on old planner

* add case
---
 .../doris/planner/SingleNodePlanner.java      |  4 +-
 .../data/mv_p0/test_28741/test_28741.out      |  4 ++
 .../suites/mv_p0/test_28741/test_28741.groovy | 72 +++++++++++++++++++
 3 files changed, 78 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/data/mv_p0/test_28741/test_28741.out
 create mode 100644 regression-test/suites/mv_p0/test_28741/test_28741.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index 1ba48b0b9df9a6..3de53090c9b0ed 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -1363,18 +1363,18 @@ public boolean selectMaterializedView(QueryStmt queryStmt, Analyzer analyzer) th
                 if (olapScanNode.getSelectedPartitionIds().size() == 0 && !FeConstants.runningUnitTest) {
                     continue;
                 }
+                boolean tupleSelectFailed = false;
 
                 try {
                     // select index by the old Rollup selector
                     olapScanNode.selectBestRollupByRollupSelector(analyzer);
                 } catch (UserException e) {
-                    LOG.debug("May no rollup index matched");
+                    tupleSelectFailed = true;
                 }
 
                 // select index by the new Materialized selector
                 MaterializedViewSelector.BestIndexInfo bestIndexInfo = materializedViewSelector
                         .selectBestMV(olapScanNode);
-                boolean tupleSelectFailed = false;
                 if (bestIndexInfo == null) {
                     tupleSelectFailed = true;
                 } else {
diff --git a/regression-test/data/mv_p0/test_28741/test_28741.out b/regression-test/data/mv_p0/test_28741/test_28741.out
new file mode 100644
index 00000000000000..18ef056f8510ca
--- /dev/null
+++ b/regression-test/data/mv_p0/test_28741/test_28741.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select --
+-
+
diff --git a/regression-test/suites/mv_p0/test_28741/test_28741.groovy b/regression-test/suites/mv_p0/test_28741/test_28741.groovy
new file mode 100644
index 00000000000000..a6c2c878a91a25
--- /dev/null
+++ b/regression-test/suites/mv_p0/test_28741/test_28741.groovy
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite ("test_28741") {
+
+    sql """ DROP TABLE IF EXISTS test; """
+
+    sql """
+            CREATE TABLE test (
+            a tinyint(4) NULL DEFAULT "0" ,
+            b int(11) DEFAULT "0" ,
+            c int(11) DEFAULT "0" ,
+            t DATETIMEV2 COMMENT "时间",
+            d bigint(20) SUM NULL DEFAULT "0",
+            e bigint(20) SUM NULL DEFAULT "0"
+            )
+            AGGREGATE KEY(a, b, c, t)
+            DISTRIBUTED BY HASH(a) BUCKETS AUTO
+            PROPERTIES
+            (
+            "replication_num" = "1"
+            );
+        """
+    createMV ("CREATE MATERIALIZED VIEW mv_test AS SELECT a,b,t,SUM(d) FROM test GROUP BY 1,2,3")
+
+    sql "INSERT INTO test(a,b,c,t,d,e) VALUES (1,2,3,'2023-12-19 18:21:00', 56, 78)"
+
+    sql """
+    ALTER TABLE test ADD COLUMN a1 INT KEY AFTER a, ADD COLUMN b1 VARCHAR(1024) KEY AFTER b, ADD COLUMN d1 BIGINT SUM AFTER d;
+    """
+
+    def getJobState = { tableName ->
+         def jobStateResult = sql """  SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+         return jobStateResult[0][9]
+    }
+
+    int max_try_time = 100
+    while (max_try_time--){
+        String result = getJobState("test")
+        if (result == "FINISHED") {
+            sleep(1000)
+            break
+        } else {
+            sleep(1000)
+            assertTrue(max_try_time>1)
+        }
+    }
+
+    sql "INSERT INTO test(a,a1,b,b1,c,t,d,d1,e) VALUES (1,1,2,'-',3,'2023-12-20 17:21:00', 56, 78, 89)"
+
+    explain {
+        sql("select b1 from test where t >= '2023-12-20 17:21:00'")
+        contains "(test)"
+    }
+    qt_select "select b1 from test where t >= '2023-12-20 17:21:00'"
+}

From 04abcc12454f153b30a9711c01ed9713ba0e45cd Mon Sep 17 00:00:00 2001
From: qiye <jianliang5669@gmail.com>
Date: Tue, 2 Jan 2024 18:29:09 +0800
Subject: [PATCH 251/299] [cases](index compaction)Add index compaction cases
 (#29393)

Add some cases for index compaction:

1. index compaction with duplicate key table
2. index compaction with unique key table
3. optimize index compaction with multi segments in one  inverted index
---
 .../test_index_compaction_dup_keys.out        | 123 +++++++
 .../test_index_compaction_unique_keys.out     |  69 ++++
 ...x_compaction_with_multi_index_segments.out | 105 ++++++
 .../test_index_compaction_dup_keys.groovy     | 225 ++++++++++++
 .../test_index_compaction_unique_keys.groovy  | 229 ++++++++++++
 ...ompaction_with_multi_index_segments.groovy | 341 +++++++++++-------
 6 files changed, 963 insertions(+), 129 deletions(-)
 create mode 100644 regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.out
 create mode 100644 regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.out
 create mode 100644 regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
 create mode 100644 regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy

diff --git a/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.out b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.out
new file mode 100644
index 00000000000000..11f5a6ed34c33f
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.out
@@ -0,0 +1,123 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1	andy	andy love apple	100
+1	bason	bason hate pear	99
+2	andy	andy love apple	100
+2	bason	bason hate pear	99
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+1	bason	bason hate pear	99
+2	andy	andy love apple	100
+2	bason	bason hate pear	99
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+1	andy	andy love apple	100
+1	bason	bason hate pear	99
+1	bason	bason hate pear	99
+2	andy	andy love apple	100
+2	andy	andy love apple	100
+2	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	andy	andy love apple	100
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+3	andy	andy love apple	100
+
+-- !sql --
+1	bason	bason hate pear	99
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+1	andy	andy love apple	100
+1	bason	bason hate pear	99
+1	bason	bason hate pear	99
+2	andy	andy love apple	100
+2	andy	andy love apple	100
+2	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	andy	andy love apple	100
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+3	andy	andy love apple	100
+
+-- !sql --
+1	bason	bason hate pear	99
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
diff --git a/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.out b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.out
new file mode 100644
index 00000000000000..3edad07916c9b5
--- /dev/null
+++ b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.out
@@ -0,0 +1,69 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
+-- !sql --
+1	bason	bason hate pear	99
+2	bason	bason hate pear	99
+3	bason	bason hate pear	99
+
diff --git a/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out
index 57ad3c1080c604..4b086658735be4 100644
--- a/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out
+++ b/regression-test/data/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.out
@@ -212,3 +212,108 @@
 2018-02-21T12:00	7	I'm using the builds
 2018-02-21T12:00	7	I'm using the builds
 
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+2018-02-21T19:00	8	I'm using the builds
+2018-02-21T20:00	9	I'm using the builds
+2018-02-21T21:00	10	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+2018-02-21T19:00	8	I'm using the builds
+2018-02-21T20:00	9	I'm using the builds
+2018-02-21T21:00	10	I'm using the builds
+
+-- !sql --
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+2018-02-21T19:00	8	I'm using the builds
+2018-02-21T20:00	9	I'm using the builds
+2018-02-21T21:00	10	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+2018-02-21T19:00	8	I'm using the builds
+2018-02-21T20:00	9	I'm using the builds
+2018-02-21T21:00	10	I'm using the builds
+
+-- !sql --
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+2018-02-21T19:00	8	I'm using the builds
+2018-02-21T20:00	9	I'm using the builds
+2018-02-21T21:00	10	I'm using the builds
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+2018-02-21T19:00	8	I'm using the builds
+2018-02-21T20:00	9	I'm using the builds
+2018-02-21T21:00	10	I'm using the builds
+
+-- !sql --
+
+-- !sql --
+2018-02-21T12:00	1	I'm using the builds
+2018-02-21T13:00	2	I'm using the builds
+2018-02-21T14:00	3	I'm using the builds
+2018-02-21T15:00	4	I'm using the builds
+2018-02-21T16:00	5	I'm using the builds
+2018-02-21T17:00	6	I'm using the builds
+2018-02-21T18:00	7	I'm using the builds
+
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
new file mode 100644
index 00000000000000..e555d85456c35d
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_dup_keys.groovy
@@ -0,0 +1,225 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_index_compaction_dup_keys", "p0") {
+    def tableName = "test_index_compaction_dup_keys"
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    boolean disableAutoCompaction = false
+  
+    def set_be_config = { key, value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+        }
+    }
+
+    def trigger_full_compaction_on_tablets = { String[][] tablets ->
+        for (String[] tablet : tablets) {
+            String tablet_id = tablet[0]
+            backend_id = tablet[2]
+            times = 1
+            
+            String compactionStatus;
+            do{
+                def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+                ++times
+                sleep(2000)
+                compactionStatus = parseJson(out.trim()).status.toLowerCase();
+            } while (compactionStatus!="success" && times<=10)
+
+
+            if (compactionStatus == "fail") {
+                assertEquals(disableAutoCompaction, false)
+                logger.info("Compaction was done automatically!")
+            }
+            if (disableAutoCompaction) {
+                assertEquals("success", compactionStatus)
+            }
+        }
+    }
+
+    def wait_full_compaction_done = { String[][] tablets ->
+        for (String[] tablet in tablets) {
+            boolean running = true
+            do {
+                Thread.sleep(1000)
+                String tablet_id = tablet[0]
+                backend_id = tablet[2]
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("success", compactionStatus.status.toLowerCase())
+                running = compactionStatus.run_status
+            } while (running)
+        }
+    }
+
+    def get_rowset_count = {String[][] tablets ->
+        int rowsetCount = 0
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            def compactionStatusUrlIndex = 18
+            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        return rowsetCount
+    }
+
+    def check_config = { String key, String value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+            logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def configList = parseJson(out.trim())
+            assert configList instanceof List
+            for (Object ele in (List) configList) {
+                assert ele instanceof List<String>
+                if (((List<String>) ele)[0] == key) {
+                    assertEquals(value, ((List<String>) ele)[2])
+                }
+            }
+        }
+    }
+
+    boolean invertedIndexCompactionEnable = false
+    boolean has_update_be_config = false
+    try {
+        String backend_id;
+        backend_id = backendId_to_backendIP.keySet()[0]
+        def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+        
+        logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def configList = parseJson(out.trim())
+        assert configList instanceof List
+
+        for (Object ele in (List) configList) {
+            assert ele instanceof List<String>
+            if (((List<String>) ele)[0] == "inverted_index_compaction_enable") {
+                invertedIndexCompactionEnable = Boolean.parseBoolean(((List<String>) ele)[2])
+                logger.info("inverted_index_compaction_enable: ${((List<String>) ele)[2]}")
+            }
+            if (((List<String>) ele)[0] == "disable_auto_compaction") {
+                disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
+                logger.info("disable_auto_compaction: ${((List<String>) ele)[2]}")
+            }
+        }
+        set_be_config.call("inverted_index_compaction_enable", "true")
+        has_update_be_config = true
+        // check updated config
+        check_config.call("inverted_index_compaction_enable", "true");
+
+        sql """ DROP TABLE IF EXISTS ${tableName}; """
+        sql """
+            CREATE TABLE ${tableName} (
+                `id` int(11) NULL,
+                `name` varchar(255) NULL,
+                `hobbies` text NULL,
+                `score` int(11) NULL,
+                index index_name (name) using inverted,
+                index index_hobbies (hobbies) using inverted properties("parser"="english"),
+                index index_score (score) using inverted
+            ) ENGINE=OLAP
+            DUPLICATE KEY(`id`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY HASH(`id`) BUCKETS 1
+            PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" = "true");
+        """
+
+        sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+        //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+        String[][] tablets = sql """ show tablets from ${tableName}; """
+
+        int replicaNum = 1
+        // before full compaction, there are 7 rowsets.
+        int rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 7 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        trigger_full_compaction_on_tablets.call(tablets)
+
+        // wait for full compaction done
+        wait_full_compaction_done.call(tablets)
+
+        // after full compaction, there is only 1 rowset.
+        rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+        // insert more data and trigger full compaction again
+        sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+        rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 7 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        trigger_full_compaction_on_tablets.call(tablets)
+
+        // wait for full compaction done
+        wait_full_compaction_done.call(tablets)
+
+        // after full compaction, there is only 1 rowset.
+        rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+    } finally {
+        if (has_update_be_config) {
+            set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString())
+        }
+    }
+}
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
new file mode 100644
index 00000000000000..b05059e917246d
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_unique_keys.groovy
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_index_compaction_unique_keys", "p0") {
+    def tableName = "test_index_compaction_unique_keys"
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    boolean disableAutoCompaction = false
+  
+    def set_be_config = { key, value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+        }
+    }
+
+    def trigger_full_compaction_on_tablets = { String[][] tablets ->
+        for (String[] tablet : tablets) {
+            String tablet_id = tablet[0]
+            backend_id = tablet[2]
+            times = 1
+            
+            String compactionStatus;
+            do{
+                def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+                ++times
+                sleep(2000)
+                compactionStatus = parseJson(out.trim()).status.toLowerCase();
+            } while (compactionStatus!="success" && times<=10)
+
+
+            if (compactionStatus == "fail") {
+                assertEquals(disableAutoCompaction, false)
+                logger.info("Compaction was done automatically!")
+            }
+            if (disableAutoCompaction) {
+                assertEquals("success", compactionStatus)
+            }
+        }
+    }
+
+    def wait_full_compaction_done = { String[][] tablets ->
+        for (String[] tablet in tablets) {
+            boolean running = true
+            do {
+                Thread.sleep(1000)
+                String tablet_id = tablet[0]
+                backend_id = tablet[2]
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("success", compactionStatus.status.toLowerCase())
+                running = compactionStatus.run_status
+            } while (running)
+        }
+    }
+
+    def get_rowset_count = {String[][] tablets ->
+        int rowsetCount = 0
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            def compactionStatusUrlIndex = 18
+            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        return rowsetCount
+    }
+
+    def check_config = { String key, String value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+            logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def configList = parseJson(out.trim())
+            assert configList instanceof List
+            for (Object ele in (List) configList) {
+                assert ele instanceof List<String>
+                if (((List<String>) ele)[0] == key) {
+                    assertEquals(value, ((List<String>) ele)[2])
+                }
+            }
+        }
+    }
+
+    boolean invertedIndexCompactionEnable = false
+    boolean has_update_be_config = false
+    try {
+        String backend_id;
+        backend_id = backendId_to_backendIP.keySet()[0]
+        def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+        
+        logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+        assertEquals(code, 0)
+        def configList = parseJson(out.trim())
+        assert configList instanceof List
+
+        for (Object ele in (List) configList) {
+            assert ele instanceof List<String>
+            if (((List<String>) ele)[0] == "inverted_index_compaction_enable") {
+                invertedIndexCompactionEnable = Boolean.parseBoolean(((List<String>) ele)[2])
+                logger.info("inverted_index_compaction_enable: ${((List<String>) ele)[2]}")
+            }
+            if (((List<String>) ele)[0] == "disable_auto_compaction") {
+                disableAutoCompaction = Boolean.parseBoolean(((List<String>) ele)[2])
+                logger.info("disable_auto_compaction: ${((List<String>) ele)[2]}")
+            }
+        }
+        set_be_config.call("inverted_index_compaction_enable", "true")
+        has_update_be_config = true
+        // check updated config
+        check_config.call("inverted_index_compaction_enable", "true");
+
+        sql """ DROP TABLE IF EXISTS ${tableName}; """
+        sql """
+            CREATE TABLE ${tableName} (
+                `id` int(11) NULL,
+                `name` varchar(255) NULL,
+                `hobbies` text NULL,
+                `score` int(11) NULL,
+                index index_name (name) using inverted,
+                index index_hobbies (hobbies) using inverted properties("parser"="english"),
+                index index_score (score) using inverted
+            ) ENGINE=OLAP
+            UNIQUE KEY(`id`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY HASH(`id`) BUCKETS 1
+            PROPERTIES ( 
+                "replication_num" = "1",
+                "disable_auto_compaction" = "true",
+                "enable_unique_key_merge_on_write" = "true"
+            );
+        """
+
+        sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+        //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+        String[][] tablets = sql """ show tablets from ${tableName}; """
+
+        int replicaNum = 1
+        // before full compaction, there are 7 rowsets.
+        int rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 7 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        trigger_full_compaction_on_tablets.call(tablets)
+
+        // wait for full compaction done
+        wait_full_compaction_done.call(tablets)
+
+        // after full compaction, there is only 1 rowset.
+        rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+        // insert more data and trigger full compaction again
+        sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (2, "bason", "bason hate pear", 99); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "andy", "andy love apple", 100); """
+        sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+        rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 7 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        trigger_full_compaction_on_tablets.call(tablets)
+
+        // wait for full compaction done
+        wait_full_compaction_done.call(tablets)
+
+        // after full compaction, there is only 1 rowset.
+        rowsetCount = get_rowset_count.call(tablets);
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """
+        qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """
+
+    } finally {
+        if (has_update_be_config) {
+            set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString())
+        }
+    }
+}
diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
index 765ee55515a420..fa54aae13f3447 100644
--- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
+++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_with_multi_index_segments.groovy
@@ -19,30 +19,99 @@ import org.codehaus.groovy.runtime.IOGroovyMethods
 
 suite("test_index_compaction_with_multi_index_segments", "p0") {
     def tableName = "test_index_compaction_with_multi_index_segments"
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    boolean disableAutoCompaction = false
   
     def set_be_config = { key, value ->
-        def backendId_to_backendIP = [:]
-        def backendId_to_backendHttpPort = [:]
-        getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
-
         for (String backend_id: backendId_to_backendIP.keySet()) {
             def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
             logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
         }
     }
 
-    boolean disableAutoCompaction = true
+    def trigger_full_compaction_on_tablets = { String[][] tablets ->
+        for (String[] tablet : tablets) {
+            String tablet_id = tablet[0]
+            backend_id = tablet[2]
+            times = 1
+            
+            String compactionStatus;
+            do{
+                def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+                ++times
+                sleep(2000)
+                compactionStatus = parseJson(out.trim()).status.toLowerCase();
+            } while (compactionStatus!="success" && times<=10)
+
+
+            if (compactionStatus == "fail") {
+                assertEquals(disableAutoCompaction, false)
+                logger.info("Compaction was done automatically!")
+            }
+            if (disableAutoCompaction) {
+                assertEquals("success", compactionStatus)
+            }
+        }
+    }
+
+    def wait_full_compaction_done = { String[][] tablets ->
+        for (String[] tablet in tablets) {
+            boolean running = true
+            do {
+                Thread.sleep(1000)
+                String tablet_id = tablet[0]
+                backend_id = tablet[2]
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("success", compactionStatus.status.toLowerCase())
+                running = compactionStatus.run_status
+            } while (running)
+        }
+    }
+
+    def get_rowset_count = {String[][] tablets ->
+        int rowsetCount = 0
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            def compactionStatusUrlIndex = 18
+            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
+            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def tabletJson = parseJson(out.trim())
+            assert tabletJson.rowsets instanceof List
+            rowsetCount +=((List<String>) tabletJson.rowsets).size()
+        }
+        return rowsetCount
+    }
+
+    def check_config = { String key, String value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+            logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+            assertEquals(code, 0)
+            def configList = parseJson(out.trim())
+            assert configList instanceof List
+            for (Object ele in (List) configList) {
+                assert ele instanceof List<String>
+                if (((List<String>) ele)[0] == key) {
+                    assertEquals(value, ((List<String>) ele)[2])
+                }
+            }
+        }
+    }
+
     boolean invertedIndexCompactionEnable = false
     int invertedIndexMaxBufferedDocs = -1;
     boolean has_update_be_config = false
 
     try {
-        String backend_id;
-        def backendId_to_backendIP = [:]
-        def backendId_to_backendHttpPort = [:]
-        getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
-
-        backend_id = backendId_to_backendIP.keySet()[0]
+        String backend_id = backendId_to_backendIP.keySet()[0]
         def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
         
         logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
@@ -65,6 +134,19 @@ suite("test_index_compaction_with_multi_index_segments", "p0") {
         set_be_config.call("inverted_index_max_buffered_docs", "5")
         has_update_be_config = true
 
+        // check config
+        check_config.call("inverted_index_compaction_enable", "true")
+        check_config.call("inverted_index_max_buffered_docs", "5")
+
+        /**
+        * test duplicated tables
+        * 1. insert 10 rows
+        * 2. insert another 10 rows
+        * 3. full compaction
+        * 4. insert 10 rows, again
+        * 5. full compaction
+        */
+        table_name = "test_index_compaction_with_multi_index_segments_dups"
         sql """ DROP TABLE IF EXISTS ${tableName}; """
         sql """
             CREATE TABLE ${tableName} (
@@ -114,74 +196,19 @@ suite("test_index_compaction_with_multi_index_segments", "p0") {
         //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
         String[][] tablets = sql """ show tablets from ${tableName}; """
 
-        def replicaNum = get_table_replica_num(tableName)
-        logger.info("get table replica num: " + replicaNum)
+        int replicaNum = 1
         // before full compaction, there are 3 rowsets.
-        int rowsetCount = 0
-        for (String[] tablet in tablets) {
-            String tablet_id = tablet[0]
-            def compactionStatusUrlIndex = 18
-            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
-            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
-            assertEquals(code, 0)
-            def tabletJson = parseJson(out.trim())
-            assert tabletJson.rowsets instanceof List
-            rowsetCount +=((List<String>) tabletJson.rowsets).size()
-        }
+        int rowsetCount = get_rowset_count.call(tablets)
         assert (rowsetCount == 3 * replicaNum)
 
         // trigger full compactions for all tablets in ${tableName}
-        for (String[] tablet in tablets) {
-            String tablet_id = tablet[0]
-            backend_id = tablet[2]
-            times = 1
-
-            do{
-                (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
-                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
-                ++times
-                sleep(2000)
-            } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10)
-
-            def compactJson = parseJson(out.trim())
-            if (compactJson.status.toLowerCase() == "fail") {
-                assertEquals(disableAutoCompaction, false)
-                logger.info("Compaction was done automatically!")
-            }
-            if (disableAutoCompaction) {
-                assertEquals("success", compactJson.status.toLowerCase())
-            }
-        }
+        trigger_full_compaction_on_tablets.call(tablets)
 
         // wait for full compaction done
-        for (String[] tablet in tablets) {
-            boolean running = true
-            do {
-                Thread.sleep(1000)
-                String tablet_id = tablet[0]
-                backend_id = tablet[2]
-                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
-                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
-                assertEquals(code, 0)
-                def compactionStatus = parseJson(out.trim())
-                assertEquals("success", compactionStatus.status.toLowerCase())
-                running = compactionStatus.run_status
-            } while (running)
-        }
+        wait_full_compaction_done.call(tablets)
 
         // after full compaction, there is only 1 rowset.
-        
-        rowsetCount = 0
-        for (String[] tablet in tablets) {
-            String tablet_id = tablet[0]
-            def compactionStatusUrlIndex = 18
-            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
-            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
-            assertEquals(code, 0)
-            def tabletJson = parseJson(out.trim())
-            assert tabletJson.rowsets instanceof List
-            rowsetCount +=((List<String>) tabletJson.rowsets).size()
-        }
+        rowsetCount = get_rowset_count.call(tablets)
         assert (rowsetCount == 1 * replicaNum)
 
         qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
@@ -203,74 +230,96 @@ suite("test_index_compaction_with_multi_index_segments", "p0") {
 
         tablets = sql """ show tablets from ${tableName}; """
 
-        replicaNum = get_table_replica_num(tableName)
-        logger.info("get table replica num: " + replicaNum)
         // before full compaction, there are 2 rowsets.
-        rowsetCount = 0
-        for (String[] tablet in tablets) {
-            String tablet_id = tablet[0]
-            def compactionStatusUrlIndex = 18
-            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
-            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
-            assertEquals(code, 0)
-            def tabletJson = parseJson(out.trim())
-            assert tabletJson.rowsets instanceof List
-            rowsetCount +=((List<String>) tabletJson.rowsets).size()
-        }
+        rowsetCount = get_rowset_count.call(tablets)
         assert (rowsetCount == 2 * replicaNum)
 
         // trigger full compactions for all tablets in ${tableName}
-        for (String[] tablet in tablets) {
-            String tablet_id = tablet[0]
-            backend_id = tablet[2]
-            times = 1
+        trigger_full_compaction_on_tablets.call(tablets)
 
-            do{
-                (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
-                logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
-                ++times
-                sleep(2000)
-            } while (parseJson(out.trim()).status.toLowerCase()!="success" && times<=10)
+        // wait for full compaction done
+        wait_full_compaction_done.call(tablets)
 
-            def compactJson = parseJson(out.trim())
-            if (compactJson.status.toLowerCase() == "fail") {
-                assertEquals(disableAutoCompaction, false)
-                logger.info("Compaction was done automatically!")
-            }
-            if (disableAutoCompaction) {
-                assertEquals("success", compactJson.status.toLowerCase())
-            }
-        }
+        // after full compaction, there is only 1 rowset.
+        rowsetCount = get_rowset_count.call(tablets)
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "using" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "the" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where comment_id < 8 order by file_time, comment_id, body """
+
+        /**
+        * test unique tables
+        * 1. insert 10 rows
+        * 2. insert another 10 rows
+        * 3. full compaction
+        * 4. insert 10 rows, again
+        * 5. full compaction
+        */
+        table_name = "test_index_compaction_with_multi_index_segments_unique"
+        sql """ DROP TABLE IF EXISTS ${tableName}; """
+        sql """
+            CREATE TABLE ${tableName} (
+                `file_time` DATETIME NOT NULL,
+                `comment_id` int(11)  NULL,
+                `body` TEXT NULL DEFAULT "",
+                INDEX idx_comment_id (`comment_id`) USING INVERTED COMMENT '''',
+                INDEX idx_body (`body`) USING INVERTED PROPERTIES("parser" = "unicode") COMMENT ''''
+            ) ENGINE=OLAP
+            UNIQUE KEY(`file_time`)
+            COMMENT 'OLAP'
+            DISTRIBUTED BY HASH(`file_time`) BUCKETS 1
+            PROPERTIES (
+            "replication_allocation" = "tag.location.default: 1",
+            "disable_auto_compaction" = "true",
+            "enable_unique_key_merge_on_write" = "true"
+            );
+        """
+
+        // insert 10 rows
+        sql """ INSERT INTO ${tableName} VALUES ("2018-02-21 12:00:00", 1, "I\'m using the builds"),
+                                                ("2018-02-21 13:00:00", 2, "I\'m using the builds"),
+                                                ("2018-02-21 14:00:00", 3, "I\'m using the builds"),
+                                                ("2018-02-21 15:00:00", 4, "I\'m using the builds"),
+                                                ("2018-02-21 16:00:00", 5, "I\'m using the builds"),
+                                                ("2018-02-21 17:00:00", 6, "I\'m using the builds"),
+                                                ("2018-02-21 18:00:00", 7, "I\'m using the builds"),
+                                                ("2018-02-21 19:00:00", 8, "I\'m using the builds"),
+                                                ("2018-02-21 20:00:00", 9, "I\'m using the builds"),
+                                                ("2018-02-21 21:00:00", 10, "I\'m using the builds"); """
+        // insert another 10 rows
+        sql """ INSERT INTO ${tableName} VALUES ("2018-02-21 12:00:00", 1, "I\'m using the builds"),
+                                                ("2018-02-21 13:00:00", 2, "I\'m using the builds"),
+                                                ("2018-02-21 14:00:00", 3, "I\'m using the builds"),
+                                                ("2018-02-21 15:00:00", 4, "I\'m using the builds"),
+                                                ("2018-02-21 16:00:00", 5, "I\'m using the builds"),
+                                                ("2018-02-21 17:00:00", 6, "I\'m using the builds"),
+                                                ("2018-02-21 18:00:00", 7, "I\'m using the builds"),
+                                                ("2018-02-21 19:00:00", 8, "I\'m using the builds"),
+                                                ("2018-02-21 20:00:00", 9, "I\'m using the builds"),
+                                                ("2018-02-21 21:00:00", 10, "I\'m using the builds"); """
+
+        qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "using" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "the" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where comment_id < 8 order by file_time, comment_id, body """
+
+        //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
+        tablets = sql """ show tablets from ${tableName}; """
+
+        // before full compaction, there are 3 rowsets.
+        rowsetCount = get_rowset_count.call(tablets)
+        assert (rowsetCount == 3 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        trigger_full_compaction_on_tablets.call(tablets)
 
         // wait for full compaction done
-        for (String[] tablet in tablets) {
-            boolean running = true
-            do {
-                Thread.sleep(1000)
-                String tablet_id = tablet[0]
-                backend_id = tablet[2]
-                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
-                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
-                assertEquals(code, 0)
-                def compactionStatus = parseJson(out.trim())
-                assertEquals("success", compactionStatus.status.toLowerCase())
-                running = compactionStatus.run_status
-            } while (running)
-        }
+        wait_full_compaction_done.call(tablets)
 
         // after full compaction, there is only 1 rowset.
-        
-        rowsetCount = 0
-        for (String[] tablet in tablets) {
-            String tablet_id = tablet[0]
-            def compactionStatusUrlIndex = 18
-            (code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
-            logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
-            assertEquals(code, 0)
-            def tabletJson = parseJson(out.trim())
-            assert tabletJson.rowsets instanceof List
-            rowsetCount +=((List<String>) tabletJson.rowsets).size()
-        }
+        rowsetCount = get_rowset_count.call(tablets)
         assert (rowsetCount == 1 * replicaNum)
 
         qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
@@ -278,6 +327,40 @@ suite("test_index_compaction_with_multi_index_segments", "p0") {
         qt_sql """ select * from ${tableName} where body match "the" order by file_time, comment_id, body """
         qt_sql """ select * from ${tableName} where comment_id < 8 order by file_time, comment_id, body """
 
+        // insert 10 rows, again
+        sql """ INSERT INTO ${tableName} VALUES ("2018-02-21 12:00:00", 1, "I\'m using the builds"),
+                                                ("2018-02-21 13:00:00", 2, "I\'m using the builds"),
+                                                ("2018-02-21 14:00:00", 3, "I\'m using the builds"),
+                                                ("2018-02-21 15:00:00", 4, "I\'m using the builds"),
+                                                ("2018-02-21 16:00:00", 5, "I\'m using the builds"),
+                                                ("2018-02-21 17:00:00", 6, "I\'m using the builds"),
+                                                ("2018-02-21 18:00:00", 7, "I\'m using the builds"),
+                                                ("2018-02-21 19:00:00", 8, "I\'m using the builds"),
+                                                ("2018-02-21 20:00:00", 9, "I\'m using the builds"),
+                                                ("2018-02-21 21:00:00", 10, "I\'m using the builds"); """
+
+        tablets = sql """ show tablets from ${tableName}; """
+
+        // before full compaction, there are 2 rowsets.
+        rowsetCount = get_rowset_count.call(tablets)
+        assert (rowsetCount == 2 * replicaNum)
+
+        // trigger full compactions for all tablets in ${tableName}
+        trigger_full_compaction_on_tablets.call(tablets)
+
+        // wait for full compaction done
+        wait_full_compaction_done.call(tablets)
+
+        // after full compaction, there is only 1 rowset.
+        rowsetCount = get_rowset_count.call(tablets)
+        assert (rowsetCount == 1 * replicaNum)
+
+        qt_sql """ select * from ${tableName} order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "using" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where body match "the" order by file_time, comment_id, body """
+        qt_sql """ select * from ${tableName} where comment_id < 8 order by file_time, comment_id, body """
+
+
     } finally {
         if (has_update_be_config) {
             set_be_config.call("inverted_index_compaction_enable", invertedIndexCompactionEnable.toString())

From 3eca457edd00fab3b2b6cd878dd1769b9c3c25ea Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Tue, 2 Jan 2024 19:38:34 +0800
Subject: [PATCH 252/299] [fix](statistics)Fix set partition loaded replayer
 try to write edit log bug (#29395)

While observer or checkpointer replay SetPartitionLoaded log, it shouldn't try to write the log. This pr is to fix the bug.
---
 .../main/java/org/apache/doris/statistics/AnalysisManager.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 63ae94e37d9e39..7c246b45875300 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -936,7 +936,7 @@ public void updateUpdatedRows(long tblId, long rows) {
     // Set to true means new partition loaded data
     public void setNewPartitionLoaded(long tblId) {
         TableStatsMeta statsStatus = idToTblStats.get(tblId);
-        if (statsStatus != null) {
+        if (statsStatus != null && Env.getCurrentEnv().isMaster() && !Env.isCheckpointThread()) {
             statsStatus.newPartitionLoaded.set(true);
             logCreateTableStats(statsStatus);
         }

From af39217d149d55699ee6eb48feba8bb716e5fe36 Mon Sep 17 00:00:00 2001
From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com>
Date: Tue, 2 Jan 2024 19:40:23 +0800
Subject: [PATCH 253/299] [improve](agg)support push down min/max on unique
 table (#29242)

---
 .../apache/doris/nereids/rules/RuleType.java  |   2 +
 .../implementation/AggregateStrategies.java   | 166 +++++++++++++++++
 .../org/apache/doris/qe/SessionVariable.java  |  15 ++
 .../explain/test_pushdown_explain.out         | 120 +++++++++++++
 .../explain/test_pushdown_explain.groovy      | 170 ++++++++++++++++++
 5 files changed, 473 insertions(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 57e62380137c54..0ffe509d8bc306 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -378,6 +378,8 @@ public enum RuleType {
     STORAGE_LAYER_AGGREGATE_WITH_PROJECT(RuleTypeClass.IMPLEMENTATION),
     STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT_FOR_FILE_SCAN(RuleTypeClass.IMPLEMENTATION),
     STORAGE_LAYER_AGGREGATE_WITH_PROJECT_FOR_FILE_SCAN(RuleTypeClass.IMPLEMENTATION),
+    STORAGE_LAYER_AGGREGATE_MINMAX_ON_UNIQUE(RuleTypeClass.IMPLEMENTATION),
+    STORAGE_LAYER_AGGREGATE_MINMAX_ON_UNIQUE_WITHOUT_PROJECT(RuleTypeClass.IMPLEMENTATION),
     COUNT_ON_INDEX(RuleTypeClass.IMPLEMENTATION),
     COUNT_ON_INDEX_WITHOUT_PROJECT(RuleTypeClass.IMPLEMENTATION),
     ONE_PHASE_AGGREGATE_WITHOUT_DISTINCT(RuleTypeClass.IMPLEMENTATION),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
index 89373cc95c3e49..b4fce67beb11f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java
@@ -40,12 +40,15 @@
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
 import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateParam;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
 import org.apache.doris.nereids.trees.expressions.functions.agg.GroupConcat;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
+import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctCount;
 import org.apache.doris.nereids.trees.expressions.functions.agg.MultiDistinctSum;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
@@ -140,6 +143,72 @@ public List<Rule> buildRules() {
                         return pushdownCountOnIndex(agg, project, filter, olapScan, ctx.cascadesContext);
                     })
             ),
+            RuleType.STORAGE_LAYER_AGGREGATE_MINMAX_ON_UNIQUE_WITHOUT_PROJECT.build(
+                logicalAggregate(
+                        logicalFilter(
+                                logicalOlapScan().when(this::isUniqueKeyTable))
+                                .when(filter -> {
+                                    if (filter.getConjuncts().size() != 1) {
+                                        return false;
+                                    }
+                                    Expression childExpr = filter.getConjuncts().iterator().next().children().get(0);
+                                    if (childExpr instanceof SlotReference) {
+                                        Optional<Column> column = ((SlotReference) childExpr).getColumn();
+                                        return column.isPresent() ? column.get().isDeleteSignColumn() : false;
+                                    }
+                                    return false;
+                                })
+                        )
+                        .when(agg -> enablePushDownMinMaxOnUnique())
+                        .when(agg -> agg.getGroupByExpressions().isEmpty())
+                        .when(agg -> {
+                            Set<AggregateFunction> funcs = agg.getAggregateFunctions();
+                            return !funcs.isEmpty() && funcs.stream()
+                                    .allMatch(f -> (f instanceof Min) || (f instanceof Max));
+                        })
+                        .thenApply(ctx -> {
+                            LogicalAggregate<LogicalFilter<LogicalOlapScan>> agg = ctx.root;
+                            LogicalFilter<LogicalOlapScan> filter = agg.child();
+                            LogicalOlapScan olapScan = filter.child();
+                            return pushdownMinMaxOnUniqueTable(agg, null, filter, olapScan,
+                                    ctx.cascadesContext);
+                        })
+            ),
+            RuleType.STORAGE_LAYER_AGGREGATE_MINMAX_ON_UNIQUE.build(
+                    logicalAggregate(
+                            logicalProject(
+                                    logicalFilter(
+                                            logicalOlapScan().when(this::isUniqueKeyTable))
+                                            .when(filter -> {
+                                                if (filter.getConjuncts().size() != 1) {
+                                                    return false;
+                                                }
+                                                Expression childExpr = filter.getConjuncts().iterator().next()
+                                                        .children().get(0);
+                                                if (childExpr instanceof SlotReference) {
+                                                    Optional<Column> column = ((SlotReference) childExpr).getColumn();
+                                                    return column.isPresent() ? column.get().isDeleteSignColumn()
+                                                            : false;
+                                                }
+                                                return false;
+                                            }))
+                        )
+                        .when(agg -> enablePushDownMinMaxOnUnique())
+                        .when(agg -> agg.getGroupByExpressions().isEmpty())
+                        .when(agg -> {
+                            Set<AggregateFunction> funcs = agg.getAggregateFunctions();
+                            return !funcs.isEmpty()
+                                    && funcs.stream().allMatch(f -> (f instanceof Min) || (f instanceof Max));
+                        })
+                        .thenApply(ctx -> {
+                            LogicalAggregate<LogicalProject<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;
+                            LogicalProject<LogicalFilter<LogicalOlapScan>> project = agg.child();
+                            LogicalFilter<LogicalOlapScan> filter = project.child();
+                            LogicalOlapScan olapScan = filter.child();
+                            return pushdownMinMaxOnUniqueTable(agg, project, filter, olapScan,
+                                    ctx.cascadesContext);
+                        })
+            ),
             RuleType.STORAGE_LAYER_AGGREGATE_WITHOUT_PROJECT.build(
                 logicalAggregate(
                     logicalOlapScan()
@@ -238,6 +307,19 @@ && couldConvertToMulti(agg))
         );
     }
 
+    private boolean enablePushDownMinMaxOnUnique() {
+        ConnectContext connectContext = ConnectContext.get();
+        return connectContext != null && connectContext.getSessionVariable().isEnablePushDownMinMaxOnUnique();
+    }
+
+    private boolean isUniqueKeyTable(LogicalOlapScan logicalScan) {
+        if (logicalScan != null) {
+            KeysType keysType = logicalScan.getTable().getKeysType();
+            return keysType == KeysType.UNIQUE_KEYS;
+        }
+        return false;
+    }
+
     private boolean enablePushDownCountOnIndex() {
         ConnectContext connectContext = ConnectContext.get();
         return connectContext != null && connectContext.getSessionVariable().isEnablePushDownCountOnIndex();
@@ -314,6 +396,90 @@ private LogicalAggregate<? extends Plan> pushdownCountOnIndex(
         }
     }
 
+    //select /*+SET_VAR(enable_pushdown_minmax_on_unique=true) */min(user_id) from table_unique;
+    //push pushAggOp=MINMAX to scan node
+    private LogicalAggregate<? extends Plan> pushdownMinMaxOnUniqueTable(
+            LogicalAggregate<? extends Plan> aggregate,
+            @Nullable LogicalProject<? extends Plan> project,
+            LogicalFilter<? extends Plan> filter,
+            LogicalOlapScan olapScan,
+            CascadesContext cascadesContext) {
+        final LogicalAggregate<? extends Plan> canNotPush = aggregate;
+        Set<AggregateFunction> aggregateFunctions = aggregate.getAggregateFunctions();
+        if (checkWhetherPushDownMinMax(aggregateFunctions, project, olapScan.getOutput())) {
+            PhysicalOlapScan physicalOlapScan = (PhysicalOlapScan) new LogicalOlapScanToPhysicalOlapScan()
+                    .build()
+                    .transform(olapScan, cascadesContext)
+                    .get(0);
+            if (project != null) {
+                return aggregate.withChildren(ImmutableList.of(
+                        project.withChildren(ImmutableList.of(
+                                filter.withChildren(ImmutableList.of(
+                                        new PhysicalStorageLayerAggregate(
+                                                physicalOlapScan,
+                                                PushDownAggOp.MIN_MAX)))))));
+            } else {
+                return aggregate.withChildren(ImmutableList.of(
+                        filter.withChildren(ImmutableList.of(
+                                new PhysicalStorageLayerAggregate(
+                                        physicalOlapScan,
+                                        PushDownAggOp.MIN_MAX)))));
+            }
+        } else {
+            return canNotPush;
+        }
+    }
+
+    private boolean checkWhetherPushDownMinMax(Set<AggregateFunction> aggregateFunctions,
+            @Nullable LogicalProject<? extends Plan> project, List<Slot> outPutSlots) {
+        boolean onlyContainsSlotOrNumericCastSlot = aggregateFunctions.stream()
+                .map(ExpressionTrait::getArguments)
+                .flatMap(List::stream)
+                .allMatch(argument -> {
+                    if (argument instanceof SlotReference) {
+                        return true;
+                    }
+                    return false;
+                });
+        if (!onlyContainsSlotOrNumericCastSlot) {
+            return false;
+        }
+        List<Expression> argumentsOfAggregateFunction = aggregateFunctions.stream()
+                .flatMap(aggregateFunction -> aggregateFunction.getArguments().stream())
+                .collect(ImmutableList.toImmutableList());
+
+        if (project != null) {
+            argumentsOfAggregateFunction = Project.findProject(
+                    argumentsOfAggregateFunction, project.getProjects())
+                    .stream()
+                    .map(p -> p instanceof Alias ? p.child(0) : p)
+                    .collect(ImmutableList.toImmutableList());
+        }
+        onlyContainsSlotOrNumericCastSlot = argumentsOfAggregateFunction
+                .stream()
+                .allMatch(argument -> {
+                    if (argument instanceof SlotReference) {
+                        return true;
+                    }
+                    return false;
+                });
+        if (!onlyContainsSlotOrNumericCastSlot) {
+            return false;
+        }
+        Set<SlotReference> aggUsedSlots = ExpressionUtils.collect(argumentsOfAggregateFunction,
+                SlotReference.class::isInstance);
+        List<SlotReference> usedSlotInTable = (List<SlotReference>) Project.findProject(aggUsedSlots,
+                outPutSlots);
+        for (SlotReference slot : usedSlotInTable) {
+            Column column = slot.getColumn().get();
+            PrimitiveType colType = column.getType().getPrimitiveType();
+            if (colType.isComplexType() || colType.isHllType() || colType.isBitmapType()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     /**
      * sql: select count(*) from tbl
      * <p>
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index d594fd395e772f..bbe5eddaa29691 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -480,6 +480,8 @@ public class SessionVariable implements Serializable, Writable {
     public static final String MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_FOREIGN_TABLE
             = "materialized_view_rewrite_enable_contain_foreign_table";
 
+    public static final String ENABLE_PUSHDOWN_MINMAX_ON_UNIQUE = "enable_pushdown_minmax_on_unique";
+
     // When set use fix replica = true, the fixed replica maybe bad, try to use the health one if
     // this session variable is set to true.
     public static final String FALLBACK_OTHER_REPLICA_WHEN_FIXED_CORRUPT = "fallback_other_replica_when_fixed_corrupt";
@@ -1221,6 +1223,11 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) {
             "是否启用count_on_index pushdown。", "Set whether to pushdown count_on_index."})
     public boolean enablePushDownCountOnIndex = true;
 
+    // Whether enable pushdown minmax to scan node of unique table.
+    @VariableMgr.VarAttr(name = ENABLE_PUSHDOWN_MINMAX_ON_UNIQUE, needForward = true, description = {
+        "是否启用pushdown minmax on unique table。", "Set whether to pushdown minmax on unique table."})
+    public boolean enablePushDownMinMaxOnUnique = false;
+
     // Whether drop table when create table as select insert data appear error.
     @VariableMgr.VarAttr(name = DROP_TABLE_IF_CTAS_FAILED, needForward = true)
     public boolean dropTableIfCtasFailed = true;
@@ -2438,6 +2445,14 @@ public void setDisableJoinReorder(boolean disableJoinReorder) {
         this.disableJoinReorder = disableJoinReorder;
     }
 
+    public boolean isEnablePushDownMinMaxOnUnique() {
+        return enablePushDownMinMaxOnUnique;
+    }
+
+    public void setEnablePushDownMinMaxOnUnique(boolean enablePushDownMinMaxOnUnique) {
+        this.enablePushDownMinMaxOnUnique = enablePushDownMinMaxOnUnique;
+    }
+
     /**
      * Nereids only support vectorized engine.
      *
diff --git a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out
index 72d126351a3050..a5fa13665d0c20 100644
--- a/regression-test/data/nereids_p0/explain/test_pushdown_explain.out
+++ b/regression-test/data/nereids_p0/explain/test_pushdown_explain.out
@@ -2,3 +2,123 @@
 -- !select --
 1
 
+-- !select_table_unique0 --
+1	c
+
+-- !select_table_unique0_min --
+a
+
+-- !select_table_unique1 --
+1	c
+2	e
+
+-- !select_table_unique1_max --
+g
+
+-- !select_table_unique2 --
+1	c
+2	e
+
+-- !select_table_unique2_max --
+h
+
+-- !select_table_unique3 --
+1	c
+2	e
+4	k
+
+-- !select_table_unique3_max --
+l
+
+-- !select_0 --
+1	asd	cc
+2	qwe	vvx
+3	ffsd	mnm
+4	qdf	ll
+5	cvfv	vff
+
+-- !select_1 --
+1
+
+-- !select_2 --
+5
+
+-- !select_3 --
+asd
+
+-- !select_4 --
+qwe
+
+-- !select_5 --
+cc
+
+-- !select_6 --
+vvx
+
+-- !select_00 --
+1	asd	zzz
+2	qwe	vvx
+3	ffsd	mnm
+4	qdf	ll
+5	cvfv	vff
+
+-- !select_7 --
+1
+
+-- !select_8 --
+5
+
+-- !select_9 --
+asd
+
+-- !select_10 --
+qwe
+
+-- !select_11 --
+cc
+
+-- !select_12 --
+zzz
+
+-- !select_000 --
+1	asd	zzz
+3	ffsd	mnm
+4	qdf	ll
+5	cvfv	vff
+
+-- !select_13 --
+1
+
+-- !select_14 --
+5
+
+-- !select_15 --
+asd
+
+-- !select_16 --
+qdf
+
+-- !select_17 --
+ll
+
+-- !select_18 --
+zzz
+
+-- !select_19 --
+1
+
+-- !select_20 --
+5
+
+-- !select_21 --
+asd
+
+-- !select_22 --
+qwe
+
+-- !select_23 --
+cc
+
+-- !select_24 --
+vvx
+
diff --git a/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy b/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy
index e82b524b5a2307..8406e0972bcafa 100644
--- a/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy
+++ b/regression-test/suites/nereids_p0/explain/test_pushdown_explain.groovy
@@ -65,4 +65,174 @@ suite("test_pushdown_explain") {
         sql("select count(cast(lo_orderkey as bigint)) from test_lineorder;")
         contains "pushAggOp=COUNT"
     }
+
+    sql "DROP TABLE IF EXISTS table_unique0"
+    sql """ 
+        CREATE TABLE `table_unique0` (
+            `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+            `username` VARCHAR(50) NOT NULL COMMENT '\"用户昵称\"'
+        ) ENGINE=OLAP
+        UNIQUE KEY(`user_id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "disable_auto_compaction" = "true"
+        );
+    """
+
+    // set seession variables
+    sql "set enable_pushdown_minmax_on_unique = true;"
+
+    sql """ insert into table_unique0 values(1,"a"); """
+    sql """ insert into table_unique0 values(1,"b"); """
+    sql """ insert into table_unique0 values(1,"c"); """
+    qt_select_table_unique0 "select * from table_unique0 order by user_id;" // 1, c
+    qt_select_table_unique0_min "select min(username) from table_unique0;"  // a is read from zone map
+
+    sql """ insert into table_unique0 values(2,"g"); """
+    sql """ insert into table_unique0 values(2,"f"); """
+    sql """ insert into table_unique0 values(2,"e"); """
+    qt_select_table_unique1 "select * from table_unique0 order by user_id;" // 2, e
+    qt_select_table_unique1_max "select max(username) from table_unique0;"  // g is read from zone map
+
+    sql """ insert into table_unique0 values(3,"h"); """   
+    sql """ insert into table_unique0(user_id,username,__DORIS_DELETE_SIGN__) values(3,'h',1); """ // delete id = 3
+    qt_select_table_unique2 "select * from table_unique0 order by user_id;" // no user_id = 3
+    qt_select_table_unique2_max "select max(username) from table_unique0;"  // h is read from zone map
+
+    sql """ insert into table_unique0 values(4,"l"); """  
+    sql """ update table_unique0 set username = "k" where user_id = 4; """ 
+    qt_select_table_unique3 "select * from table_unique0 order by user_id;" // 4 ,k 
+    qt_select_table_unique3_max "select max(username) from table_unique0;"  // l is read from zone map
+
+    sql "DROP TABLE IF EXISTS table_unique"
+    sql """ 
+        CREATE TABLE `table_unique` (
+            `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+            `username` VARCHAR(50) NOT NULL COMMENT '\"用户昵称\"',
+            `val` VARCHAR(50) NULL
+        ) ENGINE=OLAP
+        UNIQUE KEY(`user_id`, `username`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "disable_auto_compaction" = "true"
+        );
+    """
+    sql """ 
+        insert into table_unique values(1,"asd","cc"),(2,"qwe","vvx"),(3,"ffsd","mnm"),(4,"qdf","ll"),(5,"cvfv","vff");
+    """
+
+    sql "set enable_pushdown_minmax_on_unique = false;"
+    explain {
+        sql("select min(user_id) from table_unique;")
+        contains "pushAggOp=NONE"
+    }
+    explain {
+        sql("select max(user_id) from table_unique;")
+        contains "pushAggOp=NONE"
+    }
+    explain {
+        sql("select min(username) from table_unique;")
+        contains "pushAggOp=NONE"
+    }
+    explain {
+        sql("select max(username) from table_unique;")
+        contains "pushAggOp=NONE"
+    }
+
+
+    // set seession variables
+    sql "set enable_pushdown_minmax_on_unique = true;"
+    explain {
+        sql("select min(user_id) from table_unique;")
+        contains "pushAggOp=MINMAX"
+    }
+    explain {
+        sql("select max(user_id) from table_unique;")
+        contains "pushAggOp=MINMAX"
+    }
+    explain {
+        sql("select min(username) from table_unique;")
+        contains "pushAggOp=MINMAX"
+    }
+    explain {
+        sql("select max(username) from table_unique;")
+        contains "pushAggOp=MINMAX"
+    }
+    qt_select_0 "select * from table_unique order by user_id;"
+    qt_select_1 "select min(user_id) from table_unique;"
+    qt_select_2 "select max(user_id) from table_unique;"
+    qt_select_3 "select min(username) from table_unique;"
+    qt_select_4 "select max(username) from table_unique;"
+    qt_select_5 "select min(val) from table_unique;"
+    qt_select_6 "select max(val) from table_unique;"
+    sql """
+        update table_unique set val = "zzz" where user_id = 1;
+    """
+    qt_select_00 "select * from table_unique order by user_id;"
+    qt_select_7 "select min(user_id) from table_unique;"
+    qt_select_8 "select max(user_id) from table_unique;"
+    qt_select_9 "select min(username) from table_unique;"
+    qt_select_10 "select max(username) from table_unique;"
+    qt_select_11 "select min(val) from table_unique;"
+    qt_select_12 "select max(val) from table_unique;"
+
+    sql """
+        delete from table_unique where user_id = 2;
+    """
+    qt_select_000 "select * from table_unique order by user_id;"
+    qt_select_13 "select min(user_id) from table_unique;"
+    qt_select_14 "select max(user_id) from table_unique;"
+    qt_select_15 "select min(username) from table_unique;"
+    qt_select_16 "select max(username) from table_unique;"
+    qt_select_17 "select min(val) from table_unique;"
+    qt_select_18 "select max(val) from table_unique;"
+
+
+    sql "DROP TABLE IF EXISTS table_agg"
+    sql """ 
+        CREATE TABLE `table_agg` (
+            `user_id` LARGEINT NOT NULL COMMENT '\"用户id\"',
+            `username` VARCHAR(50) NOT NULL COMMENT '\"用户昵称\"',
+            `val` VARCHAR(50) max NULL 
+        ) ENGINE=OLAP
+        AGGREGATE KEY(`user_id`, `username`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
+        PROPERTIES (
+        "replication_allocation" = "tag.location.default: 1",
+        "disable_auto_compaction" = "true"
+        );
+    """
+
+    sql """ 
+        insert into table_agg values(1,"asd","cc"),(2,"qwe","vvx"),(3,"ffsd","mnm"),(4,"qdf","ll"),(5,"cvfv","vff");
+    """
+
+    explain {
+        sql("select min(user_id) from table_agg;")
+        contains "pushAggOp=MINMAX"
+    }
+    explain {
+        sql("select max(user_id) from table_agg;")
+        contains "pushAggOp=MINMAX"
+    }
+    explain {
+        sql("select min(username) from table_agg;")
+        contains "pushAggOp=MINMAX"
+    }
+    explain {
+        sql("select max(username) from table_agg;")
+        contains "pushAggOp=MINMAX"
+    }
+
+    qt_select_19 "select min(user_id) from table_agg;"
+    qt_select_20 "select max(user_id) from table_agg;"
+    qt_select_21 "select min(username) from table_agg;"
+    qt_select_22 "select max(username) from table_agg;"
+    qt_select_23 "select min(val) from table_agg;"
+    qt_select_24 "select max(val) from table_agg;"
 }

From f2fa62f5242ba45de9b22cfb0b271af79609e4df Mon Sep 17 00:00:00 2001
From: jakevin <jakevingoo@gmail.com>
Date: Tue, 2 Jan 2024 20:13:59 +0800
Subject: [PATCH 254/299] [minor](Nereids): cleanup code (#29413)

---
 .../translator/PhysicalPlanTranslator.java    |  3 ++-
 .../nereids/properties/DistributionSpec.java  |  1 -
 .../plans/physical/PhysicalDistribute.java    |  4 ++--
 .../apache/doris/planner/EmptySetNode.java    |  1 -
 .../org/apache/doris/planner/PlanNode.java    | 22 -------------------
 .../doris/nereids/util/PlanChecker.java       |  1 -
 6 files changed, 4 insertions(+), 28 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 6ad21d348905b8..f8b23935318dac 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -313,10 +313,11 @@ public PlanFragment visitPhysicalDistribute(PhysicalDistribute<? extends Plan> d
         DataPartition dataPartition = toDataPartition(distribute.getDistributionSpec(), validOutputIds, context);
         exchangeNode.setPartitionType(dataPartition.getType());
         PlanFragment parentFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, dataPartition);
-        exchangeNode.setNumInstances(inputFragment.getPlanRoot().getNumInstances());
         if (distribute.getDistributionSpec() instanceof DistributionSpecGather) {
             // gather to one instance
             exchangeNode.setNumInstances(1);
+        } else {
+            exchangeNode.setNumInstances(inputFragment.getPlanRoot().getNumInstances());
         }
 
         // process multicast sink
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java
index f748d6e9e16b38..067a5dd328f252 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java
@@ -44,7 +44,6 @@ public GroupExpression addEnforcer(Group child) {
         // If we don't set LogicalProperties explicitly, node will compute a applicable LogicalProperties for itself.
         PhysicalDistribute<GroupPlan> distribution = new PhysicalDistribute<>(
                 this,
-                child.getLogicalProperties(),
                 new GroupPlan(child));
         return new GroupExpression(distribution, Lists.newArrayList(child));
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
index e3b547dd8843d9..d62225cdcd0601 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalDistribute.java
@@ -52,8 +52,8 @@ public class PhysicalDistribute<CHILD_TYPE extends Plan> extends PhysicalUnary<C
 
     protected DistributionSpec distributionSpec;
 
-    public PhysicalDistribute(DistributionSpec spec, LogicalProperties logicalProperties, CHILD_TYPE child) {
-        this(spec, Optional.empty(), logicalProperties, child);
+    public PhysicalDistribute(DistributionSpec spec, CHILD_TYPE child) {
+        this(spec, Optional.empty(), child.getLogicalProperties(), child);
     }
 
     public PhysicalDistribute(DistributionSpec spec, Optional<GroupExpression> groupExpression,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java
index e4b3cfd2264efa..867c220d9feecf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EmptySetNode.java
@@ -43,7 +43,6 @@ public class EmptySetNode extends PlanNode {
     public EmptySetNode(PlanNodeId id, ArrayList<TupleId> tupleIds) {
         super(id, tupleIds, "EMPTYSET", StatisticalType.EMPTY_SET_NODE);
         cardinality = 0L;
-        offset = 0;
         Preconditions.checkArgument(tupleIds.size() > 0);
     }
 
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
index a61e214f05f26e..3f32ee59060ee6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/PlanNode.java
@@ -28,15 +28,12 @@
 import org.apache.doris.analysis.ExprId;
 import org.apache.doris.analysis.ExprSubstitutionMap;
 import org.apache.doris.analysis.FunctionCallExpr;
-import org.apache.doris.analysis.FunctionName;
 import org.apache.doris.analysis.SlotId;
 import org.apache.doris.analysis.SlotRef;
 import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.analysis.TupleId;
 import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Function;
 import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Type;
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.NotImplementedException;
 import org.apache.doris.common.TreeNode;
@@ -45,7 +42,6 @@
 import org.apache.doris.statistics.StatisticalType;
 import org.apache.doris.statistics.StatsDeriveResult;
 import org.apache.doris.thrift.TExplainLevel;
-import org.apache.doris.thrift.TFunctionBinaryType;
 import org.apache.doris.thrift.TPlan;
 import org.apache.doris.thrift.TPlanNode;
 import org.apache.doris.thrift.TPushAggOp;
@@ -405,24 +401,6 @@ public List<StatsDeriveResult> getChildrenStats() {
         return statsDeriveResultList;
     }
 
-    protected void initCompoundPredicate(Expr expr) {
-        if (expr instanceof CompoundPredicate) {
-            CompoundPredicate compoundPredicate = (CompoundPredicate) expr;
-            compoundPredicate.setType(Type.BOOLEAN);
-            List<Type> args = new ArrayList<>();
-            args.add(Type.BOOLEAN);
-            args.add(Type.BOOLEAN);
-            Function function = new Function(new FunctionName("", compoundPredicate.getOp().toString()),
-                    args, Type.BOOLEAN, false);
-            function.setBinaryType(TFunctionBinaryType.BUILTIN);
-            expr.setFn(function);
-        }
-
-        for (Expr child : expr.getChildren()) {
-            initCompoundPredicate(child);
-        }
-    }
-
     public static Expr convertConjunctsToAndCompoundPredicate(List<Expr> conjuncts) {
         List<Expr> targetConjuncts = Lists.newArrayList(conjuncts);
         while (targetConjuncts.size() > 1) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
index afdc5a8769a24a..60713bd53545c4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanChecker.java
@@ -259,7 +259,6 @@ public PlanChecker implement() {
             PhysicalQuickSort<? extends Plan> sort = (PhysicalQuickSort) plan;
             plan = sort.withChildren(new PhysicalDistribute<>(
                     DistributionSpecGather.INSTANCE,
-                    plan.child(0).getLogicalProperties(),
                     plan.child(0)));
         }
         physicalPlan = ((PhysicalPlan) plan);

From 243b075eef8b21d772fc619575fdf343825ec376 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Tue, 2 Jan 2024 20:24:53 +0800
Subject: [PATCH 255/299] [improve](move-memtable) report failed tablets with
 status (#29351)

---
 be/src/runtime/load_stream.cpp               | 36 ++++++++---------
 be/src/runtime/load_stream.h                 | 16 ++++----
 be/src/vec/sink/load_stream_stub.cpp         | 14 ++++---
 be/src/vec/sink/load_stream_stub.h           |  6 +--
 be/src/vec/sink/writer/vtablet_writer_v2.cpp | 41 ++++++++++++++++++++
 be/src/vec/sink/writer/vtablet_writer_v2.h   |  2 +
 be/test/runtime/load_stream_test.cpp         |  6 +--
 gensrc/proto/internal_service.proto          |  9 ++++-
 8 files changed, 90 insertions(+), 40 deletions(-)

diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index 6003f0f4d94234..c29374c51bafd7 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -291,8 +291,7 @@ Status IndexStream::_init_tablet_stream(TabletStreamSharedPtr& tablet_stream, in
 }
 
 Status IndexStream::close(const std::vector<PTabletID>& tablets_to_commit,
-                          std::vector<int64_t>* success_tablet_ids,
-                          std::vector<int64_t>* failed_tablet_ids) {
+                          std::vector<int64_t>* success_tablet_ids, FailedTablets* failed_tablets) {
     std::lock_guard lock_guard(_lock);
     SCOPED_TIMER(_close_wait_timer);
     // open all need commit tablets
@@ -311,7 +310,7 @@ Status IndexStream::close(const std::vector<PTabletID>& tablets_to_commit,
             success_tablet_ids->push_back(tablet_stream->id());
         } else {
             LOG(INFO) << "close tablet stream " << *tablet_stream << ", status=" << st;
-            failed_tablet_ids->push_back(tablet_stream->id());
+            failed_tablets->emplace_back(tablet_stream->id(), st);
         }
     }
     return Status::OK();
@@ -347,8 +346,7 @@ Status LoadStream::init(const POpenLoadStreamRequest* request) {
 }
 
 Status LoadStream::close(int64_t src_id, const std::vector<PTabletID>& tablets_to_commit,
-                         std::vector<int64_t>* success_tablet_ids,
-                         std::vector<int64_t>* failed_tablet_ids) {
+                         std::vector<int64_t>* success_tablet_ids, FailedTablets* failed_tablets) {
     std::lock_guard<bthread::Mutex> lock_guard(_lock);
     SCOPED_TIMER(_close_wait_timer);
 
@@ -371,26 +369,28 @@ Status LoadStream::close(int64_t src_id, const std::vector<PTabletID>& tablets_t
 
     for (auto& [_, index_stream] : _index_streams_map) {
         RETURN_IF_ERROR(
-                index_stream->close(_tablets_to_commit, success_tablet_ids, failed_tablet_ids));
+                index_stream->close(_tablets_to_commit, success_tablet_ids, failed_tablets));
     }
     LOG(INFO) << "close load " << *this << ", success_tablet_num=" << success_tablet_ids->size()
-              << ", failed_tablet_num=" << failed_tablet_ids->size();
+              << ", failed_tablet_num=" << failed_tablets->size();
     return Status::OK();
 }
 
-void LoadStream::_report_result(StreamId stream, const Status& st,
+void LoadStream::_report_result(StreamId stream, const Status& status,
                                 const std::vector<int64_t>& success_tablet_ids,
-                                const std::vector<int64_t>& failed_tablet_ids) {
+                                const FailedTablets& failed_tablets) {
     LOG(INFO) << "report result " << *this << ", success tablet num " << success_tablet_ids.size()
-              << ", failed tablet num " << failed_tablet_ids.size();
+              << ", failed tablet num " << failed_tablets.size();
     butil::IOBuf buf;
-    PWriteStreamSinkResponse response;
-    st.to_protobuf(response.mutable_status());
+    PLoadStreamResponse response;
+    status.to_protobuf(response.mutable_status());
     for (auto& id : success_tablet_ids) {
         response.add_success_tablet_ids(id);
     }
-    for (auto& id : failed_tablet_ids) {
-        response.add_failed_tablet_ids(id);
+    for (auto& [id, st] : failed_tablets) {
+        auto pb = response.add_failed_tablets();
+        pb->set_id(id);
+        st.to_protobuf(pb->mutable_status());
     }
 
     if (_enable_profile && _close_load_cnt == _total_streams) {
@@ -419,7 +419,7 @@ void LoadStream::_report_result(StreamId stream, const Status& st,
 
 void LoadStream::_report_schema(StreamId stream, const PStreamHeader& hdr) {
     butil::IOBuf buf;
-    PWriteStreamSinkResponse response;
+    PLoadStreamResponse response;
     Status st = Status::OK();
     for (const auto& req : hdr.tablets()) {
         TabletManager* tablet_mgr = StorageEngine::instance()->tablet_manager();
@@ -531,10 +531,10 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf*
     } break;
     case PStreamHeader::CLOSE_LOAD: {
         std::vector<int64_t> success_tablet_ids;
-        std::vector<int64_t> failed_tablet_ids;
+        FailedTablets failed_tablets;
         std::vector<PTabletID> tablets_to_commit(hdr.tablets().begin(), hdr.tablets().end());
-        auto st = close(hdr.src_id(), tablets_to_commit, &success_tablet_ids, &failed_tablet_ids);
-        _report_result(id, st, success_tablet_ids, failed_tablet_ids);
+        auto st = close(hdr.src_id(), tablets_to_commit, &success_tablet_ids, &failed_tablets);
+        _report_result(id, st, success_tablet_ids, failed_tablets);
         brpc::StreamClose(id);
     } break;
     case PStreamHeader::GET_SCHEMA: {
diff --git a/be/src/runtime/load_stream.h b/be/src/runtime/load_stream.h
index 7e16fe417ca32b..e3bbafe01b66be 100644
--- a/be/src/runtime/load_stream.h
+++ b/be/src/runtime/load_stream.h
@@ -41,6 +41,7 @@ class OlapTableSchemaParam;
 
 // origin_segid(index) -> new_segid(value in vector)
 using SegIdMapping = std::vector<uint32_t>;
+using FailedTablets = std::vector<std::pair<int64_t, Status>>;
 class TabletStream {
 public:
     TabletStream(PUniqueId load_id, int64_t id, int64_t txn_id, LoadStreamMgr* load_stream_mgr,
@@ -83,7 +84,7 @@ class IndexStream {
     Status append_data(const PStreamHeader& header, butil::IOBuf* data);
 
     Status close(const std::vector<PTabletID>& tablets_to_commit,
-                 std::vector<int64_t>* success_tablet_ids, std::vector<int64_t>* failed_tablet_ids);
+                 std::vector<int64_t>* success_tablet_ids, FailedTablets* failed_tablet_ids);
 
 private:
     Status _init_tablet_stream(TabletStreamSharedPtr& tablet_stream, int64_t tablet_id,
@@ -118,7 +119,7 @@ class LoadStream : public brpc::StreamInputHandler {
     }
 
     Status close(int64_t src_id, const std::vector<PTabletID>& tablets_to_commit,
-                 std::vector<int64_t>* success_tablet_ids, std::vector<int64_t>* failed_tablet_ids);
+                 std::vector<int64_t>* success_tablet_ids, FailedTablets* failed_tablet_ids);
 
     // callbacks called by brpc
     int on_received_messages(StreamId id, butil::IOBuf* const messages[], size_t size) override;
@@ -132,19 +133,18 @@ class LoadStream : public brpc::StreamInputHandler {
     void _dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf* data);
     Status _append_data(const PStreamHeader& header, butil::IOBuf* data);
 
-    void _report_result(StreamId stream, const Status& st,
+    void _report_result(StreamId stream, const Status& status,
                         const std::vector<int64_t>& success_tablet_ids,
-                        const std::vector<int64_t>& failed_tablet_ids);
+                        const FailedTablets& failed_tablets);
     void _report_schema(StreamId stream, const PStreamHeader& hdr);
 
     // report failure for one message
     void _report_failure(StreamId stream, const Status& status, const PStreamHeader& header) {
-        std::vector<int64_t> success; // empty
-        std::vector<int64_t> failure;
+        FailedTablets failed_tablets;
         if (header.has_tablet_id()) {
-            failure.push_back(header.tablet_id());
+            failed_tablets.emplace_back(header.tablet_id(), status);
         }
-        _report_result(stream, status, success, failure);
+        _report_result(stream, status, {}, failed_tablets);
     }
 
 private:
diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 4b557cae93d06e..6863faa4ac3416 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -33,7 +33,7 @@ int LoadStreamStub::LoadStreamReplyHandler::on_received_messages(brpc::StreamId
                                                                  size_t size) {
     for (size_t i = 0; i < size; i++) {
         butil::IOBufAsZeroCopyInputStream wrapper(*messages[i]);
-        PWriteStreamSinkResponse response;
+        PLoadStreamResponse response;
         response.ParseFromZeroCopyStream(&wrapper);
 
         Status st = Status::create(response.status());
@@ -50,14 +50,16 @@ int LoadStreamStub::LoadStreamReplyHandler::on_received_messages(brpc::StreamId
                 _success_tablets.push_back(tablet_id);
             }
         }
-        if (response.failed_tablet_ids_size() > 0) {
+        if (response.failed_tablets_size() > 0) {
             ss << ", failed tablet ids:";
-            for (auto tablet_id : response.failed_tablet_ids()) {
-                ss << " " << tablet_id;
+            for (auto pb : response.failed_tablets()) {
+                Status st = Status::create(pb.status());
+                ss << " " << pb.id() << ":" << st;
             }
             std::lock_guard<bthread::Mutex> lock(_failed_tablets_mutex);
-            for (auto tablet_id : response.failed_tablet_ids()) {
-                _failed_tablets.push_back(tablet_id);
+            for (auto pb : response.failed_tablets()) {
+                Status st = Status::create(pb.status());
+                _failed_tablets.emplace(pb.id(), st);
             }
         }
         ss << ", status: " << st;
diff --git a/be/src/vec/sink/load_stream_stub.h b/be/src/vec/sink/load_stream_stub.h
index edbbbda1e64cbf..5c9b3d2f31dff5 100644
--- a/be/src/vec/sink/load_stream_stub.h
+++ b/be/src/vec/sink/load_stream_stub.h
@@ -113,7 +113,7 @@ class LoadStreamStub {
             return _success_tablets;
         }
 
-        std::vector<int64_t> failed_tablets() {
+        std::unordered_map<int64_t, Status> failed_tablets() {
             std::lock_guard<bthread::Mutex> lock(_failed_tablets_mutex);
             return _failed_tablets;
         }
@@ -131,7 +131,7 @@ class LoadStreamStub {
         bthread::Mutex _success_tablets_mutex;
         bthread::Mutex _failed_tablets_mutex;
         std::vector<int64_t> _success_tablets;
-        std::vector<int64_t> _failed_tablets;
+        std::unordered_map<int64_t, Status> _failed_tablets;
 
         LoadStreamStub* _stub = nullptr;
     };
@@ -217,7 +217,7 @@ class LoadStreamStub {
 
     std::vector<int64_t> success_tablets() { return _handler.success_tablets(); }
 
-    std::vector<int64_t> failed_tablets() { return _handler.failed_tablets(); }
+    std::unordered_map<int64_t, Status> failed_tablets() { return _handler.failed_tablets(); }
 
     brpc::StreamId stream_id() const { return _stream_id; }
 
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index 7a83f670fbe667..d1e65c804d7bb9 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -541,17 +541,45 @@ Status VTabletWriterV2::close(Status exec_status) {
             }
         }
 
+        std::unordered_map<int64_t, int> success_tablets;
+        std::unordered_map<int64_t, int> failed_tablets;
+
         std::vector<TTabletCommitInfo> tablet_commit_infos;
         for (const auto& [node_id, streams] : _streams_for_node) {
             for (const auto& stream : streams->streams()) {
+                std::unordered_set<int64_t> known_tablets;
+                for (auto [tablet_id, _] : stream->failed_tablets()) {
+                    if (known_tablets.contains(tablet_id)) {
+                        continue;
+                    }
+                    known_tablets.insert(tablet_id);
+                    failed_tablets[tablet_id]++;
+                }
                 for (auto tablet_id : stream->success_tablets()) {
+                    if (known_tablets.contains(tablet_id)) {
+                        continue;
+                    }
+                    known_tablets.insert(tablet_id);
                     TTabletCommitInfo commit_info;
                     commit_info.tabletId = tablet_id;
                     commit_info.backendId = node_id;
                     tablet_commit_infos.emplace_back(std::move(commit_info));
+                    success_tablets[tablet_id]++;
                 }
             }
         }
+        for (auto [tablet_id, replicas] : success_tablets) {
+            if (replicas > _num_replicas / 2) {
+                continue;
+            }
+            return _failed_reason(tablet_id);
+        }
+        for (auto [tablet_id, replicas] : failed_tablets) {
+            if (replicas <= (_num_replicas - 1) / 2) {
+                continue;
+            }
+            return _failed_reason(tablet_id);
+        }
         _state->tablet_commit_infos().insert(_state->tablet_commit_infos().end(),
                                              std::make_move_iterator(tablet_commit_infos.begin()),
                                              std::make_move_iterator(tablet_commit_infos.end()));
@@ -576,6 +604,19 @@ Status VTabletWriterV2::close(Status exec_status) {
     return status;
 }
 
+Status VTabletWriterV2::_failed_reason(int64_t tablet_id) {
+    Status st = Status::InternalError("tablet {} failed", tablet_id);
+    auto backends = _location->find_tablet(tablet_id)->node_ids;
+    for (auto& backend_id : backends) {
+        auto failed_tablets = _streams_for_node[backend_id]->streams()[0]->failed_tablets();
+        if (failed_tablets.contains(tablet_id)) {
+            st = failed_tablets[tablet_id];
+            break;
+        }
+    }
+    return st;
+}
+
 Status VTabletWriterV2::_close_load(const Streams& streams) {
     auto node_id = streams[0]->dst_id();
     std::vector<PTabletID> tablets_to_commit;
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.h b/be/src/vec/sink/writer/vtablet_writer_v2.h
index e99bbd92696cfb..c829b254c77e63 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.h
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.h
@@ -140,6 +140,8 @@ class VTabletWriterV2 final : public AsyncResultWriter {
     Status _select_streams(int64_t tablet_id, int64_t partition_id, int64_t index_id,
                            Streams& streams);
 
+    Status _failed_reason(int64_t tablet_id);
+
     Status _close_load(const Streams& streams);
 
     Status _cancel(Status status);
diff --git a/be/test/runtime/load_stream_test.cpp b/be/test/runtime/load_stream_test.cpp
index 6509f5aa35d856..0a0ce76f191e89 100644
--- a/be/test/runtime/load_stream_test.cpp
+++ b/be/test/runtime/load_stream_test.cpp
@@ -324,7 +324,7 @@ class LoadStreamMgrTest : public testing::Test {
         int on_received_messages(StreamId id, butil::IOBuf* const messages[],
                                  size_t size) override {
             for (size_t i = 0; i < size; i++) {
-                PWriteStreamSinkResponse response;
+                PLoadStreamResponse response;
                 butil::IOBufAsZeroCopyInputStream wrapper(*messages[i]);
                 response.ParseFromZeroCopyStream(&wrapper);
                 LOG(INFO) << "response " << response.DebugString();
@@ -332,8 +332,8 @@ class LoadStreamMgrTest : public testing::Test {
                 for (auto& id : response.success_tablet_ids()) {
                     g_response_stat.success_tablet_ids.push_back(id);
                 }
-                for (auto& id : response.failed_tablet_ids()) {
-                    g_response_stat.failed_tablet_ids.push_back(id);
+                for (auto& tablet : response.failed_tablets()) {
+                    g_response_stat.failed_tablet_ids.push_back(tablet.id());
                 }
                 g_response_stat.num++;
             }
diff --git a/gensrc/proto/internal_service.proto b/gensrc/proto/internal_service.proto
index 46e2e194f068bf..cefcd073b846a3 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -767,10 +767,15 @@ message POpenLoadStreamResponse {
     repeated PTabletSchemaWithIndex tablet_schemas = 2;
 }
 
-message PWriteStreamSinkResponse {
+message PFailedTablet {
+    optional int64 id = 1;
+    optional PStatus status = 2;
+}
+
+message PLoadStreamResponse {
     optional PStatus status = 1;
     repeated int64 success_tablet_ids = 2;
-    repeated int64 failed_tablet_ids = 3;
+    repeated PFailedTablet failed_tablets = 3;
     optional bytes load_stream_profile = 4;
     repeated PTabletSchemaWithIndex tablet_schemas = 5;
 }

From 9490d5e9a26497779cd02ed80bb56020f7ffbaa0 Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Tue, 2 Jan 2024 20:28:59 +0800
Subject: [PATCH 256/299] [Debug](Variant) sanitize variant in
 `write_column_to_mysql` (#29380)

---
 be/src/vec/columns/column_object.cpp          | 34 +++++++++++++++++++
 be/src/vec/columns/column_object.h            |  5 +++
 .../serde/data_type_object_serde.cpp          |  2 ++
 3 files changed, 41 insertions(+)

diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index fb2bf8c9072e53..8a21bc76d284d4 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -27,6 +27,8 @@
 #include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 
+#include <algorithm>
+#include <cstdlib>
 #include <functional>
 #include <limits>
 #include <map>
@@ -1473,4 +1475,36 @@ void ColumnObject::for_each_imutable_subcolumn(ImutableColumnCallback callback)
     }
 }
 
+std::string ColumnObject::debug_string() const {
+    std::stringstream res;
+    res << get_family_name() << "(num_row = " << num_rows;
+    for (auto& entry : subcolumns) {
+        if (entry->data.is_finalized()) {
+            res << "[column:" << entry->data.data[0]->dump_structure()
+                << ",type:" << entry->data.data_types[0]->get_name()
+                << ",path:" << entry->path.get_path() << "],";
+        }
+    }
+    res << ")";
+    return res.str();
+}
+
+Status ColumnObject::sanitize() const {
+    RETURN_IF_CATCH_EXCEPTION(check_consistency());
+    for (const auto& subcolumn : subcolumns) {
+        if (subcolumn->data.is_finalized()) {
+            auto column = subcolumn->data.get_least_common_type()->create_column();
+            std::string original = subcolumn->data.get_finalized_column().get_family_name();
+            std::string expected = column->get_family_name();
+            if (original != expected) {
+                return Status::InternalError("Incompatible type between {} and {}, debug_info:",
+                                             original, expected, debug_string());
+            }
+        }
+    }
+
+    VLOG_DEBUG << "sanitized " << debug_string();
+    return Status::OK();
+}
+
 } // namespace doris::vectorized
diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h
index 385411cb486af6..86f3bd19ceb04e 100644
--- a/be/src/vec/columns/column_object.h
+++ b/be/src/vec/columns/column_object.h
@@ -476,5 +476,10 @@ class ColumnObject final : public COWHelper<IColumn, ColumnObject> {
     void strip_outer_array();
 
     bool empty() const;
+
+    // Check if all columns and types are aligned
+    Status sanitize() const;
+
+    std::string debug_string() const;
 };
 } // namespace doris::vectorized
diff --git a/be/src/vec/data_types/serde/data_type_object_serde.cpp b/be/src/vec/data_types/serde/data_type_object_serde.cpp
index 94ee6a44da5c1d..5b227951e42a5d 100644
--- a/be/src/vec/data_types/serde/data_type_object_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_object_serde.cpp
@@ -19,6 +19,7 @@
 
 #include <rapidjson/stringbuffer.h>
 
+#include "common/status.h"
 #include "vec/columns/column_object.h"
 #include "vec/common/assert_cast.h"
 #include "vec/common/schema_util.h"
@@ -34,6 +35,7 @@ Status DataTypeObjectSerDe::write_column_to_mysql(const IColumn& column,
     if (!variant.is_finalized()) {
         const_cast<ColumnObject&>(variant).finalize();
     }
+    RETURN_IF_ERROR(variant.sanitize());
     if (variant.is_scalar_variant()) {
         // Serialize scalar types, like int, string, array, faster path
         const auto& root = variant.get_subcolumn({});

From 5db496d84490fe00e1dc35bcbd2b6715690efb5d Mon Sep 17 00:00:00 2001
From: lihangyu <15605149486@163.com>
Date: Tue, 2 Jan 2024 20:29:17 +0800
Subject: [PATCH 257/299] [Improve](Variant) make output stable (#29389)

---
 be/src/vec/columns/column_object.cpp          | 19 ++--
 be/src/vec/columns/subcolumn_tree.h           | 19 ++++
 .../variant_p0/compaction/test_compaction.out | 96 +++++++++----------
 .../data/variant_p0/complexjson.out           | 10 +-
 .../data/variant_p0/delete_update.out         | 10 +-
 .../data/variant_p0/insert_into_select.out    | 10 +-
 regression-test/data/variant_p0/load.out      | 76 +++++++--------
 .../test_alter_table_column_with_delete.out   | 10 +-
 .../data/variant_p0/sql/gh_data.out           | 20 ++--
 .../data/variant_p0/with_index/load.out       |  6 +-
 .../data/variant_p0/with_index/var_index.out  | 10 +-
 regression-test/suites/variant_p0/desc.groovy |  4 +-
 12 files changed, 151 insertions(+), 139 deletions(-)

diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index 8a21bc76d284d4..77366d05cb58b2 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -34,6 +34,7 @@
 #include <map>
 #include <memory>
 #include <optional>
+#include <vector>
 
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "common/exception.h"
@@ -1009,12 +1010,12 @@ void get_json_by_column_tree(rapidjson::Value& root, rapidjson::Document::Alloca
         return;
     }
     root.SetObject();
-    for (auto it = node_root->children.begin(); it != node_root->children.end(); ++it) {
-        auto child = it->get_second();
+    // sort to make output stable
+    std::vector<StringRef> sorted_keys = node_root->get_sorted_chilren_keys();
+    for (const StringRef& key : sorted_keys) {
         rapidjson::Value value(rapidjson::kObjectType);
-        get_json_by_column_tree(value, allocator, child.get());
-        root.AddMember(rapidjson::StringRef(it->get_first().data, it->get_first().size), value,
-                       allocator);
+        get_json_by_column_tree(value, allocator, node_root->get_child_node(key).get());
+        root.AddMember(rapidjson::StringRef(key.data, key.size), value, allocator);
     }
 }
 
@@ -1081,14 +1082,6 @@ bool ColumnObject::serialize_one_row_to_json_format(int row, rapidjson::StringBu
     VLOG_DEBUG << "dump structure " << JsonFunctions::print_json_value(*doc_structure);
 #endif
     for (const auto& subcolumn : subcolumns) {
-        if (subcolumn->data.data.empty() || subcolumn->data.get_finalized_column_ptr() == nullptr) {
-            // TODO this is a tmp defensive code to prevent from crash and
-            // print more info about crash info
-            LOG(WARNING) << "Dump crash debug info"
-                         << ", structure:" << JsonFunctions::print_json_value(*doc_structure)
-                         << ", num_rows: " << num_rows << ", row_position: " << row;
-            return false;
-        }
         find_and_set_leave_value(subcolumn->data.get_finalized_column_ptr(), subcolumn->path,
                                  subcolumn->data.get_least_common_type_serde(), root,
                                  doc_structure->GetAllocator(), row);
diff --git a/be/src/vec/columns/subcolumn_tree.h b/be/src/vec/columns/subcolumn_tree.h
index f895b94dedf3f2..53391bf797fdb2 100644
--- a/be/src/vec/columns/subcolumn_tree.h
+++ b/be/src/vec/columns/subcolumn_tree.h
@@ -19,8 +19,11 @@
 // and modified by Doris
 
 #pragma once
+#include <memory>
+
 #include "vec/columns/column.h"
 #include "vec/common/hash_table/hash_map.h"
+#include "vec/common/string_ref.h"
 #include "vec/data_types/data_type.h"
 #include "vec/json/path_in_data.h"
 namespace doris::vectorized {
@@ -72,6 +75,22 @@ class SubcolumnsTree {
             StringRef key_ref {strings_pool.insert(key.data(), key.length()), key.length()};
             children[key_ref] = std::move(next_node);
         }
+
+        std::vector<StringRef> get_sorted_chilren_keys() const {
+            std::vector<StringRef> sorted_keys;
+            for (auto it = children.begin(); it != children.end(); ++it) {
+                sorted_keys.push_back(it->get_first());
+            }
+            std::sort(sorted_keys.begin(), sorted_keys.end());
+            return sorted_keys;
+        }
+        std::shared_ptr<const Node> get_child_node(StringRef key) const {
+            auto it = children.find(key);
+            if (it != children.end()) {
+                return it->get_second();
+            }
+            return nullptr;
+        }
     };
 
     using NodeKind = typename Node::Kind;
diff --git a/regression-test/data/variant_p0/compaction/test_compaction.out b/regression-test/data/variant_p0/compaction/test_compaction.out
index 5118c2824e100d..50943aef7b658a 100644
--- a/regression-test/data/variant_p0/compaction/test_compaction.out
+++ b/regression-test/data/variant_p0/compaction/test_compaction.out
@@ -36,22 +36,22 @@
 17	{"a":[1]}
 18	{"a":["1",2,1.1]}
 18	{"a":["1",2,1.1]}
-19	{"b":{"c":1},"a":1}
-19	{"b":{"c":1},"a":1}
-20	{"b":{"c":[{"a":1}]},"a":1}
-20	{"b":{"c":[{"a":1}]},"a":1}
-21	{"b":{"c":[{"a":1}]},"a":1}
-21	{"b":{"c":[{"a":1}]},"a":1}
-22	{"b":{"c":[{"a":1}]},"a":1}
-22	{"b":{"c":[{"a":1}]},"a":1}
-1022	{"b":10,"a":1}
-1022	{"b":10,"a":1}
-1029	{"b":{"c":1},"a":1}
-1029	{"b":{"c":1},"a":1}
-1999	{"b":{"c":1},"a":1}
-1999	{"b":{"c":1},"a":1}
-19921	{"b":10,"a":1}
-19921	{"b":10,"a":1}
+19	{"a":1,"b":{"c":1}}
+19	{"a":1,"b":{"c":1}}
+20	{"a":1,"b":{"c":[{"a":1}]}}
+20	{"a":1,"b":{"c":[{"a":1}]}}
+21	{"a":1,"b":{"c":[{"a":1}]}}
+21	{"a":1,"b":{"c":[{"a":1}]}}
+22	{"a":1,"b":{"c":[{"a":1}]}}
+22	{"a":1,"b":{"c":[{"a":1}]}}
+1022	{"a":1,"b":10}
+1022	{"a":1,"b":10}
+1029	{"a":1,"b":{"c":1}}
+1029	{"a":1,"b":{"c":1}}
+1999	{"a":1,"b":{"c":1}}
+1999	{"a":1,"b":{"c":1}}
+19921	{"a":1,"b":10}
+19921	{"a":1,"b":10}
 
 -- !sql_2 --
 14	[null]
@@ -124,22 +124,22 @@
 17	{"a":[1]}
 18	{"a":["1",2,1.1]}
 18	{"a":["1",2,1.1]}
-19	{"b":{"c":1},"a":1}
-19	{"b":{"c":1},"a":1}
-20	{"b":{"c":[{"a":1}]},"a":1}
-20	{"b":{"c":[{"a":1}]},"a":1}
-21	{"b":{"c":[{"a":1}]},"a":1}
-21	{"b":{"c":[{"a":1}]},"a":1}
-22	{"b":{"c":[{"a":1}]},"a":1}
-22	{"b":{"c":[{"a":1}]},"a":1}
-1022	{"b":10,"a":1}
-1022	{"b":10,"a":1}
-1029	{"b":{"c":1},"a":1}
-1029	{"b":{"c":1},"a":1}
-1999	{"b":{"c":1},"a":1}
-1999	{"b":{"c":1},"a":1}
-19921	{"b":10,"a":1}
-19921	{"b":10,"a":1}
+19	{"a":1,"b":{"c":1}}
+19	{"a":1,"b":{"c":1}}
+20	{"a":1,"b":{"c":[{"a":1}]}}
+20	{"a":1,"b":{"c":[{"a":1}]}}
+21	{"a":1,"b":{"c":[{"a":1}]}}
+21	{"a":1,"b":{"c":[{"a":1}]}}
+22	{"a":1,"b":{"c":[{"a":1}]}}
+22	{"a":1,"b":{"c":[{"a":1}]}}
+1022	{"a":1,"b":10}
+1022	{"a":1,"b":10}
+1029	{"a":1,"b":{"c":1}}
+1029	{"a":1,"b":{"c":1}}
+1999	{"a":1,"b":{"c":1}}
+1999	{"a":1,"b":{"c":1}}
+19921	{"a":1,"b":10}
+19921	{"a":1,"b":10}
 
 -- !sql_22 --
 14	[null]
@@ -194,14 +194,14 @@
 16	{"a":"1223"}
 17	{"a":[1]}
 18	{"a":["1",2,1.1]}
-19	{"b":{"c":1},"a":1}
-20	{"b":{"c":[{"a":1}]},"a":1}
-21	{"b":{"c":[{"a":1}]},"a":1}
-22	{"b":{"c":[{"a":1}]},"a":1}
-1022	{"b":10,"a":1}
-1029	{"b":{"c":1},"a":1}
-1999	{"b":{"c":1},"a":1}
-19921	{"b":10,"a":1}
+19	{"a":1,"b":{"c":1}}
+20	{"a":1,"b":{"c":[{"a":1}]}}
+21	{"a":1,"b":{"c":[{"a":1}]}}
+22	{"a":1,"b":{"c":[{"a":1}]}}
+1022	{"a":1,"b":10}
+1029	{"a":1,"b":{"c":1}}
+1999	{"a":1,"b":{"c":1}}
+19921	{"a":1,"b":10}
 
 -- !sql_2 --
 14	[null]
@@ -247,14 +247,14 @@
 16	{"a":"1223"}
 17	{"a":[1]}
 18	{"a":["1",2,1.1]}
-19	{"b":{"c":1},"a":1}
-20	{"b":{"c":[{"a":1}]},"a":1}
-21	{"b":{"c":[{"a":1}]},"a":1}
-22	{"b":{"c":[{"a":1}]},"a":1}
-1022	{"b":10,"a":1}
-1029	{"b":{"c":1},"a":1}
-1999	{"b":{"c":1},"a":1}
-19921	{"b":10,"a":1}
+19	{"a":1,"b":{"c":1}}
+20	{"a":1,"b":{"c":[{"a":1}]}}
+21	{"a":1,"b":{"c":[{"a":1}]}}
+22	{"a":1,"b":{"c":[{"a":1}]}}
+1022	{"a":1,"b":10}
+1029	{"a":1,"b":{"c":1}}
+1999	{"a":1,"b":{"c":1}}
+19921	{"a":1,"b":10}
 
 -- !sql_22 --
 14	[null]
diff --git a/regression-test/data/variant_p0/complexjson.out b/regression-test/data/variant_p0/complexjson.out
index d5a5154c670e91..9f55cd03a22574 100644
--- a/regression-test/data/variant_p0/complexjson.out
+++ b/regression-test/data/variant_p0/complexjson.out
@@ -1,17 +1,17 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-1	{"key_0":[{"key_1":[{"key_3":[{"key_7":1025,"key_6":25.5,"key_4":1048576,"key_5":0.0001048576},{"key_7":2,"key_6":"","key_4":null}]}]},{"key_1":[{"key_3":[{"key_7":-922337203685477600.0,"key_6":"aqbjfiruu","key_5":-1},{"key_7":65537,"key_6":"","key_4":""}]},{"key_3":[{"key_7":21474836.48,"key_4":"ghdqyeiom","key_5":1048575}]}]}],"id":1}
+1	{"id":1,"key_0":[{"key_1":[{"key_3":[{"key_7":1025,"key_6":25.5,"key_4":1048576,"key_5":0.0001048576},{"key_7":2,"key_6":"","key_4":null}]}]},{"key_1":[{"key_3":[{"key_7":-922337203685477600.0,"key_6":"aqbjfiruu","key_5":-1},{"key_7":65537,"key_6":"","key_4":""}]},{"key_3":[{"key_7":21474836.48,"key_4":"ghdqyeiom","key_5":1048575}]}]}]}
 
 -- !sql --
-1	{"key_1":[{"key_2":[{"key_3":[{"key_8":65537},{"key_4":[{"key_5":-0.02},{"key_7":1023},{"key_7":1,"key_6":9223372036854775807}]},{"key_4":[{"key_7":65537,"key_6":null}]}]}]}],"id":1}
+1	{"id":1,"key_1":[{"key_2":[{"key_3":[{"key_8":65537},{"key_4":[{"key_5":-0.02},{"key_7":1023},{"key_7":1,"key_6":9223372036854775807}]},{"key_4":[{"key_7":65537,"key_6":null}]}]}]}]}
 
 -- !sql --
-1	{"key_0":{"key_4":1,"key_1":{"key_2":1025,"key_3":1},"key_5":256},"key_11":"anve","key_10":65536}
+1	{"key_0":{"key_1":{"key_2":1025,"key_3":1},"key_4":1,"key_5":256},"key_10":65536,"key_11":"anve"}
 2	{"key_0":[{"key_12":"buwvq","key_11":2.55e-8}]}
 
 -- !sql --
-1	{"key_0":[{"key_1":{"key_2":[1,2,3],"key_8":"sffjx"},"key_10":65535,"key_0":-1},{"key_10":10.23,"key_0":922337203.685}],"id":1}
+1	{"id":1,"key_0":[{"key_1":{"key_2":[1,2,3],"key_8":"sffjx"},"key_10":65535,"key_0":-1},{"key_10":10.23,"key_0":922337203.685}]}
 
 -- !sql --
-1	{"key_0":[{"key_1":[{"key_2":{"key_3":[{"key_4":255},{"key_4":65535},{"key_7":255,"key_6":3}],"key_5":[{"key_7":"nnpqx","key_6":1},{"key_7":255,"key_6":3}]}}]}],"id":1}
+1	{"id":1,"key_0":[{"key_1":[{"key_2":{"key_3":[{"key_4":255},{"key_4":65535},{"key_7":255,"key_6":3}],"key_5":[{"key_7":"nnpqx","key_6":1},{"key_7":255,"key_6":3}]}}]}]}
 
diff --git a/regression-test/data/variant_p0/delete_update.out b/regression-test/data/variant_p0/delete_update.out
index 6f4eb1123fbc29..0e478bdeb0df5c 100644
--- a/regression-test/data/variant_p0/delete_update.out
+++ b/regression-test/data/variant_p0/delete_update.out
@@ -1,16 +1,16 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
 2	{"updated_value":123}
-3	{"c":3.0,"a":3,"b":[3]}
-4	{"c":4.0,"a":4,"b":[4]}
-5	{"c":5.0,"a":5,"b":[5]}
+3	{"a":3,"b":[3],"c":3.0}
+4	{"a":4,"b":[4],"c":4.0}
+5	{"a":5,"b":[5],"c":5.0}
 
 -- !sql --
 2	{"updated_value":123}	{"updated_value":123}
-6	{"c":4.0,"a":4,"b":[4]}	{"updated_value" : 123}
+6	{"a":4,"b":[4],"c":4.0}	{"updated_value" : 123}
 7	{"updated_value":1111}	yyy
 
 -- !sql --
 2	{"updated_value":123}	{"updated_value":123}
-6	{"c":4.0,"a":4,"b":[4]}	{"updated_value" : 123}
+6	{"a":4,"b":[4],"c":4.0}	{"updated_value" : 123}
 
diff --git a/regression-test/data/variant_p0/insert_into_select.out b/regression-test/data/variant_p0/insert_into_select.out
index ea1714d2ca100b..3414b2f566487f 100644
--- a/regression-test/data/variant_p0/insert_into_select.out
+++ b/regression-test/data/variant_p0/insert_into_select.out
@@ -22,9 +22,9 @@
 5	[5]	5
 
 -- !sql --
-{"c":1.0,"a":1,"b":[1]}
-{"c":2.0,"a":2,"b":[1]}
-{"c":3.0,"a":3,"b":[3]}
-{"c":4.0,"a":4,"b":[4]}
-{"c":5.0,"a":5,"b":[5]}
+{"a":1,"b":[1],"c":1.0}
+{"a":2,"b":[1],"c":2.0}
+{"a":3,"b":[3],"c":3.0}
+{"a":4,"b":[4],"c":4.0}
+{"a":5,"b":[5],"c":5.0}
 
diff --git a/regression-test/data/variant_p0/load.out b/regression-test/data/variant_p0/load.out
index 7def94d2fadeef..a3d237dc1568d7 100644
--- a/regression-test/data/variant_p0/load.out
+++ b/regression-test/data/variant_p0/load.out
@@ -86,9 +86,9 @@
 123	{"A":123}
 123456	{"A":123456}
 123456789101112	{"A":123456789101112}
-191191	{"c":123,"A":191191,"a":123.0}
-1800	{"c":[12345],"A":1800,"a":1.10111}
-17211	{"c":111111,"A":17211,"a":1.1111}
+191191	{"A":191191,"a":123.0,"c":123}
+1800	{"A":1800,"a":1.10111,"c":[12345]}
+17211	{"A":17211,"a":1.1111,"c":111111}
 
 -- !sql_13 --
 \N	123
@@ -106,10 +106,10 @@
 \N	123456789101112	{"A":123456789101112}	\N
 \N	\N	{"AA":[123456]}	\N
 \N	\N	{"AA":[123456789101112]}	\N
-123	191191	{"c":123,"A":191191,"a":123.0}	\N
-123	\N	{"c":123456,"a":"123"}	\N
-1.10111	1800	{"c":[12345],"A":1800,"a":1.10111}	\N
-1.1111	17211	{"c":111111,"A":17211,"a":1.1111}	\N
+123	191191	{"A":191191,"a":123.0,"c":123}	\N
+123	\N	{"a":"123","c":123456}	\N
+1.10111	1800	{"A":1800,"a":1.10111,"c":[12345]}	\N
+1.1111	17211	{"A":17211,"a":1.1111,"c":111111}	\N
 
 -- !sql_19 --
 \N	\N	{"oamama":1.1}	1.1
@@ -118,7 +118,7 @@
 123456
 
 -- !sql_21_1 --
-12	{"yyy":456,"xxx":123}
+12	{"xxx":123,"yyy":456}
 
 -- !sql_21_2 --
 [123456]
@@ -143,7 +143,7 @@
 [123]
 
 -- !sql_25 --
-50000	55000.000000002256	6150000
+50000	55000.00000000226	6150000
 
 -- !sql_26 --
 5000
@@ -173,27 +173,27 @@
 
 -- !sql_30 --
 {"a":1123}
-{"a":11245,"c":{"c":456,"e":7.111},"b":[123,{"xx":1}]}
-{"xxxx":"kaana","a":1234}
-{"xxxx":"kaana","a":1234}
-{"xxxx":"kaana","a":1234}
-{"xxxx":"kaana","a":1234}
-{"xxxx":"kaana","a":1234}
-{"xxxx":"kaana","a":1234}
-{"xxxx":"kaana","a":1234}
-{"xxxx":"kaana","a":1234}
+{"a":11245,"b":[123,{"xx":1}],"c":{"c":456,"e":7.111}}
+{"a":1234,"xxxx":"kaana"}
+{"a":1234,"xxxx":"kaana"}
+{"a":1234,"xxxx":"kaana"}
+{"a":1234,"xxxx":"kaana"}
+{"a":1234,"xxxx":"kaana"}
+{"a":1234,"xxxx":"kaana"}
+{"a":1234,"xxxx":"kaana"}
+{"a":1234,"xxxx":"kaana"}
 
 -- !sql_31 --
-{"a":1123,"c":{"c":456,"e":7.111},"oooo":{"xxxx":{"xxx":123}},"b":[123,{"xx":1}]}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
-{"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana","a":1234}
+{"a":1123,"b":[123,{"xx":1}],"c":{"c":456,"e":7.111},"oooo":{"xxxx":{"xxx":123}}}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
+{"a":1234,"ddd":{"aaa":123,"mxmxm":[456,"789"]},"xxxx":"kaana"}
 
 -- !sql_32 --
 "2023-06-21 16:35:58.468  INFO [sino-webhook,6dee61f0605a70f3,6dee61f0605a70f3,true] 1 --- [io-8001-exec-13] c.s.c.a.CustomRequestBodyAdviceAdapter   : REQUEST DATA: {String={\\"id\\":\\"3293880a36cd163754ea4f90270331f6\\",\\"token\\":\\"3293880a36cd163754ea4f90270331f6\\",\\"line_items\\":[{\\"id\\":43073082228985,\\"properties\\":{},\\"quantity\\":1000000,\\"variant_id\\":43073082228985,\\"key\\":\\"43073082228985:381f0b4b03d0c76493aa028c4ed006a9\\",\\"discounted_price\\":\\"28.95\\",\\"discounts\\":[],\\"gift_card\\":false,\\"grams\\":0,\\"line_price\\":\\"28950000.00\\",\\"original_line_price\\":\\"28950000.00\\",\\"original_price\\":\\"28.95\\",\\"price\\":\\"28.95\\",\\"product_id\\":7706860617977,\\"sku\\":\\"56011440_Shirt(A)||Brown||M9\\",\\"taxable\\":false,\\"title\\":\\"Men's Hawaiian Short Sleeve Shirt - Shirt(A) \\\\/ Brown \\\\/ M\\",\\"total_discount\\":\\"0.00\\",\\"vendor\\":\\"XT\\",\\"discounted_price_set\\":{\\"shop_money\\":{\\"amount\\":\\"28.95\\",\\"currency_code\\":\\"USD\\"},\\"presentment_money\\":{\\"amount\\":\\"28.95\\",\\"currency_code\\":\\"USD\\"}},\\"line_price_set\\":{\\"shop_money\\":{\\"amount\\":\\"28950000.0\\",\\"currency_code\\":\\"USD\\"},\\"presentment_money\\":{\\"amount\\":\\"28950000.0\\",\\"currency_code\\":\\"USD\\"}},\\"original_line_price_set\\":{\\"shop_money\\":{\\"amount\\":\\"28950000.0\\",\\"currency_code\\":\\"USD\\"},\\"presentment_money\\":{\\"amount\\":\\"28950000.0\\",\\"currency_code\\":\\"USD\\"}},\\"price_set\\":{\\"shop_money\\":{\\"amount\\":\\"28.95\\",\\"currency_code\\":\\"USD\\"},\\"presentment_money\\":{\\"amount\\":\\"28.95\\",\\"currency_code\\":\\"USD\\"}},\\"total_discount_set\\":{\\"shop_money\\":{\\"amount\\":\\"0.0\\",\\"currency_code\\":\\"USD\\"},\\"presentment_money\\":{\\"amount\\":\\"0.0\\",\\"currency_code\\":\\"USD\\"}}}],\\"note\\":null,\\"updated_at\\":\\"2023-06-21T08:35:56.674Z\\",\\"created_at\\":\\"2023-06-21T08:35:48.174Z\\"}}"
@@ -227,16 +227,16 @@
 \N	\N	\N
 
 -- !sql_36_2 --
-7702	{"url":"https://api.github.com/repos/magik6k/BitBuffer","id":28677864,"name":"magik6k/BitBuffer"}
-7701	{"url":"https://api.github.com/repos/wllmtrng/wllmtrng.github.io","id":18089434,"name":"wllmtrng/wllmtrng.github.io"}
-7700	{"url":"https://api.github.com/repos/ACID-Scripts/TBC","id":23724137,"name":"ACID-Scripts/TBC"}
-7699	{"url":"https://api.github.com/repos/chrisjimenez/IpsePuppet","id":28678128,"name":"chrisjimenez/IpsePuppet"}
-7698	{"url":"https://api.github.com/repos/antonioortegajr/beerfind.me","id":28573267,"name":"antonioortegajr/beerfind.me"}
-7697	{"url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs","id":20463939,"name":"XLabs/Xamarin-Forms-Labs"}
-7696	{"url":"https://api.github.com/repos/Fathalian/Guild","id":26995510,"name":"Fathalian/Guild"}
-7695	{"url":"https://api.github.com/repos/kevinhofmaenner/blackjack","id":28652857,"name":"kevinhofmaenner/blackjack"}
-7694	{"url":"https://api.github.com/repos/marklrh/ocaml-cohttp-test","id":27470715,"name":"marklrh/ocaml-cohttp-test"}
-7693	{"url":"https://api.github.com/repos/plouc/mozaik","id":28498113,"name":"plouc/mozaik"}
+7702	{"id":28677864,"name":"magik6k/BitBuffer","url":"https://api.github.com/repos/magik6k/BitBuffer"}
+7701	{"id":18089434,"name":"wllmtrng/wllmtrng.github.io","url":"https://api.github.com/repos/wllmtrng/wllmtrng.github.io"}
+7700	{"id":23724137,"name":"ACID-Scripts/TBC","url":"https://api.github.com/repos/ACID-Scripts/TBC"}
+7699	{"id":28678128,"name":"chrisjimenez/IpsePuppet","url":"https://api.github.com/repos/chrisjimenez/IpsePuppet"}
+7698	{"id":28573267,"name":"antonioortegajr/beerfind.me","url":"https://api.github.com/repos/antonioortegajr/beerfind.me"}
+7697	{"id":20463939,"name":"XLabs/Xamarin-Forms-Labs","url":"https://api.github.com/repos/XLabs/Xamarin-Forms-Labs"}
+7696	{"id":26995510,"name":"Fathalian/Guild","url":"https://api.github.com/repos/Fathalian/Guild"}
+7695	{"id":28652857,"name":"kevinhofmaenner/blackjack","url":"https://api.github.com/repos/kevinhofmaenner/blackjack"}
+7694	{"id":27470715,"name":"marklrh/ocaml-cohttp-test","url":"https://api.github.com/repos/marklrh/ocaml-cohttp-test"}
+7693	{"id":28498113,"name":"plouc/mozaik","url":"https://api.github.com/repos/plouc/mozaik"}
 
 -- !sql_36_3 --
 2	{"updated_value":10}
diff --git a/regression-test/data/variant_p0/schema_change/test_alter_table_column_with_delete.out b/regression-test/data/variant_p0/schema_change/test_alter_table_column_with_delete.out
index a23f0b96c4e98e..8b458573cf8c52 100644
--- a/regression-test/data/variant_p0/schema_change/test_alter_table_column_with_delete.out
+++ b/regression-test/data/variant_p0/schema_change/test_alter_table_column_with_delete.out
@@ -1,12 +1,12 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-1	{"c":3,"b":2,"a":1}	1.0
+1	{"a":1,"b":2,"c":3}	1.0
 3	{"g":7,"h":8,"i":9}	3.0
-4	{"k":11,"l":12,"j":10}	4.0
+4	{"j":10,"k":11,"l":12}	4.0
 
 -- !sql --
-1	{"c":3,"b":2,"a":1}	1
-2	{"y":5,"x":4,"f":3}	2
+1	{"a":1,"b":2,"c":3}	1
+2	{"f":3,"x":4,"y":5}	2
 3	{"g":7,"h":8,"i":9}	3
-4	{"k":11,"l":12,"j":10}	4
+4	{"j":10,"k":11,"l":12}	4
 
diff --git a/regression-test/data/variant_p0/sql/gh_data.out b/regression-test/data/variant_p0/sql/gh_data.out
index 4c34edabe48205..77e38f4ac2647c 100644
--- a/regression-test/data/variant_p0/sql/gh_data.out
+++ b/regression-test/data/variant_p0/sql/gh_data.out
@@ -46,16 +46,16 @@ disclose/diodata	2
 73801003
 
 -- !gh_data_12 --
-27	{"url":"https://api.github.com/repos/leonardomso/33-js-concepts","id":147350463,"name":"leonardomso/33-js-concepts"}
-36	{"url":"https://api.github.com/repos/odeke-em/drive","id":26109545,"name":"odeke-em/drive"}
-46	{"url":"https://api.github.com/repos/GO-LiFE/GoFIT_SDK_Android","id":141905736,"name":"GO-LiFE/GoFIT_SDK_Android"}
-56	{"url":"https://api.github.com/repos/MrXujiang/h5-Dooring","id":289417971,"name":"MrXujiang/h5-Dooring"}
-86	{"url":"https://api.github.com/repos/redsaph/cleartext","id":106453399,"name":"redsaph/cleartext"}
-98	{"url":"https://api.github.com/repos/sherlock-project/sherlock","id":162998479,"name":"sherlock-project/sherlock"}
-101	{"url":"https://api.github.com/repos/okandavut/react-spotify-nowplaying","id":326215605,"name":"okandavut/react-spotify-nowplaying"}
-112	{"url":"https://api.github.com/repos/sentriz/gonic","id":178435468,"name":"sentriz/gonic"}
-122	{"url":"https://api.github.com/repos/netlify-labs/react-netlify-identity-widget","id":182606378,"name":"netlify-labs/react-netlify-identity-widget"}
-169	{"url":"https://api.github.com/repos/microsoft/BotBuilder-Samples","id":68730444,"name":"microsoft/BotBuilder-Samples"}
+27	{"id":147350463,"name":"leonardomso/33-js-concepts","url":"https://api.github.com/repos/leonardomso/33-js-concepts"}
+36	{"id":26109545,"name":"odeke-em/drive","url":"https://api.github.com/repos/odeke-em/drive"}
+46	{"id":141905736,"name":"GO-LiFE/GoFIT_SDK_Android","url":"https://api.github.com/repos/GO-LiFE/GoFIT_SDK_Android"}
+56	{"id":289417971,"name":"MrXujiang/h5-Dooring","url":"https://api.github.com/repos/MrXujiang/h5-Dooring"}
+86	{"id":106453399,"name":"redsaph/cleartext","url":"https://api.github.com/repos/redsaph/cleartext"}
+98	{"id":162998479,"name":"sherlock-project/sherlock","url":"https://api.github.com/repos/sherlock-project/sherlock"}
+101	{"id":326215605,"name":"okandavut/react-spotify-nowplaying","url":"https://api.github.com/repos/okandavut/react-spotify-nowplaying"}
+112	{"id":178435468,"name":"sentriz/gonic","url":"https://api.github.com/repos/sentriz/gonic"}
+122	{"id":182606378,"name":"netlify-labs/react-netlify-identity-widget","url":"https://api.github.com/repos/netlify-labs/react-netlify-identity-widget"}
+169	{"id":68730444,"name":"microsoft/BotBuilder-Samples","url":"https://api.github.com/repos/microsoft/BotBuilder-Samples"}
 
 -- !gh_data_13 --
 2051941	1
diff --git a/regression-test/data/variant_p0/with_index/load.out b/regression-test/data/variant_p0/with_index/load.out
index 61ffbfbe2e01af..9d7795e993d9a6 100644
--- a/regression-test/data/variant_p0/with_index/load.out
+++ b/regression-test/data/variant_p0/with_index/load.out
@@ -13,16 +13,16 @@
 0
 
 -- !sql_inv5 --
-1	{"b1":3,"a1":0}	hello world
+1	{"a1":0,"b1":3}	hello world
 2	{"a2":123}	world
 3	{"a3":123}	hello world
-4	{"b2":3,"b1":0}	hello world
+4	{"b1":0,"b2":3}	hello world
 5	{"b2":123}	world
 6	{"b3":123}	hello world
 
 -- !sql_inv6 --
 9	{"a3":123}	hello world
 8	{"a2":123}	world
-7	{"b1":3,"a1":0}	hello world
+7	{"a1":0,"b1":3}	hello world
 6	{"b3":123}	hello world
 
diff --git a/regression-test/data/variant_p0/with_index/var_index.out b/regression-test/data/variant_p0/with_index/var_index.out
index a1fa5c0444d961..d8b7417852b781 100644
--- a/regression-test/data/variant_p0/with_index/var_index.out
+++ b/regression-test/data/variant_p0/with_index/var_index.out
@@ -1,10 +1,10 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !sql --
-2	{"c":1181111,"a":18811,"b":"hello world"}
-3	{"c":11111,"a":18811,"b":"hello wworld"}
-4	{"c":8181111,"a":1234,"b":"hello xxx world"}
+2	{"a":18811,"b":"hello world","c":1181111}
+3	{"a":18811,"b":"hello wworld","c":11111}
+4	{"a":1234,"b":"hello xxx world","c":8181111}
 
 -- !sql --
-2	{"c":1181111,"a":18811,"b":"hello world"}
-4	{"c":8181111,"a":1234,"b":"hello xxx world"}
+2	{"a":18811,"b":"hello world","c":1181111}
+4	{"a":1234,"b":"hello xxx world","c":8181111}
 
diff --git a/regression-test/suites/variant_p0/desc.groovy b/regression-test/suites/variant_p0/desc.groovy
index 281893dd428e09..f97d4e8e8989bb 100644
--- a/regression-test/suites/variant_p0/desc.groovy
+++ b/regression-test/suites/variant_p0/desc.groovy
@@ -45,7 +45,7 @@ suite("regression_test_variant_desc", "nonConcurrent"){
         }
     }
 
-    def create_table = { table_name, buckets="auto" ->
+    def create_table = { table_name, buckets="1" ->
         sql "DROP TABLE IF EXISTS ${table_name}"
         sql """
             CREATE TABLE IF NOT EXISTS ${table_name} (
@@ -58,7 +58,7 @@ suite("regression_test_variant_desc", "nonConcurrent"){
         """
     }
 
-    def create_table_partition = { table_name, buckets="auto" ->
+    def create_table_partition = { table_name, buckets="1" ->
         sql "DROP TABLE IF EXISTS ${table_name}"
         sql """
             CREATE TABLE IF NOT EXISTS ${table_name} (

From d54cde51ba491d7c941c45a6cddfe107b018494f Mon Sep 17 00:00:00 2001
From: meiyi <myimeiyi@gmail.com>
Date: Tue, 2 Jan 2024 21:58:00 +0800
Subject: [PATCH 258/299] [fix](group commit) fix http stream group commit
 begin a txn (#29399)

---
 .../doris/analysis/NativeInsertStmt.java      | 36 +++++++++++--------
 ..._http_stream_lineitem_schema_change.groovy |  2 +-
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
index da82c4406c323d..576ebba9b0e4e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
@@ -382,7 +382,7 @@ public void analyze(Analyzer analyzer) throws UserException {
             label = new LabelName(db.getFullName(),
                     insertType.labePrefix + DebugUtil.printId(analyzer.getContext().queryId()).replace("-", "_"));
         }
-        if (!isExplain() && !isTransactionBegin) {
+        if (!isExplain() && !isTransactionBegin && !isGroupCommitStreamLoadSql) {
             if (targetTable instanceof OlapTable) {
                 LoadJobSourceType sourceType = LoadJobSourceType.INSERT_STREAMING;
                 transactionId = Env.getCurrentGlobalTransactionMgr().beginTransaction(db.getId(),
@@ -1020,15 +1020,17 @@ public void complete() throws UserException {
             if (!allowAutoPartition) {
                 ((OlapTableSink) dataSink).setAutoPartition(false);
             }
-            // add table indexes to transaction state
-            TransactionState txnState = Env.getCurrentGlobalTransactionMgr()
-                    .getTransactionState(db.getId(), transactionId);
-            if (txnState == null) {
-                throw new DdlException("txn does not exist: " + transactionId);
-            }
-            txnState.addTableIndexes((OlapTable) targetTable);
-            if (isPartialUpdate) {
-                txnState.setSchemaForPartialUpdate((OlapTable) targetTable);
+            if (!isGroupCommitStreamLoadSql) {
+                // add table indexes to transaction state
+                TransactionState txnState = Env.getCurrentGlobalTransactionMgr()
+                        .getTransactionState(db.getId(), transactionId);
+                if (txnState == null) {
+                    throw new DdlException("txn does not exist: " + transactionId);
+                }
+                txnState.addTableIndexes((OlapTable) targetTable);
+                if (isPartialUpdate) {
+                    txnState.setSchemaForPartialUpdate((OlapTable) targetTable);
+                }
             }
         }
     }
@@ -1110,11 +1112,17 @@ public RedirectStatus getRedirectStatus() {
     }
 
     public void analyzeGroupCommit(Analyzer analyzer) throws AnalysisException {
-        if (isGroupCommitStreamLoadSql && targetTable != null && (targetTable instanceof OlapTable)
-                && !((OlapTable) targetTable).getTableProperty().getUseSchemaLightChange()) {
-            throw new AnalysisException(
-                    "table light_schema_change is false, can't do http_stream with group commit mode");
+        // check if http stream meets group commit requirements.
+        // If not meets, throw exception (consider fallback to non group commit mode).
+        if (isGroupCommitStreamLoadSql) {
+            if (targetTable != null && (targetTable instanceof OlapTable)
+                    && !((OlapTable) targetTable).getTableProperty().getUseSchemaLightChange()) {
+                throw new AnalysisException(
+                        "table light_schema_change is false, can't do http_stream with group commit mode");
+            }
+            return;
         }
+        // check if 'insert into' meets group commit requirements. If meets, set isGroupCommit to true
         if (isGroupCommit) {
             return;
         }
diff --git a/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy b/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy
index dd0d40e645ca50..7d43f0c91e74e4 100644
--- a/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy
+++ b/regression-test/suites/insert_p2/test_group_commit_http_stream_lineitem_schema_change.groovy
@@ -368,7 +368,7 @@ l_comment) select c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c14, c15, c
 
 
     def process = { table_name ->
-        for (int i = 1; i <= 4; i++) {
+        for (int i = 1; i <= 5; i++) {
             switch (i) {
                 case SC.TRUNCATE_TABLE.value:
                     truncate(table_name)

From 69524ccf98f57877440d8e09d878fb76f3e011ad Mon Sep 17 00:00:00 2001
From: meiyi <myimeiyi@gmail.com>
Date: Tue, 2 Jan 2024 23:20:23 +0800
Subject: [PATCH 259/299] [improve](group commit) Group commit support commit
 by data size (#29428)

---
 be/src/runtime/group_commit_mgr.cpp           | 41 +++++++-----
 be/src/runtime/group_commit_mgr.h             | 64 +++++++++++--------
 .../doris/analysis/NativeInsertStmt.java      |  2 -
 .../doris/service/FrontendServiceImpl.java    |  2 +
 gensrc/thrift/FrontendService.thrift          |  1 +
 5 files changed, 66 insertions(+), 44 deletions(-)

diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index 0c3589edbe0379..2971138d5b6d53 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -36,11 +36,11 @@ Status LoadBlockQueue::add_block(RuntimeState* runtime_state,
     while (!runtime_state->is_cancelled() && status.ok() &&
            _all_block_queues_bytes->load(std::memory_order_relaxed) >
                    config::group_commit_queue_mem_limit) {
-        _put_cond.wait_for(
-                l, std::chrono::milliseconds(LoadBlockQueue::MAX_BLOCK_QUEUE_ADD_WAIT_TIME));
+        _put_cond.wait_for(l,
+                           std::chrono::milliseconds(LoadBlockQueue::MEM_BACK_PRESSURE_WAIT_TIME));
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
                 std::chrono::steady_clock::now() - start);
-        if (duration.count() > LoadBlockQueue::WAL_MEM_BACK_PRESSURE_TIME_OUT) {
+        if (duration.count() > LoadBlockQueue::MEM_BACK_PRESSURE_WAIT_TIMEOUT) {
             return Status::TimedOut(
                     "Wal memory back pressure wait too much time! Load block queue txn id: {}, "
                     "label: {}, instance id: {}",
@@ -60,8 +60,14 @@ Status LoadBlockQueue::add_block(RuntimeState* runtime_state,
                 return st;
             }
         }
+        _data_bytes += block->bytes();
         _all_block_queues_bytes->fetch_add(block->bytes(), std::memory_order_relaxed);
     }
+    if (_data_bytes >= _group_commit_data_bytes) {
+        VLOG_DEBUG << "group commit meets commit condition for data size, label=" << label
+                   << ", instance_id=" << load_instance_id << ", data_bytes=" << _data_bytes;
+        _need_commit = true;
+    }
     _get_cond.notify_all();
     return Status::OK();
 }
@@ -71,25 +77,25 @@ Status LoadBlockQueue::get_block(RuntimeState* runtime_state, vectorized::Block*
     *find_block = false;
     *eos = false;
     std::unique_lock l(mutex);
-    if (!need_commit) {
+    if (!_need_commit) {
         auto left_milliseconds =
                 _group_commit_interval_ms - std::chrono::duration_cast<std::chrono::milliseconds>(
                                                     std::chrono::steady_clock::now() - _start_time)
                                                     .count();
         if (left_milliseconds <= 0) {
-            need_commit = true;
+            _need_commit = true;
         }
     }
     while (!runtime_state->is_cancelled() && status.ok() && _block_queue.empty() &&
-           (!need_commit || (need_commit && !_load_ids.empty()))) {
+           (!_need_commit || (_need_commit && !_load_ids.empty()))) {
         auto left_milliseconds = _group_commit_interval_ms;
         auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(
                                 std::chrono::steady_clock::now() - _start_time)
                                 .count();
-        if (!need_commit) {
+        if (!_need_commit) {
             left_milliseconds = _group_commit_interval_ms - duration;
             if (left_milliseconds <= 0) {
-                need_commit = true;
+                _need_commit = true;
                 break;
             }
         } else {
@@ -120,7 +126,7 @@ Status LoadBlockQueue::get_block(RuntimeState* runtime_state, vectorized::Block*
         _block_queue.pop_front();
         _all_block_queues_bytes->fetch_sub(block->bytes(), std::memory_order_relaxed);
     }
-    if (_block_queue.empty() && need_commit && _load_ids.empty()) {
+    if (_block_queue.empty() && _need_commit && _load_ids.empty()) {
         *eos = true;
     } else {
         *eos = false;
@@ -139,7 +145,7 @@ void LoadBlockQueue::remove_load_id(const UniqueId& load_id) {
 
 Status LoadBlockQueue::add_load_id(const UniqueId& load_id) {
     std::unique_lock l(mutex);
-    if (need_commit) {
+    if (_need_commit) {
         return Status::InternalError("block queue is set need commit, id=" +
                                      load_instance_id.to_string());
     }
@@ -175,7 +181,7 @@ Status GroupCommitTable::get_first_block_load_queue(
         for (int i = 0; i < 3; i++) {
             bool is_schema_version_match = true;
             for (auto it = _load_block_queues.begin(); it != _load_block_queues.end(); ++it) {
-                if (!it->second->need_commit) {
+                if (!it->second->need_commit()) {
                     if (base_schema_version == it->second->schema_version) {
                         if (it->second->add_load_id(load_id).ok()) {
                             load_block_queue = it->second;
@@ -282,7 +288,8 @@ Status GroupCommitTable::_create_group_commit_load(
     {
         load_block_queue = std::make_shared<LoadBlockQueue>(
                 instance_id, label, txn_id, schema_version, _all_block_queues_bytes,
-                result.wait_internal_group_commit_finish, result.group_commit_interval_ms);
+                result.wait_internal_group_commit_finish, result.group_commit_interval_ms,
+                result.group_commit_data_bytes);
         std::unique_lock l(_lock);
         _load_block_queues.emplace(instance_id, load_block_queue);
         _need_plan_fragment = false;
@@ -377,7 +384,7 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t db_id, int64_t table_
     if (status.ok() && st.ok() &&
         (result_status.ok() || result_status.is<ErrorCode::PUBLISH_TIMEOUT>())) {
         RETURN_IF_ERROR(_exec_env->wal_mgr()->delete_wal(
-                txn_id, load_block_queue->block_queue_pre_allocated.load()));
+                txn_id, load_block_queue->block_queue_pre_allocated()));
         RETURN_IF_ERROR(_exec_env->wal_mgr()->erase_wal_status_queue(table_id, txn_id));
     } else {
         std::string wal_path;
@@ -485,7 +492,7 @@ Status LoadBlockQueue::create_wal(int64_t db_id, int64_t tb_id, int64_t wal_id,
                                   const std::string& import_label, WalManager* wal_manager,
                                   std::vector<TSlotDescriptor>& slot_desc, int be_exe_version) {
     RETURN_IF_ERROR(ExecEnv::GetInstance()->wal_mgr()->add_wal_path(db_id, tb_id, wal_id,
-                                                                    import_label, wal_base_path));
+                                                                    import_label, _wal_base_path));
     _v_wal_writer = std::make_shared<vectorized::VWalWriter>(
             tb_id, wal_id, import_label, wal_manager, slot_desc, be_exe_version);
     return _v_wal_writer->init();
@@ -502,17 +509,17 @@ bool LoadBlockQueue::has_enough_wal_disk_space(size_t pre_allocated) {
     auto* wal_mgr = ExecEnv::GetInstance()->wal_mgr();
     size_t available_bytes = 0;
     {
-        Status st = wal_mgr->get_wal_dir_available_size(wal_base_path, &available_bytes);
+        Status st = wal_mgr->get_wal_dir_available_size(_wal_base_path, &available_bytes);
         if (!st.ok()) {
             LOG(WARNING) << "get wal disk available size filed!";
         }
     }
     if (pre_allocated < available_bytes) {
-        Status st = wal_mgr->update_wal_dir_pre_allocated(wal_base_path, pre_allocated, true);
+        Status st = wal_mgr->update_wal_dir_pre_allocated(_wal_base_path, pre_allocated, true);
         if (!st.ok()) {
             LOG(WARNING) << "update wal dir pre_allocated failed, reason: " << st.to_string();
         }
-        block_queue_pre_allocated.fetch_add(pre_allocated);
+        _block_queue_pre_allocated.fetch_add(pre_allocated);
         return true;
     } else {
         return false;
diff --git a/be/src/runtime/group_commit_mgr.h b/be/src/runtime/group_commit_mgr.h
index 125256535fec30..49bdd5f87a70c7 100644
--- a/be/src/runtime/group_commit_mgr.h
+++ b/be/src/runtime/group_commit_mgr.h
@@ -45,15 +45,17 @@ class LoadBlockQueue {
     LoadBlockQueue(const UniqueId& load_instance_id, std::string& label, int64_t txn_id,
                    int64_t schema_version,
                    std::shared_ptr<std::atomic_size_t> all_block_queues_bytes,
-                   bool wait_internal_group_commit_finish, int64_t group_commit_interval_ms)
+                   bool wait_internal_group_commit_finish, int64_t group_commit_interval_ms,
+                   int64_t group_commit_data_bytes)
             : load_instance_id(load_instance_id),
               label(label),
               txn_id(txn_id),
               schema_version(schema_version),
               wait_internal_group_commit_finish(wait_internal_group_commit_finish),
+              _group_commit_interval_ms(group_commit_interval_ms),
               _start_time(std::chrono::steady_clock::now()),
-              _all_block_queues_bytes(all_block_queues_bytes),
-              _group_commit_interval_ms(group_commit_interval_ms) {};
+              _group_commit_data_bytes(group_commit_data_bytes),
+              _all_block_queues_bytes(all_block_queues_bytes) {};
 
     Status add_block(RuntimeState* runtime_state, std::shared_ptr<vectorized::Block> block,
                      bool write_wal);
@@ -62,44 +64,54 @@ class LoadBlockQueue {
     Status add_load_id(const UniqueId& load_id);
     void remove_load_id(const UniqueId& load_id);
     void cancel(const Status& st);
+    bool need_commit() { return _need_commit; }
+
     Status create_wal(int64_t db_id, int64_t tb_id, int64_t wal_id, const std::string& import_label,
                       WalManager* wal_manager, std::vector<TSlotDescriptor>& slot_desc,
                       int be_exe_version);
     Status close_wal();
     bool has_enough_wal_disk_space(size_t pre_allocated);
+    size_t block_queue_pre_allocated() { return _block_queue_pre_allocated.load(); }
 
-    // 1s
-    static constexpr size_t MAX_BLOCK_QUEUE_ADD_WAIT_TIME = 1000;
-    // 120s
-    static constexpr size_t WAL_MEM_BACK_PRESSURE_TIME_OUT = 120000;
     UniqueId load_instance_id;
     std::string label;
     int64_t txn_id;
     int64_t schema_version;
-    bool need_commit = false;
     bool wait_internal_group_commit_finish = false;
+
+    // the execute status of this internal group commit
     std::mutex mutex;
-    bool process_finish = false;
     std::condition_variable internal_group_commit_finish_cv;
+    bool process_finish = false;
     Status status = Status::OK();
-    std::string wal_base_path;
-    std::atomic_size_t block_queue_pre_allocated = 0;
 
 private:
     void _cancel_without_lock(const Status& st);
-    std::chrono::steady_clock::time_point _start_time;
 
-    std::condition_variable _put_cond;
-    std::condition_variable _get_cond;
     // the set of load ids of all blocks in this queue
     std::set<UniqueId> _load_ids;
     std::list<std::shared_ptr<vectorized::Block>> _block_queue;
 
-    // memory consumption of all tables' load block queues, used for back pressure.
-    std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
-    // group commit interval in ms, can be changed by 'ALTER TABLE my_table SET ("group_commit_interval_ms"="1000");'
-    int64_t _group_commit_interval_ms;
+    // wal
+    std::string _wal_base_path;
     std::shared_ptr<vectorized::VWalWriter> _v_wal_writer;
+    std::atomic_size_t _block_queue_pre_allocated = 0;
+
+    // commit
+    bool _need_commit = false;
+    // commit by time interval, can be changed by 'ALTER TABLE my_table SET ("group_commit_interval_ms"="1000");'
+    int64_t _group_commit_interval_ms;
+    std::chrono::steady_clock::time_point _start_time;
+    // commit by data size
+    int64_t _group_commit_data_bytes;
+    int64_t _data_bytes = 0;
+
+    // memory back pressure, memory consumption of all tables' load block queues
+    std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
+    std::condition_variable _put_cond;
+    std::condition_variable _get_cond;
+    static constexpr size_t MEM_BACK_PRESSURE_WAIT_TIME = 1000;      // 1s
+    static constexpr size_t MEM_BACK_PRESSURE_WAIT_TIMEOUT = 120000; // 120s
 };
 
 class GroupCommitTable {
@@ -108,9 +120,9 @@ class GroupCommitTable {
                      int64_t table_id, std::shared_ptr<std::atomic_size_t> all_block_queue_bytes)
             : _exec_env(exec_env),
               _thread_pool(thread_pool),
+              _all_block_queues_bytes(all_block_queue_bytes),
               _db_id(db_id),
-              _table_id(table_id),
-              _all_block_queues_bytes(all_block_queue_bytes) {};
+              _table_id(table_id) {};
     Status get_first_block_load_queue(int64_t table_id, int64_t base_schema_version,
                                       const UniqueId& load_id,
                                       std::shared_ptr<LoadBlockQueue>& load_block_queue,
@@ -131,15 +143,17 @@ class GroupCommitTable {
 
     ExecEnv* _exec_env = nullptr;
     ThreadPool* _thread_pool = nullptr;
+    // memory consumption of all tables' load block queues, used for memory back pressure.
+    std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
+
     int64_t _db_id;
     int64_t _table_id;
+
     std::mutex _lock;
     std::condition_variable _cv;
     // fragment_instance_id to load_block_queue
     std::unordered_map<UniqueId, std::shared_ptr<LoadBlockQueue>> _load_block_queues;
     bool _need_plan_fragment = false;
-    // memory consumption of all tables' load block queues, used for back pressure.
-    std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
 };
 
 class GroupCommitMgr {
@@ -159,13 +173,13 @@ class GroupCommitMgr {
 
 private:
     ExecEnv* _exec_env = nullptr;
+    std::unique_ptr<doris::ThreadPool> _thread_pool;
+    // memory consumption of all tables' load block queues, used for memory back pressure.
+    std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
 
     std::mutex _lock;
     // TODO remove table when unused
     std::unordered_map<int64_t, std::shared_ptr<GroupCommitTable>> _table_map;
-    std::unique_ptr<doris::ThreadPool> _thread_pool;
-    // memory consumption of all tables' load block queues, used for back pressure.
-    std::shared_ptr<std::atomic_size_t> _all_block_queues_bytes;
 };
 
 } // namespace doris
\ No newline at end of file
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
index 576ebba9b0e4e5..176833b865c402 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java
@@ -69,7 +69,6 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import com.google.protobuf.ByteString;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -150,7 +149,6 @@ public class NativeInsertStmt extends InsertStmt {
     private boolean isGroupCommit = false;
     private int baseSchemaVersion = -1;
     private TUniqueId loadId = null;
-    private ByteString execPlanFragmentParamsBytes = null;
     private long tableId = -1;
     public boolean isGroupCommitStreamLoadSql = false;
     private GroupCommitPlanner groupCommitPlanner;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index 136084f5f1c012..cf42cb921f6ef2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -1983,6 +1983,8 @@ private void httpStreamPutImpl(TStreamLoadPutRequest request, TStreamLoadPutResu
             result.setTableId(parsedStmt.getTargetTable().getId());
             result.setBaseSchemaVersion(((OlapTable) parsedStmt.getTargetTable()).getBaseSchemaVersion());
             result.setGroupCommitIntervalMs(((OlapTable) parsedStmt.getTargetTable()).getGroupCommitIntervalMs());
+            // TODO get from table property
+            result.setGroupCommitDataBytes(134217728L);
             result.setWaitInternalGroupCommitFinish(Config.wait_internal_group_commit_finish);
         } catch (UserException e) {
             LOG.warn("exec sql error", e);
diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift
index 209947c8b78778..75e67722113b1e 100644
--- a/gensrc/thrift/FrontendService.thrift
+++ b/gensrc/thrift/FrontendService.thrift
@@ -670,6 +670,7 @@ struct TStreamLoadPutResult {
     6: optional i64 table_id
     7: optional bool wait_internal_group_commit_finish = false
     8: optional i64 group_commit_interval_ms
+    9: optional i64 group_commit_data_bytes
 }
 
 struct TStreamLoadMultiTablePutResult {

From 69a01e0cf5c704c07b78a87ea315effacfceb08b Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Tue, 2 Jan 2024 23:35:50 +0800
Subject: [PATCH 260/299] [improve](move-memtable) skip load stream stub close
 wait when cancel (#29427)

---
 be/src/vec/sink/load_stream_stub.cpp          |  1 -
 be/src/vec/sink/load_stream_stub.h            |  5 ++
 be/src/vec/sink/load_stream_stub_pool.cpp     | 16 ++--
 be/src/vec/sink/load_stream_stub_pool.h       |  2 +-
 be/src/vec/sink/writer/vtablet_writer_v2.cpp  |  4 +-
 .../vec/exec/load_stream_stub_pool_test.cpp   |  7 +-
 ...eam_stub_close_wait_fault_injection.groovy | 88 +++++++++++++++++++
 7 files changed, 109 insertions(+), 14 deletions(-)
 create mode 100644 regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy

diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 6863faa4ac3416..9ad8d8805ca852 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -21,7 +21,6 @@
 
 #include "olap/rowset/rowset_writer.h"
 #include "util/brpc_client_cache.h"
-#include "util/debug_points.h"
 #include "util/network_util.h"
 #include "util/thrift_util.h"
 #include "util/uid_util.h"
diff --git a/be/src/vec/sink/load_stream_stub.h b/be/src/vec/sink/load_stream_stub.h
index 5c9b3d2f31dff5..a5c9d7464a4190 100644
--- a/be/src/vec/sink/load_stream_stub.h
+++ b/be/src/vec/sink/load_stream_stub.h
@@ -60,6 +60,7 @@
 #include "runtime/thread_context.h"
 #include "runtime/types.h"
 #include "util/countdown_latch.h"
+#include "util/debug_points.h"
 #include "util/runtime_profile.h"
 #include "util/stopwatch.hpp"
 #include "vec/columns/column.h"
@@ -183,6 +184,10 @@ class LoadStreamStub {
     // wait remote to close stream,
     // remote will close stream when it receives CLOSE_LOAD
     Status close_wait(int64_t timeout_ms = 0) {
+        DBUG_EXECUTE_IF("LoadStreamStub::close_wait.long_wait", {
+            while (true) {
+            };
+        });
         if (!_is_init.load() || _handler.is_closed()) {
             return Status::OK();
         }
diff --git a/be/src/vec/sink/load_stream_stub_pool.cpp b/be/src/vec/sink/load_stream_stub_pool.cpp
index c867756d8fb2be..3cf168a131eff8 100644
--- a/be/src/vec/sink/load_stream_stub_pool.cpp
+++ b/be/src/vec/sink/load_stream_stub_pool.cpp
@@ -26,7 +26,7 @@ class TExpr;
 LoadStreams::LoadStreams(UniqueId load_id, int64_t dst_id, int num_use, LoadStreamStubPool* pool)
         : _load_id(load_id), _dst_id(dst_id), _use_cnt(num_use), _pool(pool) {}
 
-void LoadStreams::release() {
+void LoadStreams::release(Status status) {
     int num_use = --_use_cnt;
     DBUG_EXECUTE_IF("LoadStreams.release.keeping_streams", { num_use = 1; });
     if (num_use == 0) {
@@ -39,12 +39,14 @@ void LoadStreams::release() {
                 LOG(WARNING) << "close stream failed " << st;
             }
         }
-        for (auto& stream : _streams) {
-            auto st = stream->close_wait();
-            DBUG_EXECUTE_IF("LoadStreams.release.close_wait_failed",
-                            { st = Status::InternalError("stream close wait timeout"); });
-            if (!st.ok()) {
-                LOG(WARNING) << "close wait failed " << st;
+        if (status.ok()) {
+            for (auto& stream : _streams) {
+                auto st = stream->close_wait();
+                DBUG_EXECUTE_IF("LoadStreams.release.close_wait_failed",
+                                { st = Status::InternalError("stream close wait timeout"); });
+                if (!st.ok()) {
+                    LOG(WARNING) << "close wait failed " << st;
+                }
             }
         }
         _pool->erase(_load_id, _dst_id);
diff --git a/be/src/vec/sink/load_stream_stub_pool.h b/be/src/vec/sink/load_stream_stub_pool.h
index e41083825be07e..b34383b25f9e2d 100644
--- a/be/src/vec/sink/load_stream_stub_pool.h
+++ b/be/src/vec/sink/load_stream_stub_pool.h
@@ -76,7 +76,7 @@ class LoadStreams {
 public:
     LoadStreams(UniqueId load_id, int64_t dst_id, int num_use, LoadStreamStubPool* pool);
 
-    void release();
+    void release(Status status);
 
     Streams& streams() { return _streams; }
 
diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
index d1e65c804d7bb9..e02fba7c21c3f1 100644
--- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp
@@ -456,7 +456,7 @@ Status VTabletWriterV2::_cancel(Status status) {
         _delta_writer_for_tablet.reset();
     }
     for (const auto& [_, streams] : _streams_for_node) {
-        streams->release();
+        streams->release(status);
     }
     return Status::OK();
 }
@@ -513,7 +513,7 @@ Status VTabletWriterV2::close(Status exec_status) {
         // defer stream release to prevent memory leak
         Defer defer([&] {
             for (const auto& [_, streams] : _streams_for_node) {
-                streams->release();
+                streams->release(status);
             }
             _streams_for_node.clear();
         });
diff --git a/be/test/vec/exec/load_stream_stub_pool_test.cpp b/be/test/vec/exec/load_stream_stub_pool_test.cpp
index 24da3bb6999f36..bea5443b4ff7fd 100644
--- a/be/test/vec/exec/load_stream_stub_pool_test.cpp
+++ b/be/test/vec/exec/load_stream_stub_pool_test.cpp
@@ -32,6 +32,7 @@ TEST_F(LoadStreamStubPoolTest, test) {
     LoadStreamStubPool pool;
     int64_t src_id = 100;
     PUniqueId load_id;
+    Status st = Status::OK();
     load_id.set_hi(1);
     load_id.set_hi(2);
     auto streams1 = pool.get_or_create(load_id, src_id, 101, 5, 1);
@@ -41,9 +42,9 @@ TEST_F(LoadStreamStubPoolTest, test) {
     EXPECT_EQ(1, pool.templates_size());
     EXPECT_EQ(streams1, streams3);
     EXPECT_NE(streams1, streams2);
-    streams1->release();
-    streams2->release();
-    streams3->release();
+    streams1->release(st);
+    streams2->release(st);
+    streams3->release(st);
     EXPECT_EQ(0, pool.size());
     EXPECT_EQ(0, pool.templates_size());
 }
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
new file mode 100644
index 00000000000000..4a87f1daf6b2d9
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_stub_close_wait_fault_injection.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import org.apache.doris.regression.util.Http
+
+suite("test_load_stream_stub_close_wait_fault_injection", "nonConcurrent") {
+    sql """ set enable_memtable_on_sink_node=true """
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """
+        CREATE TABLE IF NOT EXISTS `baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+    sql """
+        CREATE TABLE IF NOT EXISTS `test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    GetDebugPoint().clearDebugPointsForAllBEs()
+    streamLoad {
+        table "baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    try {
+        GetDebugPoint().enableDebugPointForAllBEs("VTabletWriterV2.close.cancel")
+        GetDebugPoint().enableDebugPointForAllBEs("LoadStreamStub::close_wait.long_wait")
+        def res = sql "insert into test select * from baseall where k1 <= 3"
+        logger.info(res.toString())
+    } catch(Exception e) {
+        logger.info(e.getMessage())
+        assertTrue(e.getMessage().contains("cancel"))
+    } finally {
+        GetDebugPoint().disableDebugPointForAllBEs("VTabletWriterV2.close.cancel")
+        GetDebugPoint().disableDebugPointForAllBEs("LoadStreamStub::close_wait.long_wait")
+    }
+
+    sql """ DROP TABLE IF EXISTS `baseall` """
+    sql """ DROP TABLE IF EXISTS `test` """
+    sql """ set enable_memtable_on_sink_node=false """
+}
\ No newline at end of file

From 797238cbb71572e5d5b5a0bea778b2020dbe3e4a Mon Sep 17 00:00:00 2001
From: Xin Liao <liaoxinbit@126.com>
Date: Tue, 2 Jan 2024 23:45:04 +0800
Subject: [PATCH 261/299] [fix](merge-on-write) fix schema change may result in
 delete bitmap incorrect (#29386)

---
 be/src/olap/schema_change.cpp          | 137 +++++++++++++++----------
 be/src/olap/schema_change.h            |   6 +-
 be/src/olap/tablet.cpp                 |  22 ++--
 be/src/olap/tablet.h                   |   4 +
 be/src/olap/task/engine_clone_task.cpp |   9 +-
 5 files changed, 111 insertions(+), 67 deletions(-)

diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index d19d399b1efaaa..459b1931aea52e 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -62,6 +62,7 @@
 #include "olap/wrapper_field.h"
 #include "runtime/memory/mem_tracker.h"
 #include "runtime/runtime_state.h"
+#include "util/debug_points.h"
 #include "util/defer_op.h"
 #include "util/trace.h"
 #include "vec/aggregate_functions/aggregate_function.h"
@@ -711,6 +712,13 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
                   << " res=" << res;
         return res;
     }
+    new_tablet->set_alter_failed(false);
+    Defer defer([&new_tablet] {
+        // if tablet state is not TABLET_RUNNING when return, indicates that alter has failed.
+        if (new_tablet->tablet_state() != TABLET_RUNNING) {
+            new_tablet->set_alter_failed(true);
+        }
+    });
 
     LOG(INFO) << "finish to validate alter tablet request. begin to convert data from base tablet "
                  "to new tablet"
@@ -918,7 +926,8 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
             std::lock_guard<std::shared_mutex> wrlock(_mutex);
             _tablet_ids_in_converting.insert(new_tablet->tablet_id());
         }
-        res = _convert_historical_rowsets(sc_params);
+        int64_t real_alter_version = 0;
+        res = _convert_historical_rowsets(sc_params, &real_alter_version);
         {
             std::lock_guard<std::shared_mutex> wrlock(_mutex);
             _tablet_ids_in_converting.erase(new_tablet->tablet_id());
@@ -927,65 +936,12 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
             break;
         }
 
-        // For unique with merge-on-write table, should process delete bitmap here.
-        // 1. During double write, the newly imported rowsets does not calculate
-        // delete bitmap and publish successfully.
-        // 2. After conversion, calculate delete bitmap for the rowsets imported
-        // during double write. During this period, new data can still be imported
-        // witout calculating delete bitmap and publish successfully.
-        // 3. Block the new publish, calculate the delete bitmap of the
-        // incremental rowsets.
-        // 4. Switch the tablet status to TABLET_RUNNING. The newly imported
-        // data will calculate delete bitmap.
         if (new_tablet->keys_type() == UNIQUE_KEYS &&
             new_tablet->enable_unique_key_merge_on_write()) {
-            // step 2
-            int64_t max_version = new_tablet->max_version().second;
-            std::vector<RowsetSharedPtr> rowsets;
-            if (end_version < max_version) {
-                LOG(INFO)
-                        << "alter table for unique with merge-on-write, calculate delete bitmap of "
-                        << "double write rowsets for version: " << end_version + 1 << "-"
-                        << max_version;
-                RETURN_IF_ERROR(new_tablet->capture_consistent_rowsets(
-                        {end_version + 1, max_version}, &rowsets));
-            }
-            for (auto rowset_ptr : rowsets) {
-                if (rowset_ptr->version().second <= end_version) {
-                    continue;
-                }
-                std::lock_guard<std::mutex> rwlock(new_tablet->get_rowset_update_lock());
-                std::shared_lock<std::shared_mutex> wrlock(new_tablet->get_header_lock());
-                RETURN_IF_ERROR(new_tablet->update_delete_bitmap_without_lock(rowset_ptr));
-            }
-
-            // step 3
-            std::lock_guard<std::mutex> rwlock(new_tablet->get_rowset_update_lock());
-            std::lock_guard<std::shared_mutex> new_wlock(new_tablet->get_header_lock());
-            SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
-            int64_t new_max_version = new_tablet->max_version_unlocked().second;
-            rowsets.clear();
-            if (max_version < new_max_version) {
-                LOG(INFO)
-                        << "alter table for unique with merge-on-write, calculate delete bitmap of "
-                        << "incremental rowsets for version: " << max_version + 1 << "-"
-                        << new_max_version;
-                RETURN_IF_ERROR(new_tablet->capture_consistent_rowsets(
-                        {max_version + 1, new_max_version}, &rowsets));
-            }
-            for (auto rowset_ptr : rowsets) {
-                if (rowset_ptr->version().second <= max_version) {
-                    continue;
-                }
-                RETURN_IF_ERROR(new_tablet->update_delete_bitmap_without_lock(rowset_ptr));
-            }
-
-            // step 4
-            res = new_tablet->set_tablet_state(TabletState::TABLET_RUNNING);
+            res = _calc_delete_bitmap_for_mow_table(new_tablet, real_alter_version);
             if (!res) {
                 break;
             }
-            new_tablet->save_meta();
         } else {
             // set state to ready
             std::lock_guard<std::shared_mutex> new_wlock(new_tablet->get_header_lock());
@@ -1035,7 +991,10 @@ Status SchemaChangeHandler::_get_versions_to_be_changed(
     return Status::OK();
 }
 
-Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams& sc_params) {
+// The `real_alter_version` parameter indicates that the version of [0-real_alter_version] is
+// converted from a base tablet, only used for the mow table now.
+Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams& sc_params,
+                                                        int64_t* real_alter_version) {
     LOG(INFO) << "begin to convert historical rowsets for new_tablet from base_tablet."
               << " base_tablet=" << sc_params.base_tablet->tablet_id()
               << ", new_tablet=" << sc_params.new_tablet->tablet_id();
@@ -1146,7 +1105,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams
                          << "tablet=" << sc_params.new_tablet->tablet_id() << ", version='"
                          << rs_reader->version().first << "-" << rs_reader->version().second;
             StorageEngine::instance()->add_unused_rowset(new_rowset);
-            res = Status::OK();
+            return process_alter_exit();
         } else if (!res) {
             LOG(WARNING) << "failed to register new version. "
                          << " tablet=" << sc_params.new_tablet->tablet_id()
@@ -1159,6 +1118,7 @@ Status SchemaChangeHandler::_convert_historical_rowsets(const SchemaChangeParams
                         << ", version=" << rs_reader->version().first << "-"
                         << rs_reader->version().second;
         }
+        *real_alter_version = rs_reader->version().second;
 
         VLOG_TRACE << "succeed to convert a history version."
                    << " version=" << rs_reader->version().first << "-"
@@ -1377,4 +1337,67 @@ Status SchemaChangeHandler::_validate_alter_result(TabletSharedPtr new_tablet,
     return Status::OK();
 }
 
+// For unique with merge-on-write table, should process delete bitmap here.
+// 1. During double write, the newly imported rowsets does not calculate
+// delete bitmap and publish successfully.
+// 2. After conversion, calculate delete bitmap for the rowsets imported
+// during double write. During this period, new data can still be imported
+// witout calculating delete bitmap and publish successfully.
+// 3. Block the new publish, calculate the delete bitmap of the
+// incremental rowsets.
+// 4. Switch the tablet status to TABLET_RUNNING. The newly imported
+// data will calculate delete bitmap.
+Status SchemaChangeHandler::_calc_delete_bitmap_for_mow_table(TabletSharedPtr new_tablet,
+                                                              int64_t alter_version) {
+    DBUG_EXECUTE_IF("SchemaChangeHandler._calc_delete_bitmap_for_mow_table.random_failed", {
+        if (rand() % 100 < (100 * dp->param("percent", 0.1))) {
+            LOG_WARNING("SchemaChangeHandler._calc_delete_bitmap_for_mow_table.random_failed");
+            return Status::InternalError("debug schema change calc delete bitmap random failed");
+        }
+    });
+
+    // can't do compaction when calc delete bitmap, if the rowset being calculated does
+    // a compaction, it may cause the delete bitmap to be missed.
+    std::lock_guard base_compaction_lock(new_tablet->get_base_compaction_lock());
+    std::lock_guard cumu_compaction_lock(new_tablet->get_cumulative_compaction_lock());
+
+    // step 2
+    int64_t max_version = new_tablet->max_version().second;
+    std::vector<RowsetSharedPtr> rowsets;
+    if (alter_version < max_version) {
+        LOG(INFO) << "alter table for unique with merge-on-write, calculate delete bitmap of "
+                  << "double write rowsets for version: " << alter_version + 1 << "-" << max_version
+                  << " new_tablet=" << new_tablet->tablet_id();
+        std::shared_lock<std::shared_mutex> rlock(new_tablet->get_header_lock());
+        RETURN_IF_ERROR(
+                new_tablet->capture_consistent_rowsets({alter_version + 1, max_version}, &rowsets));
+    }
+    for (auto rowset_ptr : rowsets) {
+        std::lock_guard<std::mutex> rwlock(new_tablet->get_rowset_update_lock());
+        std::shared_lock<std::shared_mutex> rlock(new_tablet->get_header_lock());
+        RETURN_IF_ERROR(new_tablet->update_delete_bitmap_without_lock(rowset_ptr));
+    }
+
+    // step 3
+    std::lock_guard<std::mutex> rwlock(new_tablet->get_rowset_update_lock());
+    std::lock_guard<std::shared_mutex> new_wlock(new_tablet->get_header_lock());
+    SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
+    int64_t new_max_version = new_tablet->max_version_unlocked().second;
+    rowsets.clear();
+    if (max_version < new_max_version) {
+        LOG(INFO) << "alter table for unique with merge-on-write, calculate delete bitmap of "
+                  << "incremental rowsets for version: " << max_version + 1 << "-"
+                  << new_max_version << " new_tablet=" << new_tablet->tablet_id();
+        RETURN_IF_ERROR(new_tablet->capture_consistent_rowsets({max_version + 1, new_max_version},
+                                                               &rowsets));
+    }
+    for (auto rowset_ptr : rowsets) {
+        RETURN_IF_ERROR(new_tablet->update_delete_bitmap_without_lock(rowset_ptr));
+    }
+    // step 4
+    RETURN_IF_ERROR(new_tablet->set_tablet_state(TabletState::TABLET_RUNNING));
+    new_tablet->save_meta();
+    return Status::OK();
+}
+
 } // namespace doris
diff --git a/be/src/olap/schema_change.h b/be/src/olap/schema_change.h
index 87b1de3170518a..949c1d5514ca19 100644
--- a/be/src/olap/schema_change.h
+++ b/be/src/olap/schema_change.h
@@ -277,7 +277,8 @@ class SchemaChangeHandler {
     static Status _validate_alter_result(TabletSharedPtr new_tablet,
                                          const TAlterTabletReqV2& request);
 
-    static Status _convert_historical_rowsets(const SchemaChangeParams& sc_params);
+    static Status _convert_historical_rowsets(const SchemaChangeParams& sc_params,
+                                              int64_t* real_alter_version);
 
     static Status _parse_request(const SchemaChangeParams& sc_params, BlockChanger* changer,
                                  bool* sc_sorting, bool* sc_directly);
@@ -286,6 +287,9 @@ class SchemaChangeHandler {
     static Status _init_column_mapping(ColumnMapping* column_mapping,
                                        const TabletColumn& column_schema, const std::string& value);
 
+    static Status _calc_delete_bitmap_for_mow_table(TabletSharedPtr new_tablet,
+                                                    int64_t alter_version);
+
     static std::shared_mutex _mutex;
     static std::unordered_set<int64_t> _tablet_ids_in_converting;
     static std::set<std::string> _supported_functions;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 4f1c3229958e0b..922bec70278fe4 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1026,14 +1026,6 @@ bool Tablet::can_do_compaction(size_t path_hash, CompactionType compaction_type)
         return false;
     }
 
-    // unique key table with merge-on-write also cann't do cumulative compaction under alter
-    // process. It may cause the delete bitmap calculation error, such as two
-    // rowsets have same key.
-    if (tablet_state() != TABLET_RUNNING && keys_type() == UNIQUE_KEYS &&
-        enable_unique_key_merge_on_write()) {
-        return false;
-    }
-
     if (data_dir()->path_hash() != path_hash || !is_used() || !init_succeeded()) {
         return false;
     }
@@ -1762,6 +1754,13 @@ void Tablet::build_tablet_report_info(TTabletInfo* tablet_info,
         }
     }
 
+    // There are two cases when tablet state is TABLET_NOTREADY
+    // case 1: tablet is doing schema change. Fe knows it's state, doing nothing.
+    // case 2: tablet has finished schema change, but failed. Fe will perform recovery.
+    if (tablet_state() == TABLET_NOTREADY && is_alter_failed()) {
+        tablet_info->__set_used(false);
+    }
+
     if (tablet_state() == TABLET_SHUTDOWN) {
         tablet_info->__set_used(false);
     }
@@ -3287,6 +3286,13 @@ void Tablet::_rowset_ids_difference(const RowsetIdUnorderedSet& cur,
 
 // The caller should hold _rowset_update_lock and _meta_lock lock.
 Status Tablet::update_delete_bitmap_without_lock(const RowsetSharedPtr& rowset) {
+    DBUG_EXECUTE_IF("Tablet.update_delete_bitmap_without_lock.random_failed", {
+        if (rand() % 100 < (100 * dp->param("percent", 0.1))) {
+            LOG_WARNING("Tablet.update_delete_bitmap_without_lock.random_failed");
+            return Status::InternalError(
+                    "debug tablet update delete bitmap without lock random failed");
+        }
+    });
     int64_t cur_version = rowset->end_version();
     std::vector<segment_v2::SegmentSharedPtr> segments;
     RETURN_IF_ERROR(_load_rowset_segments(rowset, &segments));
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 03737ea241bf81..d55662d75df6b9 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -557,6 +557,8 @@ class Tablet final : public BaseTablet {
             SegmentCacheHandle* segment_cache_handle,
             std::unique_ptr<segment_v2::ColumnIterator>* column_iterator,
             OlapReaderStatistics* stats);
+    void set_alter_failed(bool alter_failed) { _alter_failed = alter_failed; }
+    bool is_alter_failed() { return _alter_failed; }
 
 private:
     Status _init_once_action();
@@ -685,6 +687,8 @@ class Tablet final : public BaseTablet {
     // may delete compaction input rowsets.
     std::mutex _cold_compaction_lock;
     int64_t _last_failed_follow_cooldown_time = 0;
+    // `_alter_failed` is used to indicate whether the tablet failed to perform a schema change
+    std::atomic<bool> _alter_failed = false;
 
     DISALLOW_COPY_AND_ASSIGN(Tablet);
 
diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp
index e27db6188bd2be..dc851928577eb8 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -171,12 +171,19 @@ Status EngineCloneTask::_do_clone() {
             StorageEngine::instance()->tablet_manager()->get_tablet(_clone_req.tablet_id);
 
     // The status of a tablet is not ready, indicating that it is a residual tablet after a schema
-    // change failure. It should not provide normal read and write, so drop it here.
+    // change failure. Clone a new tablet from remote be to overwrite it. This situation basically only
+    // occurs when the be_rebalancer_fuzzy_test configuration is enabled.
     if (tablet && tablet->tablet_state() == TABLET_NOTREADY) {
         LOG(WARNING) << "tablet state is not ready when clone, need to drop old tablet, tablet_id="
                      << tablet->tablet_id();
+        // can not drop tablet when under clone. so unregister clone tablet firstly.
+        StorageEngine::instance()->tablet_manager()->unregister_clone_tablet(_clone_req.tablet_id);
         RETURN_IF_ERROR(StorageEngine::instance()->tablet_manager()->drop_tablet(
                 tablet->tablet_id(), tablet->replica_id(), false));
+        if (!StorageEngine::instance()->tablet_manager()->register_clone_tablet(
+                    _clone_req.tablet_id)) {
+            return Status::InternalError("tablet {} is under clone", _clone_req.tablet_id);
+        }
         tablet.reset();
     }
     bool is_new_tablet = tablet == nullptr;

From 1edf5b31b61a903957e5d81eab021ba062c958ca Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Wed, 3 Jan 2024 00:06:52 +0800
Subject: [PATCH 262/299] [Regression-test](wal) Add fault injection case for
 wal mem back pressure (#29298)

---
 be/src/common/config.cpp                      |   2 +-
 be/src/common/config.h                        |   2 +-
 ...l_mem_back_pressure_fault_injection.csv.gz | Bin 0 -> 372017 bytes
 ...l_mem_back_pressure_fault_injection.groovy | 142 ++++++++++++++++++
 4 files changed, 144 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/data/fault_injection_p0/test_wal_mem_back_pressure_fault_injection.csv.gz
 create mode 100644 regression-test/suites/fault_injection_p0/test_wal_mem_back_pressure_fault_injection.groovy

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 9b30ae145c46bd..ddb235130bc723 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1121,7 +1121,7 @@ DEFINE_Int32(group_commit_insert_threads, "10");
 DEFINE_Int32(group_commit_memory_rows_for_max_filter_ratio, "10000");
 DEFINE_Bool(wait_internal_group_commit_finish, "false");
 // Max size(bytes) of group commit queues, used for mem back pressure, defult 64M.
-DEFINE_Int32(group_commit_queue_mem_limit, "67108864");
+DEFINE_mInt32(group_commit_queue_mem_limit, "67108864");
 // Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
 // group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
 DEFINE_String(group_commit_wal_max_disk_limit, "10%");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index c340abe05f3f58..cfb8ed02f328b3 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1187,7 +1187,7 @@ DECLARE_mInt32(group_commit_insert_threads);
 DECLARE_mInt32(group_commit_memory_rows_for_max_filter_ratio);
 DECLARE_Bool(wait_internal_group_commit_finish);
 // Max size(bytes) of group commit queues, used for mem back pressure.
-DECLARE_Int32(group_commit_queue_mem_limit);
+DECLARE_mInt32(group_commit_queue_mem_limit);
 // Max size(bytes) or percentage(%) of wal disk usage, used for disk space back pressure, default 10% of the disk available space.
 // group_commit_wal_max_disk_limit=1024 or group_commit_wal_max_disk_limit=10% can be automatically identified.
 DECLARE_mString(group_commit_wal_max_disk_limit);
diff --git a/regression-test/data/fault_injection_p0/test_wal_mem_back_pressure_fault_injection.csv.gz b/regression-test/data/fault_injection_p0/test_wal_mem_back_pressure_fault_injection.csv.gz
new file mode 100644
index 0000000000000000000000000000000000000000..539fb01b418a20d2b92a9428b5a41fedcd51f07e
GIT binary patch
literal 372017
zcmeI*e~i?1e#ddawcVbYuolrHEz8m-Tx@#8Hce@%vZc}`t+q`~dI^%_qFzfw(<@o6
zG$P1KV5ml$v`H_)#+3eH(!{Qb{R5Ld_n-tWsqUS*UhopJ$70U3Y55`E;jwVkac20w
zKl7RIKq`x~JCDE248xCo&+GMkzu&L-uG9Z?Z|~Q4to!nH*FW;*;YU9I(C}Yg_dnnK
z-Dm&qhU<R&&A(c9y#L78|L{LP_||P}<}Q3_?Nh(}2M2Gz`L%C-@4t_Z{O#}j^U25l
z=r2y(^}@4%|Jdjkx4!@V@2>pj_}9Mn#JP8m{K*TiZGHbokB@IVefanbKYrr$cmMtH
zw!1IB_r_Ca{`cuGJl4PM2Om83+})d}&VA>V_3ylP@%Z|aubn^j=F*i%?;d~R<ahSm
z_LXg$KRCQ%<+iiuwmf$Bdnb3Tzw6@r|MZjFww?KK>OU8~^TgB-{_TODx!VU9&+BdT
zpNqTpjr6qn_1f?K{Hh&C+Ws*gz3k54^O|<mR-g7VQy=vyBNc7X25saWm@oK(FZd$P
z_?UnR)6ayT`PKKnz3I8BzOuimu?cPH`@-Kmd6^xw_u%TvsfCwYRS)g<nw)fvIu9;Z
zpY}3SAN48Rk~)btXoEKL4$K#P!54fHH!4iPgz0C(@$YZ>^%I*`=GIFGw|T!f`I%dq
zp2}C!eZ75O{H>UgwD|WQT<$f|=FLS_pXQ3B)lwhzDMJix&<1VfV-sKS1z+$*9LF&M
z6Q-XDo8Er@_YdE7VyNtIYRvX~SFiicxtCo+ZyPwUG2c2<U*EpeYto9OkJP8V%+yDH
z8dPeo1Z^}|1X~(26>ZQ4ZsgM$U+@KA@I~CVuO1Vg-E-!@_8q>Emp2_u2LHA43u~I5
z%2(3KCHHK;xzW$dUVYqa;#--ONqur*#SWl8>XVtK&M(@a4cf>%FkkQmU+{%Fs<uhv
z;Wgsn!A-+=|Loa){kiqhjkB^~(+xkBKGovUx*ffv18tY5-kt}a_nK()<}#{Jb4Ai>
zsgL@UA%-?+gEsQffiL)iFZd!x448lk)6azEe?IZz@kM8UuIz7W%=T|C`Ip6WF1v(2
zJ@44Bl%3ie+J5Gzy(TAJqt4q0)u+A8)JJ^^x1>&@4cefMyaV$EU+@KA#El9QFk$+c
zaQKZMJ@WSPt-1Bm!EN!{gSY;C(^L6Mx~Xr;*v`ftS@P=2Ca;M$Z!VMiG*=|8minkq
z8DeOIHfSRsoA`n+_<}FuI)Mq8F#SxJ-1loQJp1_D&z1d6joJR5r+xTk`yJee33XU~
zDi_~+;h`nH|9IVBZ@=r8wmxz5kKg!*&;0S?uiW})BkjfR|JtoS`6yBYA#KnGZNxE^
zFE9ZUFaZ<d_=^dbW5V#!@iWJ+<g9t)<v+A!v!BY0;>+&7z7xI?W3(=l`lwH4ntDT{
z4cefMJc{QFzTgYKh;u$BU_xgwVfm2@EBdeGGfT#<;D<7YDyJ6a@kaWETKFRf@@lD%
z`Upw+A_g#+fC-p@37Ak@8}JYh@o*;a(DBm$;x^Z5kq8LF1Yw05Ql-!aZP13pNt|^t
z0TVC*6ELB+|8_SI9WQq~xR$kp>M|iHjG#={EA`<W-pNNPzTgYK;EU$YX_jcNNa_;x
zQJ>j$ly$sxyv%o|f_mK@zR(73#BWIBi|Wjb2@X_+6H-FZ25rzro-FVMUv!@@94{R&
za~6h)L7*m3n_YpL!%|tBtJ<F5`JLZG1FV~cFZhBl_#%eQm~c5JI9@tX<;*DNPobO8
ztx!X%6xyH-+Q`E>zTgYK;EOnWV*)009}^re9WV1f5Vi;bO>7dOFm+a_kNV7xP`k2n
zYWUusf%$`XzC3hjbaemttM~7}>D`w`?)zBjw=X@gvHgHBce~f*e2623`lwGle$Ym3
z^xzA=;ETMw&R$<kJbH0r*HmBrC8xvWhQ4KM(q~#cn(SLLwzI|a(NmR8Uem7F>eF@!
zo~J(QQ-&DYpbgr{GZ4Pu3%=kBb9imH#lvgG!^w4LPj1=QpIa|ofhr45kMxw~4wuNa
zXh86NsTeV_tUoz>LkjP^_q{bkXg)Uel&aN7>Z3kojg>ZNgElZTj*^&w2{VTYj+c(9
zIlqftNzikxf*v}g!#GFqU^TD6AN(ntkP?D6XoEKLRF^OKqBDHqc<CpNaz2GFE-Vw4
z&782z5xuOfR4vWRyv)m?xz)|W7kt4Nd=bN4Ot>0MaJ+Oh%DGa^sDdy-SfPegDYQWw
zw2_B%e8Crd!548p#{^91J|;L`I$q{|AZ!sroY*8nVd|_<AN3J@^M&BMHa_AZ9^xS$
z;vpX1zSQy3@iJ#&5wi&0gl>fzQl-!aZO}#@&hZ6b@C9GQoB<Osq5GKNc<FeV_kplQ
z2%}?@2!*M$LVeUnSkD*2`r7!|T|E5om6eC@nYU`r{Occo`24wr=l86*_4UDb?s%&w
z>8TGs-#a=G1=3>9^qQO@u}tctK4li7jhUv6<wq{8=sz*kK5^*n*}Hmd@tg#uEr!j?
zQ03IZ7SAU&@safrrg^p0M}5kSMjNz28<-hm4otuVOuz(8sI3`z*hxGbK01Ep*w(1c
zmeIl5gE!vN@)zQhn|BEBa?TYM622rsLs6J|(bPwMGEvY*4`1*FU*rimU+@KA@I~CG
zFrkx};CQ*)!L<z0)yZBM6ycSJ2kN6fjt6`Z16oYL1WdpLOsK6oc-XBxbi8zM&HF&H
zZwiAtDGZ`jT8%RcpH`!LjKZkG2`M3HgEnZxfiv!Rm~gF_;CShHnX@N!Xu(qiQz%Tm
zXQ_|+I8^XOT>3Eq6EFc2Frl`_;9<A&(DBkiDDMMB+$ZR{RzVLP(qWuOc+g=I6EFc2
znme$Wq`4xgOVmewis|9g)Q01w<E7(eNwy?(D=Cb_t)ECPyUx%CZO}$K(}hQRik?zt
z589xO*`*D~O963cep%)r2or>5qNodoHfV!3@^FqX_<}F^BJN6<P-jAA<<#)KJp=Ox
z?|gaa(CFy?@mKHPf781!jojDTF_leA#&*U+wis5uCihw_Q|HxZa^2aJTlV#rt<nuC
zR2G~b>1qAV8{z9)G;m;J`-U@jdv|?AbLtmgdk{-QT0t=CqdsMjp$*!gjR4DfYvBvN
z;0wNp;}j-fLT4~x;?au}yQcbbd!~!ghQ4KM(q~#cn(Pxmv@2xig$v#<?^o<gg{f(E
zrUiw(%*(vo>{=Xau1M+<^--Uq>(fSqpnP#PeBpSx)WNk3FICtG!USQR5rpw6pSJ!c
z8;rszj7nDny<5M~FZ7G%&TwserQ@a0KkrKgmxMT1BgCOG8f(O&X(jlJzY5yYM#Ck1
z!54gyZ%2H=7c<8fj+c&?Im5z~b8W*8fnymus&9$+qB)97tEE2b<A@dDUAG5c@C9G+
zMI*W~OPDj$$titw_4<Y5rNdIr-9l}`GGSSvhEyrEK^wG@hjV<v7kt4NG1I^VOz1u)
zI9@tl=6xV+5mztB5Ezx=MVLq%w9y(v_!EcJ0Oq=2)JJ_BCi%i)vbMhBVJGp>@zU`!
zXLB*@3f+Wmg&I<&&<1VLM!psE1z+$5U&J{d6ELCsnBaJM6^@tfK0Ve)?4~P8^CR_X
zu1IW3y*=uqJ~5b%Es8d1gEsQ9l`r^$FZe>QtVKMX$HV1EF0AN3F_cR*9e($&9$P#o
zkEv3+Rfa037N);1e{^WK*OVc#S}pa7u%Ms#{ZO<)8)Xot4ced${X)N}4W)R9hj{pD
z^Kkg+_?cr{Ba>S0K5Gx&cuUJ)h)-_b(K|X2pJ*ZQ&Rekkn{y2oZ=&}rjZm5&sZVo7
zQZ=cM`V=&wjYg;C3%=ls{Hnnhe8Cs9#TSm3j;Xnp!xSh4Y>|<u&&N|TZP12*IKZGj
z7x4vO@C9EqqNHjiFaZ;~j|q;Kj+c2KDB>ewqA;;gL#h<opbgr{!#Td-3%=lsxKCjM
zCUhSY94{R&^F9!^2%%eS5}`14R;Z8qbYCdxr{~K?g(@L10TVDmuXMbut(17!tvqzR
zbiB;_KruTB-Gpw18d9au25rzr9?tOvU+@KA#GL{YFroXH;CShHnfHOPMF^f^lL&>W
zvqF8;$Dx8R94cz#BOc-*9^xS$;^Cp)j+c&?ISY%JMd&7UE7XuG<&$XR!&g=wzGvR5
zIrFc7{NeNG7M|a;;?~y(-?`(ho-msDc<Q#n&ph=}pR!$sHfV!3LNlqialYUSzWB6#
zF}d#S$u0Z(^M6J<Y*!YX9_fiWMaxNa(ZGR?>F>)Q-M-Xo%4A=yminZtP4gr5X|71B
zCiPLDGEmb7ZP139Bo1$wfC-p@3Dd)biAOI^?3(J!%bO0Z8~T>5NuO!)XtHm~*v=Nu
zNB;_ygi2)+p$*!gjXde&3%=kBzA#7C_8>gOLp+=bJaoKtOwDx~!Uo}HZ0eyfb-t;O
z`UrpdVtT?~-{?~|MphMvz1WMr*o(cjxS{iS=y>Vin)9jX>4HK*VWEapDYQWww2_B%
ze8Crd!54A&#{^91J|;L`I$q{|AZ!tx7l;V-%P>`a+q_?~sRtp~i>5y6<KV#;4j#2}
z9}n>m5Akq%c<6ZPc$s^v*k6QhLbpN<sZwZzHfSRc=lFsz_<}Fu-iisB(0xpBymY+G
z`#{(tgqyKRgu>KWp+4#({N;=334eW~M%hSKRUGzWFZN<D_SRPS&f}rurQ>DJr=q6|
z-Gpw18d9au25rzr9?ngZFDfgihVShem_K;u%R`4oNB57vdjI~L-hFB0zBpZ7dOr^B
zKDDs@@-evBYc7>b4Vzd-8?=$5tu_GA25o#YZ44hBKXYtrSu}L1x@!;KcuPw%$0s-M
z=p7x1Ph9%##gm`pW832&=g&!3faXW)(_E2s22&sPDX2snjSj#Ue8Cq@E7(T`;S0Xt
z3%;;})b`CQ<Kgln7gqG27|Lan4qSUzk1d{4w#PPHR~f1ZQX9o+1!G}QG-s-?>Lc}0
zpEAhM25ry=W;S9F{lcte)@nVs>y>(CXY@+PO9$6{W+}LRWg&$cs$rb_=%?EEd8bvz
zMjN!DU+5S5MKc}si{^@?E>WMERv*VpKe^I60jp1ef-pf?X9Quw`eHFl+ivv{pYkc6
z@<m+QFaZ-V0TVtwCOBRSQgi<mbF9E<rUgdyOwX~O@UmkeCSU?4G<R~dO>;$3m#B~W
z6w|~1M;nfpj+c&?B^jsix1=z2=BSVQ2*=a5TYbbAe8Crd5f^$)z=ZB&g5#xNH}3;Q
zWFf#3U=?ael|mb|K^u8E#}|CT7km--DNMkG?qh=ErQ>DZ2f`L1bc;<Q6sFD!^--Vh
z3nl#&Y}u$#B?Km50w#1qul(?pm51+{w`$J(>mPsk{JDka_pG?}^}%=Uc&n%E?fKZ&
zo%fOT2R61}&gO3SZ_f1=8Ld8@RG*1QFHY>5>bsn%H}oxA)4r8_^cGL{Eg9Q+nO9BI
z$5Xfb_xrorE|l}sM}5ldN*lC68~Jd`7kt4Nd=Y1OOuz(8XvKudb!Sg*+1H={Qo65I
z7MvdGiQixz^MXs2T{Iwkxm1i66%wpOD5namK2jg`DT54c&<1T_W^AjNfC-p@37Ak@
z?>mo&j+YLuIiHFhSuiJ<E7XuGg*Iq|Hu7YaFZhBl_#*E9n1Bi0#{|dAr4FuTvac=^
zofn9Bzr0`h=t8S}!54fHM|w=a1WdpLOejwL*LI$FymY)wPg3b4p_|aHtV7cVZO}$O
zz3~NK@C9GQtp*b?q5GKNc<FeV_kplsUfZxk*jk1lK|F2HMr%}IR2*ajyz7EdAN48D
z7b!((gEnXbGpFaLR~;`MFLQ4d9Z2XVbSu=5Dup&^gEsPwmoNB&FZd$vt(brb-NyvS
zOUKK+4}>j3xEY&7C`_Fd>Z3lwU%r^0@Ylh<Y$U5H4tuc|d$AXLYpeU!;Ni9(zW3GM
zRiAtE!7n}c%=-0DZ5w`Z<g?#+;D$Zbzv02>dq)Riv0K3T%3ibl$b}XCCx+T54!u2l
zSC1{8({?@i=wDD7s`zN`cJGRh=5C?W&^gsdeaaw18?-?i0haYf!xwzP7km-dZ%n|1
z&S1ju(eX3KwnlBX+>h2Cyz!QnzYw3?yyI#uz5bk>mqn$oEI3d@HEiu``pJXf*pJjl
zeG1y96rl~;ppE<*!54hN7krV<QI40p9bC(>P#vrUVbL(@sHr}@!#jCP<qN*x3%+RX
zoMwsUili=4ANA>sqvO@;UyhfKeEH5)P){f+lq}ScDup&^gEsOlf-m@jFZd$vN|=BN
z-NyvSOUKK+4}>j3pcb1%C`_Fd>Z3lhD{#D8!KFW^Y@Dc41UGO4H}uo^(<_*O3A2j{
zj+c&?Ipd4DNa!YXE7XuGg*Iq|Hu7+eFZhBl_#*B~n1Bi0#{|bq$IHAAge^j#7MnyU
zOq~_#qdv1MaCCGm8w0BD=XZYRcYV;&v9>PaAs&|V(DBmoGVc||d?j=fx)o|jl|mb|
zK^u8E#}|CT7km--4otv=?qh=ErQ>DZ2f`LH8^Wo|%BkUddj{qY-ud#-q0!O(<FDSo
z|E70e8o959->=5U)mB03&j}(@=fq>QK^yV-5$!=6v_TvBD#90h!56c|7ZZ<OoY*zh
zmw!1ik(s-FL*KGB=`$@JP4+Dr+u7pz=$FbSuW46o^+^WwOik*eKA9-$hN2DHppE<+
znD~M(_<}FY;kCUL53dyuC)b@lxn*B}ZoPB`sw_A?(o>cxTq4(^fdd=U-<LlUPDPWe
zk3hd>b-v(>GQiRXZP12ZsaNWidVyXrb9#Z}rDJN|w=V?-^8`KDD(Im@I*jup4^}fz
z{K22X2`M3HgEnX*zYOsOUv!2q950tTxRyz=+RTMz!m^nYmhmz#=i?TR@daP-MRO-N
z+cZ}sb&2|@&nKJx94{S>(nvUcBnT6Pr9G#3L>shGr~x+`r)R$43%<y=S-#*4zUU@j
zI9@tl=6Vm4q5w;P^~nM(hm*3FTD2K}@fUxEhEq2SU+@KA@I@oC(<^7<9OZcFc$qV-
zn1=<90>?rPsZwZzHfSRc=lFsz_<}EDK7a|B(0xpBymY+G`#{(tgvhZ;gu>KWp+4#}
z8$<RFUs-who_VY0%)kEehtHo|cz(}{TVEf1=Z?2}+R(zsiRXNXBZc~?PdvuZMhzx>
z!54gyuivxR7t4=aSkZrCDF2evVRG;4vBh)R3>+W7x-wKbwJ`mC`J+R-y{61!)oQ6v
zY{v8xR~p)&jWP(+25r!WexYB~RyaJwLp=Pnc{qG@{LHbfkx4CgpS1^XyrtzY#3wiJ
z=p7x1PqYyD>|NA-?YWi<F?invmxUTqrO*a#&_<r{@&#Y;1z*H58WS*K<}ktW(lIsX
zcbNW#fD!JYFm;cskNOBn`633in1BhGfC-pTTLtm3TY2bs>FAjEfg&CemI=!WHKa<R
z4cefMJe=bTzTgYKh&u%)U_$pX!ST}ZGVcRnix51;CJ_o#XNCHxk3$7tI8@ZeM?Azs
zJj6ph#KS|o9WNa(a~2jci_lHzR;VFW3T@B^ZRFt`U+@KA@I}lSFaZ;~j|q;Kj+c2K
z2wQ|OIyQ+=m^v%eM}36#d?Bo_jgNSUhj@sGc!-CNmqI0>Qul>Q4)%HfFI0dDm{6!8
zRSIp;Mr&<c{Na0F?OpY`Cm;OMbI+__|J1hO7e_w(jR$Vnvv^)*(~_~B?UzokQJ=D<
zoHl4<mT6;h-Pw~{_Vt&3RrMZIS#Wx!C*~9_$-8Laz{d7XY3_C(S)YzE&5zWlxgsfZ
z)JJ{FKtvm~K^u7o<_o^y3%-a$J0@Vl^fO`N(TfwiruuU0r9=IOzGZ9DXIea(>=WL#
zE2MC<?RV!2?^<{&GLU}gr%WQWK^wG@r{sLW7kt4N=BU~(d2M+(xo+FH|MWL{c0RcD
GC;kuh#*^&;

literal 0
HcmV?d00001

diff --git a/regression-test/suites/fault_injection_p0/test_wal_mem_back_pressure_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_wal_mem_back_pressure_fault_injection.groovy
new file mode 100644
index 00000000000000..473abb6bb2a3d7
--- /dev/null
+++ b/regression-test/suites/fault_injection_p0/test_wal_mem_back_pressure_fault_injection.groovy
@@ -0,0 +1,142 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_wal_mem_back_pressure_fault_injection","nonConcurrent") {
+
+
+    def tableName = "wal_test"
+    sql """ DROP TABLE IF EXISTS ${tableName} """
+    sql """
+        CREATE TABLE IF NOT EXISTS `wal_baseall` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    sql """
+        CREATE TABLE IF NOT EXISTS `wal_test` (
+            `k0` boolean null comment "",
+            `k1` tinyint(4) null comment "",
+            `k2` smallint(6) null comment "",
+            `k3` int(11) null comment "",
+            `k4` bigint(20) null comment "",
+            `k5` decimal(9, 3) null comment "",
+            `k6` char(5) null comment "",
+            `k10` date null comment "",
+            `k11` datetime null comment "",
+            `k7` varchar(20) null comment "",
+            `k8` double max null comment "",
+            `k9` float sum null comment "",
+            `k12` string replace_if_not_null null comment "",
+            `k13` largeint(40) replace null comment ""
+        ) engine=olap
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 5 properties("replication_num" = "1")
+        """
+
+    streamLoad {
+        table "wal_baseall"
+        db "regression_test_fault_injection_p0"
+        set 'column_separator', ','
+        file "baseall.txt"
+    }
+
+    def enable_back_pressure = {
+        try {
+            def fes = sql_return_maparray "show frontends"
+            def bes = sql_return_maparray "show backends"
+            logger.info("frontends: ${fes}")
+                def fe = fes[0]
+                def be = bes[0]
+                    def url = "jdbc:mysql://${fe.Host}:${fe.QueryPort}/"
+                    logger.info("observer url: " + url)
+                        StringBuilder sb = new StringBuilder();
+                        sb.append("curl -X POST http://${fe.Host}:${fe.HttpPort}")
+                        sb.append("/rest/v2/manager/node/set_config/be")
+                        sb.append(" -H \"Content-Type: application/json\" -H \"Authorization: Basic cm9vdDo= \"")
+                        sb.append(""" -d \"{\\"group_commit_queue_mem_limit\\": {\\"node\\": [\\"${be.Host}:${be.HttpPort}\\"],\\"value\\": \\"0\\",\\"persist\\": \\"false\\"}}\"""")
+                        String command = sb.toString()
+                        logger.info(command)
+                        def process = command.execute()
+        } finally {
+        }
+    }
+
+    def disable_back_pressure = {
+        try {
+            def fes = sql_return_maparray "show frontends"
+            def bes = sql_return_maparray "show backends"
+            logger.info("frontends: ${fes}")
+                def fe = fes[0]
+                def be = bes[0]
+                    def url = "jdbc:mysql://${fe.Host}:${fe.QueryPort}/"
+                    logger.info("observer url: " + url)
+                        StringBuilder sb = new StringBuilder();
+                        sb.append("curl -X POST http://${fe.Host}:${fe.HttpPort}")
+                        sb.append("/rest/v2/manager/node/set_config/be")
+                        sb.append(" -H \"Content-Type: application/json\" -H \"Authorization: Basic cm9vdDo= \"")
+                        sb.append(""" -d \"{\\"group_commit_queue_mem_limit\\": {\\"node\\": [\\"${be.Host}:${be.HttpPort}\\"],\\"value\\": \\"67108864\\",\\"persist\\": \\"false\\"}}\"""")
+                        String command = sb.toString()
+                        logger.info(command)
+                        def process = command.execute()
+        } finally {
+        }
+    }
+
+    boolean finish = false
+    enable_back_pressure()
+    def thread1 = new Thread({
+    sql """ set group_commit = async_mode; """
+        try {
+            sql """insert into ${tableName} select * from wal_baseall where k1 <= 3"""
+        } catch (Exception e) {
+            logger.info(e.getMessage())
+            assertTrue(e.getMessage().contains('Communications link failure'))
+        } 
+    disable_back_pressure()
+    finish  = true
+    })
+    thread1.start()
+
+    for(int i = 0;i<10;i++){
+        def processList = sql "show processlist"
+        logger.info(processList.toString())
+        processList.each { item ->
+            logger.info(item[1].toString())
+            logger.info(item[11].toString())
+            if (item[11].toString() == "insert into ${tableName} select * from wal_baseall where k1 <= 3".toString()){
+                def res = sql "kill ${item[1]}"
+                logger.info(res.toString())
+            }
+        }
+    }
+
+    thread1.join()
+
+}
\ No newline at end of file

From f54f79515c7c9a4468aae019ab0e3e022932c0fb Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@hotmail.com>
Date: Wed, 3 Jan 2024 08:25:22 +0800
Subject: [PATCH 263/299] [Bug](fix) str_to_date "" should be null (#29402)

---
 be/src/vec/functions/function_timestamp.cpp         | 13 +++++++------
 be/src/vec/runtime/vdatetime_value.cpp              |  4 ++++
 .../datetime_functions/test_date_function.out       |  3 +++
 .../datetime_functions/test_date_function.groovy    |  1 +
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/be/src/vec/functions/function_timestamp.cpp b/be/src/vec/functions/function_timestamp.cpp
index 801fa37ca0c869..764c6a72f44087 100644
--- a/be/src/vec/functions/function_timestamp.cpp
+++ b/be/src/vec/functions/function_timestamp.cpp
@@ -228,12 +228,13 @@ struct StrToDate {
         auto& ts_val = *reinterpret_cast<DateValueType*>(&res[index]);
         if (!ts_val.from_date_format_str(r_raw_str, r_str_size, l_raw_str, l_str_size)) {
             null_map[index] = 1;
-        }
-        if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
-            if (context->get_return_type().type == doris::PrimitiveType::TYPE_DATETIME) {
-                ts_val.to_datetime();
-            } else {
-                ts_val.cast_to_date();
+        } else {
+            if constexpr (std::is_same_v<DateValueType, VecDateTimeValue>) {
+                if (context->get_return_type().type == doris::PrimitiveType::TYPE_DATETIME) {
+                    ts_val.to_datetime();
+                } else {
+                    ts_val.cast_to_date();
+                }
             }
         }
     }
diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp
index f625631ace9572..2be57b0ae20245 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -2479,6 +2479,10 @@ bool DateV2Value<T>::from_date_format_str(const char* format, int format_len, co
         }
     }
 
+    // ptr == format means input value string is "", not do parse format failed here
+    if (ptr == format) {
+        return false;
+    }
     // continue to iterate pattern if has
     // to find out if it has time part.
     while (ptr < end) {
diff --git a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
index 7ad5b93341c3b4..dc1d05c9d48c35 100644
--- a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
+++ b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
@@ -266,6 +266,9 @@ February
 -- !sql --
 2014-12-21T12:34:56
 
+-- !sql --
+\N
+
 -- !sql --
 2014-12-21T12:34:56
 
diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
index 902883c5be0d72..cbef7072b266e8 100644
--- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
+++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -330,6 +330,7 @@ suite("test_date_function") {
     sql """ truncate table ${tableName} """
     sql """ insert into ${tableName} values ("2014-12-21 12:34:56")  """
     qt_sql """ select str_to_date(test_datetime, '%Y-%m-%d %H:%i:%s') from ${tableName}; """
+    qt_sql """ select str_to_date("", "%Y-%m-%d %H:%i:%s"); """
     qt_sql """ select str_to_date("2014-12-21 12:34%3A56", '%Y-%m-%d %H:%i%%3A%s'); """
     qt_sql """ select str_to_date("2014-12-21 12:34:56.789 PM", '%Y-%m-%d %h:%i:%s.%f %p'); """
     qt_sql """ select str_to_date('2023-07-05T02:09:55.880Z','%Y-%m-%dT%H:%i:%s.%fZ') """

From 298d0c6904b2588157f28d55e4199f905e3f565f Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Wed, 3 Jan 2024 09:50:30 +0800
Subject: [PATCH 264/299] [Chore](Notice)Update Notice year (#29415)

---
 NOTICE.txt           | 2 +-
 dist/NOTICE-dist.txt | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/NOTICE.txt b/NOTICE.txt
index 7bbac0f0ed8458..e82d696233c307 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -1,5 +1,5 @@
 Apache Doris
-Copyright 2018-2023 The Apache Software Foundation
+Copyright 2018-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).
diff --git a/dist/NOTICE-dist.txt b/dist/NOTICE-dist.txt
index 14ee68b04ec4aa..e39c3a8b47eb22 100644
--- a/dist/NOTICE-dist.txt
+++ b/dist/NOTICE-dist.txt
@@ -1,5 +1,5 @@
 Apache Doris
-Copyright 2018-2023 The Apache Software Foundation
+Copyright 2018-2024 The Apache Software Foundation
 
 This product includes software developed at
 The Apache Software Foundation (http://www.apache.org/).

From 97234a64853a89c34db219999ace002b262af15c Mon Sep 17 00:00:00 2001
From: airborne12 <airborne08@gmail.com>
Date: Wed, 3 Jan 2024 09:54:31 +0800
Subject: [PATCH 265/299] [Enhancement](inverted index) strictly checkout
 inverted index properties (#29421)

---
 .../doris/analysis/InvertedIndexUtil.java     |  99 ++++++--
 .../inverted_index_p0/test_properties.groovy  | 233 ++++++++++++++++++
 2 files changed, 311 insertions(+), 21 deletions(-)
 create mode 100644 regression-test/suites/inverted_index_p0/test_properties.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
index 2cd62692337ed1..44e8bd15347d24 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InvertedIndexUtil.java
@@ -20,8 +20,11 @@
 import org.apache.doris.catalog.PrimitiveType;
 import org.apache.doris.common.AnalysisException;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 public class InvertedIndexUtil {
 
@@ -43,9 +46,11 @@ public class InvertedIndexUtil {
 
     public static String INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE = "char_replace";
 
-    public static String INVERTED_INDEX_PARSER_IGNORE_ABOVE = "ignore_above";
+    public static String INVERTED_INDEX_SUPPORT_PHRASE_KEY = "support_phrase";
 
-    public static String INVERTED_INDEX_PARSER_LOWERCASE = "lower_case";
+    public static String INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY = "ignore_above";
+
+    public static String INVERTED_INDEX_PARSER_LOWERCASE_KEY = "lower_case";
 
     public static String getInvertedIndexParser(Map<String, String> properties) {
         String parser = properties == null ? null : properties.get(INVERTED_INDEX_PARSER_KEY);
@@ -100,26 +105,9 @@ public static void checkInvertedIndexParser(String indexColName, PrimitiveType c
         if (properties != null) {
             parser = properties.get(INVERTED_INDEX_PARSER_KEY);
             if (parser == null && !properties.isEmpty()) {
-                throw new AnalysisException("invalid index properties, please check the properties");
-            }
-            String ignoreAbove = properties.get(INVERTED_INDEX_PARSER_IGNORE_ABOVE);
-            if (ignoreAbove != null) {
-                try {
-                    int ignoreAboveValue = Integer.parseInt(ignoreAbove);
-                    if (ignoreAboveValue <= 0) {
-                        throw new AnalysisException("invalid index properties, ignore_above must be positive");
-                    }
-                } catch (NumberFormatException e) {
-                    throw new AnalysisException("invalid index properties, ignore_above must be integer");
-                }
+                throw new AnalysisException("Invalid index properties, parser must not be none");
             }
-            String lowerCase = properties.get(INVERTED_INDEX_PARSER_LOWERCASE);
-            if (lowerCase != null) {
-                if (!"true".equals(lowerCase) && !"false".equals(lowerCase)) {
-                    throw new AnalysisException("invalid index properties, lowercase must be true or false");
-                }
-            }
-
+            checkInvertedIndexProperties(properties);
         }
 
         // default is "none" if not set
@@ -141,4 +129,73 @@ public static void checkInvertedIndexParser(String indexColName, PrimitiveType c
                 + " is not supported for column: " + indexColName + " of type " + colType);
         }
     }
+
+    public static void checkInvertedIndexProperties(Map<String, String> properties) throws AnalysisException {
+        Set<String> allowedKeys = new HashSet<>(Arrays.asList(
+                INVERTED_INDEX_PARSER_KEY,
+                INVERTED_INDEX_PARSER_MODE_KEY,
+                INVERTED_INDEX_SUPPORT_PHRASE_KEY,
+                INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE,
+                INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN,
+                INVERTED_INDEX_PARSER_CHAR_FILTER_REPLACEMENT,
+                INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY,
+                INVERTED_INDEX_PARSER_LOWERCASE_KEY
+        ));
+
+        for (String key : properties.keySet()) {
+            if (!allowedKeys.contains(key)) {
+                throw new AnalysisException("Invalid inverted index property key: " + key);
+            }
+        }
+
+        String parser = properties.get(INVERTED_INDEX_PARSER_KEY);
+        String parserMode = properties.get(INVERTED_INDEX_PARSER_MODE_KEY);
+        String supportPhrase = properties.get(INVERTED_INDEX_SUPPORT_PHRASE_KEY);
+        String charFilterType = properties.get(INVERTED_INDEX_PARSER_CHAR_FILTER_TYPE);
+        String charFilterPattern = properties.get(INVERTED_INDEX_PARSER_CHAR_FILTER_PATTERN);
+        String ignoreAbove = properties.get(INVERTED_INDEX_PARSER_IGNORE_ABOVE_KEY);
+        String lowerCase = properties.get(INVERTED_INDEX_PARSER_LOWERCASE_KEY);
+
+        if (parser != null && !parser.matches("none|english|unicode|chinese|standard")) {
+            throw new AnalysisException("Invalid inverted index 'parser' value: " + parser
+                    + ", parser must be none, english, unicode or chinese");
+        }
+
+        if (!"chinese".equals(parser) && parserMode != null) {
+            throw new AnalysisException("parser_mode is only available for chinese parser");
+        }
+
+        if ("chinese".equals(parser) && (parserMode != null && !parserMode.matches("fine_grained|coarse_grained"))) {
+            throw new AnalysisException("Invalid inverted index 'parser_mode' value: " + parserMode
+                    + ", parser_mode must be fine_grained or coarse_grained");
+        }
+
+        if (supportPhrase != null && !supportPhrase.matches("true|false")) {
+            throw new AnalysisException("Invalid inverted index 'support_phrase' value: " + supportPhrase
+                    + ", support_phrase must be true or false");
+        }
+
+        if (INVERTED_INDEX_CHAR_FILTER_CHAR_REPLACE.equals(charFilterType) && (charFilterPattern == null
+                || charFilterPattern.isEmpty())) {
+            throw new AnalysisException("Missing 'char_filter_pattern' for 'char_replace' filter type");
+        }
+
+        if (ignoreAbove != null) {
+            try {
+                int ignoreAboveValue = Integer.parseInt(ignoreAbove);
+                if (ignoreAboveValue <= 0) {
+                    throw new AnalysisException("Invalid inverted index 'ignore_above' value: " + ignoreAboveValue
+                            + ", ignore_above must be positive");
+                }
+            } catch (NumberFormatException e) {
+                throw new AnalysisException(
+                        "Invalid inverted index 'ignore_above' value, ignore_above must be integer");
+            }
+        }
+
+        if (lowerCase != null && !lowerCase.matches("true|false")) {
+            throw new AnalysisException(
+                    "Invalid inverted index 'lower_case' value: " + lowerCase + ", lower_case must be true or false");
+        }
+    }
 }
diff --git a/regression-test/suites/inverted_index_p0/test_properties.groovy b/regression-test/suites/inverted_index_p0/test_properties.groovy
new file mode 100644
index 00000000000000..84b0aef8f3edcf
--- /dev/null
+++ b/regression-test/suites/inverted_index_p0/test_properties.groovy
@@ -0,0 +1,233 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+suite("test_properties", "p0"){
+    // prepare test table
+    def indexTblName = "test_properties"
+
+    sql "DROP TABLE IF EXISTS ${indexTblName}"
+    def success = false;
+
+    def create_table_with_inverted_index_properties = { create_sql, error_msg->
+        try {
+            sql create_sql
+            success = true;
+        } catch(Throwable ex) {
+            logger.info("create_table_with_inverted_index_properties result: " + ex)
+            if (ex != null) {
+                def msg = ex.toString()
+                assertTrue(msg != null && msg.contains(error_msg), "Expect exception msg contains '${error_msg}', but meet '${msg}'")
+            }
+            success = false;
+        } finally {
+            sql "DROP TABLE IF EXISTS ${indexTblName}"
+        }
+    }
+
+    def empty_parser = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+        	`id`int(11)NULL,
+        	`c` text NULL,
+        	INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+        	"replication_allocation" = "tag.location.default: 1"
+        );
+        """
+    create_table_with_inverted_index_properties(empty_parser, "Invalid inverted index 'parser' value")
+    assertEquals(success, false)
+
+    def wrong_parser = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+        	`id`int(11)NULL,
+        	`c` text NULL,
+        	INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="german") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+        	"replication_allocation" = "tag.location.default: 1"
+        );
+        """
+    create_table_with_inverted_index_properties(wrong_parser, "Invalid inverted index 'parser' value")
+    assertEquals(success, false)
+
+    def wrong_parser_mode = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "parser_mode"="fine_grained") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(wrong_parser_mode, "parser_mode is only available for chinese parser")
+    assertEquals(success, false)
+
+    def valid_parser_and_mode = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="chinese", "parser_mode"="fine_grained") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(valid_parser_and_mode, "")
+    assertEquals(success, true)
+
+    def missing_char_filter_pattern = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "char_filter_type"="char_replace") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(missing_char_filter_pattern, "Missing 'char_filter_pattern' for 'char_replace' filter type")
+    assertEquals(success, false)
+
+    def invalid_property_key = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("invalid_key"="value") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(invalid_property_key, "Invalid index properties, parser must not be none")
+    assertEquals(success, false)
+
+    def invalid_property_key2 = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "invalid_key"="value") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(invalid_property_key2, "Invalid inverted index property key:")
+    assertEquals(success, false)
+
+    def invalid_ignore_above = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "ignore_above"="-1") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(invalid_ignore_above, "Invalid inverted index 'ignore_above' value")
+    assertEquals(success, false)
+
+    def non_numeric_ignore_above = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "ignore_above"="non_numeric") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(non_numeric_ignore_above, "ignore_above must be integer")
+    assertEquals(success, false)
+
+    def invalid_lower_case = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "lower_case"="invalid") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(invalid_lower_case, "Invalid inverted index 'lower_case' value")
+    assertEquals(success, false)
+
+    def invalid_support_phrase = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "support_phrase"="invalid") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(invalid_support_phrase, "Invalid inverted index 'support_phrase' value")
+    assertEquals(success, false)
+
+    def invalid_support_phrase2 = """
+        CREATE TABLE IF NOT EXISTS ${indexTblName}(
+            `id` int(11) NULL,
+            `c` text NULL,
+            INDEX c_idx(`c`) USING INVERTED PROPERTIES("parser"="english", "support_phase"="true") COMMENT ''
+        ) ENGINE=OLAP
+        DUPLICATE KEY(`id`)
+        COMMENT 'OLAP'
+        DISTRIBUTED BY HASH(`id`) BUCKETS 1
+        PROPERTIES(
+            "replication_allocation" = "tag.location.default: 1"
+        );
+    """
+    create_table_with_inverted_index_properties(invalid_support_phrase2, "Invalid inverted index property key: support_phase")
+    assertEquals(success, false)
+}

From cdf2bb24a4f8443d302e8c7b78b54d84f2ce4d5a Mon Sep 17 00:00:00 2001
From: morrySnow <101034200+morrySnow@users.noreply.github.com>
Date: Wed, 3 Jan 2024 09:55:00 +0800
Subject: [PATCH 266/299] [fix](Nereids) topn runtime filter only support
 simplest case (#29312)

only support simple case: select ... from tbl [where ...] order by ... limit ...
---
 .../nereids/processor/post/TopNScanOpt.java   | 58 ++++++++++++++-----
 .../postprocess/TopNRuntimeFilterTest.java    | 22 ++++++-
 2 files changed, 65 insertions(+), 15 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java
index 2ca1380d1b050f..bc0586f40c405a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/TopNScanOpt.java
@@ -24,27 +24,58 @@
 import org.apache.doris.nereids.trees.plans.algebra.Filter;
 import org.apache.doris.nereids.trees.plans.algebra.OlapScan;
 import org.apache.doris.nereids.trees.plans.algebra.Project;
+import org.apache.doris.nereids.trees.plans.algebra.TopN;
+import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalSort;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalDistribute;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
 import org.apache.doris.qe.ConnectContext;
 
-import com.google.common.collect.ImmutableList;
-
 /**
  * topN opt
  * refer to:
  * <a href="https://github.com/apache/doris/pull/15558">...</a>
  * <a href="https://github.com/apache/doris/pull/15663">...</a>
+ *
+ * // only support simple case: select ... from tbl [where ...] order by ... limit ...
  */
 
 public class TopNScanOpt extends PlanPostProcessor {
 
+    @Override
+    public Plan visit(Plan plan, CascadesContext context) {
+        return plan;
+    }
+
+    @Override
+    public Plan visitPhysicalSink(PhysicalSink<? extends Plan> physicalSink, CascadesContext context) {
+        if (physicalSink.child() instanceof TopN) {
+            return super.visit(physicalSink, context);
+        }
+        return physicalSink;
+    }
+
+    @Override
+    public Plan visitPhysicalDistribute(PhysicalDistribute<? extends Plan> distribute, CascadesContext context) {
+        if (distribute.child() instanceof TopN && distribute.child() instanceof AbstractPhysicalSort
+                && ((AbstractPhysicalSort<?>) distribute.child()).getSortPhase() == SortPhase.LOCAL_SORT) {
+            return super.visit(distribute, context);
+        }
+        return distribute;
+    }
+
     @Override
     public PhysicalTopN<? extends Plan> visitPhysicalTopN(PhysicalTopN<? extends Plan> topN, CascadesContext ctx) {
-        Plan child = topN.child().accept(this, ctx);
-        topN = rewriteTopN(topN);
-        if (child != topN.child()) {
-            topN = ((PhysicalTopN) topN.withChildren(child)).copyStatsAndGroupIdFrom(topN);
+        if (topN.getSortPhase() == SortPhase.LOCAL_SORT) {
+            Plan child = topN.child();
+            topN = rewriteTopN(topN);
+            if (child != topN.child()) {
+                topN = ((PhysicalTopN<? extends Plan>) topN.withChildren(child)).copyStatsAndGroupIdFrom(topN);
+            }
+            return topN;
+        } else if (topN.getSortPhase() == SortPhase.MERGE_SORT) {
+            return (PhysicalTopN<? extends Plan>) super.visit(topN, ctx);
         }
         return topN;
     }
@@ -52,13 +83,14 @@ public PhysicalTopN<? extends Plan> visitPhysicalTopN(PhysicalTopN<? extends Pla
     @Override
     public Plan visitPhysicalDeferMaterializeTopN(PhysicalDeferMaterializeTopN<? extends Plan> topN,
             CascadesContext context) {
-        Plan child = topN.child().accept(this, context);
-        if (child != topN.child()) {
-            topN = topN.withChildren(ImmutableList.of(child)).copyStatsAndGroupIdFrom(topN);
-        }
-        PhysicalTopN<? extends Plan> rewrittenTopN = rewriteTopN(topN.getPhysicalTopN());
-        if (topN.getPhysicalTopN() != rewrittenTopN) {
-            topN = topN.withPhysicalTopN(rewrittenTopN).copyStatsAndGroupIdFrom(topN);
+        if (topN.getSortPhase() == SortPhase.LOCAL_SORT) {
+            PhysicalTopN<? extends Plan> rewrittenTopN = rewriteTopN(topN.getPhysicalTopN());
+            if (topN.getPhysicalTopN() != rewrittenTopN) {
+                topN = topN.withPhysicalTopN(rewrittenTopN).copyStatsAndGroupIdFrom(topN);
+            }
+            return topN;
+        } else if (topN.getSortPhase() == SortPhase.MERGE_SORT) {
+            return super.visit(topN, context);
         }
         return topN;
     }
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java
index f4fdf6f44f069b..0ac233898adff5 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/TopNRuntimeFilterTest.java
@@ -20,8 +20,10 @@
 import org.apache.doris.nereids.datasets.ssb.SSBTestBase;
 import org.apache.doris.nereids.processor.post.PlanPostProcessors;
 import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.SortPhase;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeTopN;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalTopN;
 import org.apache.doris.nereids.util.PlanChecker;
 
 import org.junit.jupiter.api.Assertions;
@@ -41,7 +43,7 @@ public void testUseTopNRf() {
                 .implement();
         PhysicalPlan plan = checker.getPhysicalPlan();
         plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan);
-        Assertions.assertTrue(plan.children().get(0).child(0) instanceof PhysicalDeferMaterializeTopN);
+        Assertions.assertInstanceOf(PhysicalDeferMaterializeTopN.class, plan.children().get(0).child(0));
         PhysicalDeferMaterializeTopN<? extends Plan> localTopN
                 = (PhysicalDeferMaterializeTopN<? extends Plan>) plan.child(0).child(0);
         Assertions.assertTrue(localTopN.getPhysicalTopN().isEnableRuntimeFilter());
@@ -56,9 +58,25 @@ public void testNotUseTopNRf() {
                 .implement();
         PhysicalPlan plan = checker.getPhysicalPlan();
         plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan);
-        Assertions.assertTrue(plan.children().get(0).child(0) instanceof PhysicalDeferMaterializeTopN);
+        Assertions.assertInstanceOf(PhysicalDeferMaterializeTopN.class, plan.children().get(0).child(0));
         PhysicalDeferMaterializeTopN<? extends Plan> localTopN
                 = (PhysicalDeferMaterializeTopN<? extends Plan>) plan.child(0).child(0);
         Assertions.assertFalse(localTopN.getPhysicalTopN().isEnableRuntimeFilter());
     }
+
+    @Test
+    public void testNotUseTopNRfForComplexCase() {
+        String sql = "select * from (select 1) tl join (select * from customer order by c_custkey limit 5) tb";
+        PlanChecker checker = PlanChecker.from(connectContext).analyze(sql)
+                .rewrite()
+                .implement();
+        PhysicalPlan plan = checker.getPhysicalPlan();
+        plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan);
+        Assertions.assertInstanceOf(PhysicalTopN.class, plan.child(0).child(0).child(1).child(0));
+        Assertions.assertEquals(SortPhase.LOCAL_SORT, ((PhysicalTopN<? extends Plan>) plan
+                .child(0).child(0).child(1).child(0)).getSortPhase());
+        PhysicalTopN<? extends Plan> localTopN = (PhysicalTopN<? extends Plan>) plan
+                .child(0).child(0).child(1).child(0);
+        Assertions.assertFalse(localTopN.isEnableRuntimeFilter());
+    }
 }

From 358995e4ac5b7704b5cdb56a7f8b421d0c00a6da Mon Sep 17 00:00:00 2001
From: Pxl <pxl290@qq.com>
Date: Wed, 3 Jan 2024 10:02:35 +0800
Subject: [PATCH 267/299] [Chore](sort) fix block used after it was
 moved(#29416)

/mnt/disk2/pengyu/codebase/apache/doris/be/src/vec/common/sort/heap_sorter.cpp:91:23: error: 'tmp_block' used after it was moved [bugprone-use-after-move,-warnings-as-errors]
---
 be/src/vec/common/sort/heap_sorter.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/be/src/vec/common/sort/heap_sorter.cpp b/be/src/vec/common/sort/heap_sorter.cpp
index d702cbaca20eb6..19de3838301452 100644
--- a/be/src/vec/common/sort/heap_sorter.cpp
+++ b/be/src/vec/common/sort/heap_sorter.cpp
@@ -83,12 +83,11 @@ Status HeapSorter::append_block(Block* block) {
     }
     Block tmp_block = block->clone_empty();
     tmp_block.swap(*block);
+    size_t num_rows = tmp_block.rows();
     HeapSortCursorBlockView block_view_val(std::move(tmp_block), _sort_description);
-    SharedHeapSortCursorBlockView* block_view =
-            new SharedHeapSortCursorBlockView(std::move(block_view_val));
+    auto* block_view = new SharedHeapSortCursorBlockView(std::move(block_view_val));
     block_view->ref();
     Defer defer([&] { block_view->unref(); });
-    size_t num_rows = tmp_block.rows();
     if (_heap_size == _heap->size()) {
         {
             SCOPED_TIMER(_topn_filter_timer);

From 2578f61451ae1208e5414c237fa52e1a956fd987 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <acm_master@163.com>
Date: Wed, 3 Jan 2024 10:20:02 +0800
Subject: [PATCH 268/299] [Chore](ci)Temporarily cancel the mandatory
 restrictions of shellCheck (#29440)

https://github.com/apache/doris/pull/29289
I modified the shell-check attribute, once the PR was combined, I revert this
---
 .asf.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.asf.yaml b/.asf.yaml
index 2648776d4ddbbe..8c7ca098e69381 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -59,7 +59,6 @@ github:
           - BE UT (Doris BE UT)
           - Build Broker
           - Build Documents
-          - ShellCheck
           - clickbench-new (clickbench)
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)

From 83d36927b3a7937262b6fb47e8a2e4b60685482a Mon Sep 17 00:00:00 2001
From: Calvin Kirs <kirs@apache.org>
Date: Wed, 3 Jan 2024 10:39:52 +0800
Subject: [PATCH 269/299] [Choricals] (ci) Temporarily cancel the mandatory
 restrictions on the 2.0 branch shellCheck (#29442)

---
 .asf.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.asf.yaml b/.asf.yaml
index 8c7ca098e69381..c0501eb0d8592e 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -59,6 +59,7 @@ github:
           - BE UT (Doris BE UT)
           - Build Broker
           - Build Documents
+          - ShellCheck          
           - clickbench-new (clickbench)
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)
@@ -94,7 +95,6 @@ github:
           - BE UT (Doris BE UT)
           - Build Broker
           - Build Documents
-          - ShellCheck
           - clickbench-new (clickbench)
           - Build Third Party Libraries (Linux)
           - Build Third Party Libraries (macOS)

From 3a8ae590ecb4ecbfa7c6d0e26cbd75671fb0a4d6 Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Wed, 3 Jan 2024 10:40:44 +0800
Subject: [PATCH 270/299] [fix](pipelineX) Fix unexpected OOM on pipelineX
 (#29436)

---
 be/src/pipeline/exec/aggregation_sink_operator.h |  2 +-
 be/src/pipeline/pipeline_x/dependency.h          |  2 +-
 be/src/pipeline/pipeline_x/operator.cpp          | 10 ++++------
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/be/src/pipeline/exec/aggregation_sink_operator.h b/be/src/pipeline/exec/aggregation_sink_operator.h
index d02e9a0f52ace5..c72cbd092014d1 100644
--- a/be/src/pipeline/exec/aggregation_sink_operator.h
+++ b/be/src/pipeline/exec/aggregation_sink_operator.h
@@ -53,7 +53,7 @@ class AggSinkDependency final : public Dependency {
     ~AggSinkDependency() override = default;
 
     void set_ready() override {
-        if (_is_streaming_agg_state()) {
+        if (_shared_state && _is_streaming_agg_state()) {
             if (((SharedState*)Dependency::_shared_state.get())
                         ->data_queue->has_enough_space_to_push()) {
                 Dependency::set_ready();
diff --git a/be/src/pipeline/pipeline_x/dependency.h b/be/src/pipeline/pipeline_x/dependency.h
index a0504d17886a16..5161424d8d1225 100644
--- a/be/src/pipeline/pipeline_x/dependency.h
+++ b/be/src/pipeline/pipeline_x/dependency.h
@@ -59,7 +59,6 @@ struct BasicSharedState {
     DependencySPtr source_dep = nullptr;
     DependencySPtr sink_dep = nullptr;
 
-    virtual Status close(RuntimeState* state) { return Status::OK(); }
     virtual ~BasicSharedState() = default;
 };
 
@@ -90,6 +89,7 @@ class Dependency : public std::enable_shared_from_this<Dependency> {
     void set_shared_state(std::shared_ptr<BasicSharedState> shared_state) {
         _shared_state = shared_state;
     }
+    void clear_shared_state() { _shared_state.reset(); }
     virtual std::string debug_string(int indentation_level = 0);
 
     // Start the watcher. We use it to count how long this dependency block the current pipeline task.
diff --git a/be/src/pipeline/pipeline_x/operator.cpp b/be/src/pipeline/pipeline_x/operator.cpp
index 7d40e96f38ce78..bc16d70b86ca66 100644
--- a/be/src/pipeline/pipeline_x/operator.cpp
+++ b/be/src/pipeline/pipeline_x/operator.cpp
@@ -376,11 +376,9 @@ Status PipelineXLocalState<DependencyType>::close(RuntimeState* state) {
     if (_closed) {
         return Status::OK();
     }
-    if (_shared_state) {
-        RETURN_IF_ERROR(_shared_state->close(state));
-    }
     if constexpr (!std::is_same_v<DependencyType, FakeDependency>) {
         COUNTER_SET(_wait_for_dependency_timer, _dependency->watcher_elapse_time());
+        _dependency->clear_shared_state();
     }
     if (_rows_returned_counter != nullptr) {
         COUNTER_SET(_rows_returned_counter, _num_rows_returned);
@@ -439,11 +437,11 @@ Status PipelineXSinkLocalState<DependencyType>::close(RuntimeState* state, Statu
     if (_closed) {
         return Status::OK();
     }
-    if (_shared_state) {
-        RETURN_IF_ERROR(_shared_state->close(state));
-    }
     if constexpr (!std::is_same_v<DependencyType, FakeDependency>) {
         COUNTER_SET(_wait_for_dependency_timer, _dependency->watcher_elapse_time());
+        if constexpr (!std::is_same_v<LocalExchangeSinkDependency, DependencyType>) {
+            _dependency->clear_shared_state();
+        }
     }
     if (_peak_memory_usage_counter) {
         _peak_memory_usage_counter->set(_mem_tracker->peak_consumption());

From b3da9bad3e9aefcee20082964ec459c7e2f52537 Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Wed, 3 Jan 2024 11:15:18 +0800
Subject: [PATCH 271/299] [fix](move-memtable)  fix load stream stub fault
 injection test (#29431)

---
 .../test_load_stream_stub_failure_injection.groovy          | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
index ed02d3e7ac1134..7ad2b1e71ebd52 100644
--- a/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_stub_failure_injection.groovy
@@ -81,12 +81,8 @@ suite("test_stream_stub_fault_injection", "nonConcurrent") {
         }
     }
 
-    // StreamSinkFileWriter appendv write segment failed one replica
-    load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_one_replica", "")
-    // StreamSinkFileWriter appendv write segment failed two replica
-    load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_two_replica", "")
     // StreamSinkFileWriter appendv write segment failed all replica
-    load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "stream sink file writer append data failed")
+    load_with_injection("StreamSinkFileWriter.appendv.write_segment_failed_all_replica", "failed to write any replicas")
     // LoadStreams stream wait failed
     load_with_injection("LoadStreamStub._send_with_retry.stream_write_failed", "StreamWrite failed, err=32")
     // LoadStreams keeping stream when release

From 067a9a3a225ed925377873ad105fc5db12a7bfd3 Mon Sep 17 00:00:00 2001
From: qiye <jianliang5669@gmail.com>
Date: Wed, 3 Jan 2024 11:17:48 +0800
Subject: [PATCH 272/299] [opt](ES catalog)Add more description for limitations
 of docvalue_scan (#29420)

> [ignore_above](https://www.elastic.co/guide/en/elasticsearch/reference/current/ignore-above.html)
Do not index any string longer than this value. Defaults to 2147483647 so that all values would be accepted. Please however note that default dynamic mapping rules create a sub keyword field that overrides this default by setting ignore_above: 256.

When the field value is longer than the limitation, we will get an empty result.
The solution is turn off `enable_docvalue_scan`, to get data from `_source` field.
---
 docs/en/docs/lakehouse/multi-catalog/es.md    | 2 ++
 docs/zh-CN/docs/lakehouse/multi-catalog/es.md | 1 +
 2 files changed, 3 insertions(+)

diff --git a/docs/en/docs/lakehouse/multi-catalog/es.md b/docs/en/docs/lakehouse/multi-catalog/es.md
index c6293f931f9c0d..03f926a3f51873 100644
--- a/docs/en/docs/lakehouse/multi-catalog/es.md
+++ b/docs/en/docs/lakehouse/multi-catalog/es.md
@@ -196,6 +196,8 @@ By default, Doris On ES obtains all target columns from `_source`, which is in r
 
 1. Columnar storage is not available for `text` fields in ES. Thus, if you need to obtain fields containing `text` values, you will need to obtain them from `_source`.
 2. When obtaining large numbers of fields (`>= 25`), the performances of `docvalue`  and  `_source` are basically equivalent.
+3. The `keyword` type field, due to the [`ignore_above`](https://www.elastic.co/guide/en/elasticsearch/reference/current/keyword.html#keyword-params) parameter's limit, long text fields exceeding this limit will be ignored, so the result may be empty. In this case, you need to turn off `enable_docvalue_scan` and get the result from `_source`.
+
 
 ### Sniff Keyword Fields
 
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/es.md b/docs/zh-CN/docs/lakehouse/multi-catalog/es.md
index 64a12726f5cc7d..aabfe3dffd5c0b 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/es.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/es.md
@@ -191,6 +191,7 @@ ES Catalog 支持过滤条件的下推: 过滤条件下推给ES，这样只有
 
 1. `text`类型的字段在ES中是没有列式存储，因此如果要获取的字段值有`text`类型字段会自动降级为从`_source`中获取
 2. 在获取的字段数量过多的情况下(`>= 25`)，从`docvalue`中获取字段值的性能会和从`_source`中获取字段值基本一样
+3. `keyword`类型字段由于[`ignore_above`](https://www.elastic.co/guide/en/elasticsearch/reference/current/keyword.html#keyword-params)参数的限制，对于超过该限制的长文本字段会忽略，所以可能会出现结果为空的情况。此时需要关闭`enable_docvalue_scan`，从`_source`中获取结果。
 
 ### 探测keyword类型字段
 

From 14e7eb762430b251869737eeef6ee3ac83ce3707 Mon Sep 17 00:00:00 2001
From: HappenLee <happenlee@hotmail.com>
Date: Wed, 3 Jan 2024 11:21:28 +0800
Subject: [PATCH 273/299] [Opt](rf) Opt broadcast join remote runtime filter
 merge and wait (#29439)

---
 be/src/exprs/runtime_filter.cpp                      | 10 +++++++---
 be/src/exprs/runtime_filter.h                        |  2 +-
 be/src/exprs/runtime_filter_slots.h                  |  6 +++---
 be/src/pipeline/exec/hashjoin_build_sink.cpp         |  2 +-
 be/src/runtime/runtime_filter_mgr.cpp                | 10 +++++-----
 be/src/vec/exec/join/vhash_join_node.cpp             |  2 +-
 .../java/org/apache/doris/planner/RuntimeFilter.java |  4 ++++
 .../main/java/org/apache/doris/qe/Coordinator.java   | 12 ++++++++++--
 8 files changed, 32 insertions(+), 16 deletions(-)

diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp
index 8e8e3dfd8cdb66..f52a9574bf6edd 100644
--- a/be/src/exprs/runtime_filter.cpp
+++ b/be/src/exprs/runtime_filter.cpp
@@ -985,7 +985,7 @@ Status IRuntimeFilter::merge_local_filter(RuntimePredicateWrapper* wrapper, int*
     return Status::OK();
 }
 
-Status IRuntimeFilter::publish() {
+Status IRuntimeFilter::publish(bool publish_local) {
     DCHECK(is_producer());
     if (_is_global) {
         std::vector<IRuntimeFilter*> filters;
@@ -1010,13 +1010,17 @@ Status IRuntimeFilter::publish() {
             filter->update_runtime_filter_type_to_profile();
             filter->signal();
         }
-        return Status::OK();
-    } else {
+    } else if (!publish_local) {
         TNetworkAddress addr;
         DCHECK(_state != nullptr);
         RETURN_IF_ERROR(_state->runtime_filter_mgr->get_merge_addr(&addr));
         return push_to_remote(_state, &addr, _opt_remote_rf);
+    } else {
+        // remote broadcast join only push onetime in build shared hash table
+        // publish_local only set true on copy shared hash table
+        DCHECK(_is_broadcast_join);
     }
+    return Status::OK();
 }
 
 Status IRuntimeFilter::get_push_expr_ctxs(std::list<vectorized::VExprContextSPtr>& probe_ctxs,
diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h
index 32fee99173e78b..2673308ae77fef 100644
--- a/be/src/exprs/runtime_filter.h
+++ b/be/src/exprs/runtime_filter.h
@@ -228,7 +228,7 @@ class IRuntimeFilter {
 
     // publish filter
     // push filter to remote node or push down it to scan_node
-    Status publish();
+    Status publish(bool publish_local = false);
 
     RuntimeFilterType type() const { return _runtime_filter_type; }
 
diff --git a/be/src/exprs/runtime_filter_slots.h b/be/src/exprs/runtime_filter_slots.h
index 7223b652a3c71e..d539e295ae88cc 100644
--- a/be/src/exprs/runtime_filter_slots.h
+++ b/be/src/exprs/runtime_filter_slots.h
@@ -204,10 +204,10 @@ class VRuntimeFilterSlots {
     }
 
     // publish runtime filter
-    Status publish() {
+    Status publish(bool publish_local = false) {
         for (auto& pair : _runtime_filters) {
-            for (auto filter : pair.second) {
-                RETURN_IF_ERROR(filter->publish());
+            for (auto& filter : pair.second) {
+                RETURN_IF_ERROR(filter->publish(publish_local));
             }
         }
         return Status::OK();
diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 8c1ff852433e14..3d9fd501917985 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -568,7 +568,7 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
                                 RETURN_IF_ERROR(
                                         local_state._runtime_filter_slots->copy_from_shared_context(
                                                 _shared_hash_table_context));
-                                RETURN_IF_ERROR(local_state._runtime_filter_slots->publish());
+                                RETURN_IF_ERROR(local_state._runtime_filter_slots->publish(true));
                                 return Status::OK();
                             }},
                     *local_state._shared_state->hash_table_variants);
diff --git a/be/src/runtime/runtime_filter_mgr.cpp b/be/src/runtime/runtime_filter_mgr.cpp
index b67bc5ffd3b461..73a043070e35b9 100644
--- a/be/src/runtime/runtime_filter_mgr.cpp
+++ b/be/src/runtime/runtime_filter_mgr.cpp
@@ -328,6 +328,10 @@ Status RuntimeFilterMergeControllerEntity::merge(const PMergeFilterRequest* requ
     cntVal = iter->second.first;
     {
         std::lock_guard<std::mutex> l(*iter->second.second);
+        // Skip the other broadcast join runtime filter
+        if (cntVal->arrive_id.size() == 1 && cntVal->runtime_filter_desc.is_broadcast_join) {
+            return Status::OK();
+        }
         MergeRuntimeFilterParams params(request, attach_data);
         ObjectPool* pool = cntVal->pool.get();
         RuntimeFilterWrapperHolder holder;
@@ -339,11 +343,7 @@ Status RuntimeFilterMergeControllerEntity::merge(const PMergeFilterRequest* requ
         VLOG_ROW << "merge size:" << merged_size << ":" << cntVal->producer_size;
         DCHECK_LE(merged_size, cntVal->producer_size);
         cntVal->merge_time += (MonotonicMillis() - start_merge);
-        if (merged_size < cntVal->producer_size) {
-            return Status::OK();
-        } else {
-            merge_time = cntVal->merge_time;
-        }
+        merge_time = cntVal->merge_time;
     }
 
     if (merged_size == cntVal->producer_size) {
diff --git a/be/src/vec/exec/join/vhash_join_node.cpp b/be/src/vec/exec/join/vhash_join_node.cpp
index 60391716e28af6..b135ada7237b0c 100644
--- a/be/src/vec/exec/join/vhash_join_node.cpp
+++ b/be/src/vec/exec/join/vhash_join_node.cpp
@@ -824,7 +824,7 @@ Status HashJoinNode::sink(doris::RuntimeState* state, vectorized::Block* in_bloc
                                           state, arg.hash_table->size()));
                                   RETURN_IF_ERROR(_runtime_filter_slots->copy_from_shared_context(
                                           _shared_hash_table_context));
-                                  RETURN_IF_ERROR(_runtime_filter_slots->publish());
+                                  RETURN_IF_ERROR(_runtime_filter_slots->publish(true));
                                   return Status::OK();
                               }},
                     *_hash_table_variants);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
index d21f390c04dab1..646a07221a7193 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java
@@ -604,6 +604,10 @@ public void setIsBroadcast(boolean isBroadcast) {
         isBroadcastJoin = isBroadcast;
     }
 
+    public boolean isBroadcast() {
+        return isBroadcastJoin;
+    }
+
     public void computeNdvEstimate() {
         if (ndvEstimate < 0) {
             ndvEstimate = builderNode.getChild(1).getCardinalityAfterFilter();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 421fa1ac5238e9..9bfca2802ebe47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -3627,6 +3627,9 @@ List<TExecPlanFragmentParams> toThrift(int backendNum) {
                 params.params.setRuntimeFilterParams(new TRuntimeFilterParams());
                 params.params.runtime_filter_params.setRuntimeFilterMergeAddr(runtimeFilterMergeAddr);
                 if (instanceExecParam.instanceId.equals(runtimeFilterMergeInstanceId)) {
+                    Set<Integer> broadCastRf = assignedRuntimeFilters.stream().filter(RuntimeFilter::isBroadcast)
+                            .map(r -> r.getFilterId().asInt()).collect(Collectors.toSet());
+
                     for (RuntimeFilter rf : assignedRuntimeFilters) {
                         if (!ridToTargetParam.containsKey(rf.getFilterId())) {
                             continue;
@@ -3668,7 +3671,8 @@ List<TExecPlanFragmentParams> toThrift(int backendNum) {
                     }
                     for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
                         params.params.runtime_filter_params.putToRuntimeFilterBuilderNum(
-                                entry.getKey().asInt(), entry.getValue());
+                                entry.getKey().asInt(), broadCastRf.contains(entry.getKey().asInt())
+                                        ? 1 : entry.getValue());
                     }
                     for (RuntimeFilter rf : assignedRuntimeFilters) {
                         params.params.runtime_filter_params.putToRidToRuntimeFilter(
@@ -3753,6 +3757,9 @@ Map<TNetworkAddress, TPipelineFragmentParams> toTPipelineParams(int backendNum)
                 localParams.setRuntimeFilterParams(new TRuntimeFilterParams());
                 localParams.runtime_filter_params.setRuntimeFilterMergeAddr(runtimeFilterMergeAddr);
                 if (instanceExecParam.instanceId.equals(runtimeFilterMergeInstanceId)) {
+                    Set<Integer> broadCastRf = assignedRuntimeFilters.stream().filter(RuntimeFilter::isBroadcast)
+                            .map(r -> r.getFilterId().asInt()).collect(Collectors.toSet());
+
                     for (RuntimeFilter rf : assignedRuntimeFilters) {
                         if (!ridToTargetParam.containsKey(rf.getFilterId())) {
                             continue;
@@ -3795,7 +3802,8 @@ Map<TNetworkAddress, TPipelineFragmentParams> toTPipelineParams(int backendNum)
                     }
                     for (Map.Entry<RuntimeFilterId, Integer> entry : ridToBuilderNum.entrySet()) {
                         localParams.runtime_filter_params.putToRuntimeFilterBuilderNum(
-                                entry.getKey().asInt(), entry.getValue());
+                                entry.getKey().asInt(), broadCastRf.contains(entry.getKey().asInt()) ? 1 :
+                                        entry.getValue());
                     }
                     for (RuntimeFilter rf : assignedRuntimeFilters) {
                         localParams.runtime_filter_params.putToRidToRuntimeFilter(

From be1d9c335891a57b124a8fde55a96af992886ae7 Mon Sep 17 00:00:00 2001
From: Xinyi Zou <zouxinyi02@gmail.com>
Date: Wed, 3 Jan 2024 11:25:00 +0800
Subject: [PATCH 274/299] [fix](memory) Fix mem tracker web page notice #29361

---
 be/src/http/default_path_handlers.cpp         | 26 ++++++++++++-------
 be/src/runtime/memory/mem_tracker_limiter.cpp |  8 +++---
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp
index b997a3466c9cac..39a75b7ccab25c 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -160,15 +160,23 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
                                                    MemTrackerLimiter::Type::EXPERIMENTAL);
         }
     } else {
-        (*output) << "<h4>*Note: (see documentation for details)</h4>\n";
-        (*output) << "<h4>     1.`/mem_tracker?type=global` to view the memory statistics of each "
-                     "type, `global`life cycle is the same as the process, e.g. each Cache, "
-                     "StorageEngine, each Manager.</h4>\n";
-        (*output) << "<h4>     2.`/mem_tracker` counts virtual memory, which is equal to `Actual "
-                     "memory used` in `/memz`</h4>\n";
-        (*output) << "<h4>     3.`process` is equal to the sum of all types of memory, "
-                     "`/mem_tracker` can be logically divided into 4 layers: 1)`process` 2)`type` "
-                     "3)`query/load/compation task etc.` 4)`exec node etc.`</h4>\n";
+        (*output) << "<h4>*Notice:</h4>\n";
+        (*output) << "<h4>    1. MemTracker only counts the memory on part of the main execution "
+                     "path, "
+                     "which is usually less than the real process memory.</h4>\n";
+        (*output) << "<h4>    2. each `type` is the sum of a set of tracker values, "
+                     "`sum of all trackers` is the sum of all trackers of all types, .</h4>\n";
+        (*output) << "<h4>    3. `process resident memory` is the physical memory of the process, "
+                     "from /proc VmRSS VmHWM.</h4>\n";
+        (*output) << "<h4>    4. `process virtual memory` is the virtual memory of the process, "
+                     "from /proc VmSize VmPeak.</h4>\n";
+        (*output) << "<h4>    5.`/mem_tracker?type=<type name>` to view the memory details of each "
+                     "type, for example, `/mem_tracker?type=query` will list the memory of all "
+                     "queries; "
+                     "`/mem_tracker?type=global` will list the memory of all Cache, metadata and "
+                     "other "
+                     "global life cycles.</h4>\n";
+        (*output) << "<h4>see documentation for details.";
         MemTrackerLimiter::make_process_snapshots(&snapshots);
     }
 
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 60ad66bfbdd858..a4a8d61928dee2 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -148,7 +148,7 @@ void MemTrackerLimiter::make_process_snapshots(std::vector<MemTracker::Snapshot>
         all_tracker_mem_sum += it.second->current_value();
     }
 
-    snapshot.type = "tc/jemalloc_free_memory";
+    snapshot.type = "tc/jemalloc cache";
     snapshot.label = "";
     snapshot.limit = -1;
     snapshot.cur_consumption = MemInfo::allocator_cache_mem();
@@ -156,21 +156,21 @@ void MemTrackerLimiter::make_process_snapshots(std::vector<MemTracker::Snapshot>
     (*snapshots).emplace_back(snapshot);
     all_tracker_mem_sum += MemInfo::allocator_cache_mem();
 
-    snapshot.type = "all_tracker_sum (is virtual memory)";
+    snapshot.type = "sum of all trackers"; // is virtual memory
     snapshot.label = "";
     snapshot.limit = -1;
     snapshot.cur_consumption = all_tracker_mem_sum;
     snapshot.peak_consumption = -1;
     (*snapshots).emplace_back(snapshot);
 
-    snapshot.type = "process resident memory (from /proc VmRSS VmHWM)";
+    snapshot.type = "process resident memory"; // from /proc VmRSS VmHWM
     snapshot.label = "";
     snapshot.limit = -1;
     snapshot.cur_consumption = PerfCounters::get_vm_rss();
     snapshot.peak_consumption = PerfCounters::get_vm_hwm();
     (*snapshots).emplace_back(snapshot);
 
-    snapshot.type = "process virtual memory (from /proc VmSize VmPeak)";
+    snapshot.type = "process virtual memory"; // from /proc VmSize VmPeak
     snapshot.label = "";
     snapshot.limit = -1;
     snapshot.cur_consumption = PerfCounters::get_vm_size();

From 1e8bb75182587e53c9f3d2abfd76168d7ee6c232 Mon Sep 17 00:00:00 2001
From: Kaijie Chen <ckj@apache.org>
Date: Wed, 3 Jan 2024 11:26:26 +0800
Subject: [PATCH 275/299] [improve](move-memtable) add log on idle timeout
 (#29438)

---
 be/src/runtime/load_stream.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index c29374c51bafd7..3aad2566575511 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -547,6 +547,7 @@ void LoadStream::_dispatch(StreamId id, const PStreamHeader& hdr, butil::IOBuf*
 }
 
 void LoadStream::on_idle_timeout(StreamId id) {
+    LOG(WARNING) << "closing load stream on idle timeout, load_id=" << print_id(_load_id);
     brpc::StreamClose(id);
 }
 

From 08353f6027890bf95338afbff95156c3f270023a Mon Sep 17 00:00:00 2001
From: DuRipeng <453243496@qq.com>
Date: Wed, 3 Jan 2024 11:29:12 +0800
Subject: [PATCH 276/299] [Enhance](fe) Iceberg table in HMS catalog supports
 broker scan (#28107)

My organization uses HMS catalog to accelerate Lake query. Sine we have custom distributed file system and hard to integrate to FE / BE, we introduce HMS Catalog broker scan support (#24830) and implement custom distributed file system adaption in broker.

We want to expand the scope of use to Iceberg table scan in HMS Catalog. This PR introduces broker-scan-related `IcebergBrokerIO`, `BrokerInputFile`, `BrokerInputStream` for Iceberg table scan
---
 docs/en/docs/lakehouse/multi-catalog/hive.md  |   7 +
 .../docs/lakehouse/multi-catalog/hive.md      |   6 +
 .../iceberg/broker/BrokerInputFile.java       |  74 ++++++++
 .../iceberg/broker/BrokerInputStream.java     | 169 ++++++++++++++++++
 .../iceberg/broker/IcebergBrokerIO.java       |  80 +++++++++
 .../planner/external/FileQueryScanNode.java   |  10 +-
 .../iceberg/IcebergMetadataCache.java         |  17 +-
 7 files changed, 355 insertions(+), 8 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputFile.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputStream.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/IcebergBrokerIO.java

diff --git a/docs/en/docs/lakehouse/multi-catalog/hive.md b/docs/en/docs/lakehouse/multi-catalog/hive.md
index 25fddea125064d..01c78d3599f8f4 100644
--- a/docs/en/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/en/docs/lakehouse/multi-catalog/hive.md
@@ -391,6 +391,13 @@ Add following setting when creating an HMS catalog, file splitting and scanning
 "broker.name" = "test_broker"
 ```
 
+
+Doris has implemented Broker query support for HMS Catalog Iceberg based on the Iceberg `FileIO` interface. If needed, the following configuration can be added when creating the HMS Catalog.
+
+```sql
+"io-impl" = "org.apache.doris.datasource.iceberg.broker.IcebergBrokerIO"
+```
+
 ## Integrate with Apache Ranger
 
 Apache Ranger is a security framework for monitoring, enabling services, and comprehensive data security access management on the Hadoop platform.
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
index e75977c25f4766..b4efb291f89ed6 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
@@ -373,6 +373,12 @@ CREATE CATALOG hive PROPERTIES (
 "broker.name" = "test_broker"
 ```
 
+Doris 基于 Iceberg `FileIO` 接口实现了 Broker 查询 HMS Catalog Iceberg 的支持。如有需求，可以在创建 HMS Catalog 时增加如下配置。
+
+```sql
+"io-impl" = "org.apache.doris.datasource.iceberg.broker.IcebergBrokerIO"
+```
+
 ## 使用 Ranger 进行权限校验
 
 Apache Ranger是一个用来在Hadoop平台上进行监控，启用服务，以及全方位数据安全访问管理的安全框架。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputFile.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputFile.java
new file mode 100644
index 00000000000000..529df6c0af10a2
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputFile.java
@@ -0,0 +1,74 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.broker;
+
+import org.apache.doris.analysis.BrokerDesc;
+import org.apache.doris.common.util.BrokerReader;
+import org.apache.doris.thrift.TBrokerFD;
+
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.SeekableInputStream;
+
+import java.io.IOException;
+
+public class BrokerInputFile implements InputFile {
+
+    private Long fileLength = null;
+
+    private final String filePath;
+    private final BrokerDesc brokerDesc;
+    private BrokerReader reader;
+    private TBrokerFD fd;
+
+    private BrokerInputFile(String filePath, BrokerDesc brokerDesc) {
+        this.filePath = filePath;
+        this.brokerDesc = brokerDesc;
+    }
+
+    private void init() throws IOException {
+        this.reader = BrokerReader.create(this.brokerDesc);
+        this.fileLength = this.reader.getFileLength(filePath);
+        this.fd = this.reader.open(filePath);
+    }
+
+    public static BrokerInputFile create(String filePath, BrokerDesc brokerDesc) throws IOException {
+        BrokerInputFile inputFile = new BrokerInputFile(filePath, brokerDesc);
+        inputFile.init();
+        return inputFile;
+    }
+
+    @Override
+    public long getLength() {
+        return fileLength;
+    }
+
+    @Override
+    public SeekableInputStream newStream() {
+        return new BrokerInputStream(this.reader, this.fd, this.fileLength);
+    }
+
+    @Override
+    public String location() {
+        return filePath;
+    }
+
+    @Override
+    public boolean exists() {
+        return fileLength != null;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputStream.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputStream.java
new file mode 100644
index 00000000000000..b9f6d30aed4fd3
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/BrokerInputStream.java
@@ -0,0 +1,169 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.broker;
+
+import org.apache.doris.common.util.BrokerReader;
+import org.apache.doris.thrift.TBrokerFD;
+
+import org.apache.iceberg.io.SeekableInputStream;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.IOException;
+
+public class BrokerInputStream extends SeekableInputStream {
+    private static final Logger LOG = LogManager.getLogger(BrokerInputStream.class);
+    private static final int COPY_BUFFER_SIZE = 1024 * 1024; // 1MB
+
+    private final byte[] tmpBuf = new byte[COPY_BUFFER_SIZE];
+    private long currentPos = 0;
+    private long markPos = 0;
+
+    private long bufferOffset = 0;
+    private long bufferLimit = 0;
+    private final BrokerReader reader;
+    private final TBrokerFD fd;
+    private final long fileLength;
+
+    public BrokerInputStream(BrokerReader reader, TBrokerFD fd, long fileLength) {
+        this.fd = fd;
+        this.reader = reader;
+        this.fileLength = fileLength;
+    }
+
+    @Override
+    public long getPos() throws IOException {
+        return currentPos;
+    }
+
+    @Override
+    public void seek(long newPos) throws IOException {
+        currentPos = newPos;
+    }
+
+    @Override
+    public int read() throws IOException {
+        try {
+            if (currentPos < bufferOffset || currentPos > bufferLimit || bufferOffset >= bufferLimit) {
+                bufferOffset = currentPos;
+                fill();
+            }
+            if (currentPos > bufferLimit) {
+                LOG.warn("current pos {} is larger than buffer limit {}."
+                        + " should not happen.", currentPos, bufferLimit);
+                return -1;
+            }
+
+            int pos = (int) (currentPos - bufferOffset);
+            int res = Byte.toUnsignedInt(tmpBuf[pos]);
+            ++currentPos;
+            return res;
+        } catch (BrokerReader.EOFException e) {
+            return -1;
+        }
+    }
+
+    @SuppressWarnings("NullableProblems")
+    @Override
+    public int read(byte[] b) throws IOException {
+        try {
+            byte[] data = reader.pread(fd, currentPos, b.length);
+            System.arraycopy(data, 0, b, 0, data.length);
+            currentPos += data.length;
+            return data.length;
+        } catch (BrokerReader.EOFException e) {
+            return -1;
+        }
+    }
+
+    @SuppressWarnings("NullableProblems")
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        try {
+            if (currentPos < bufferOffset || currentPos > bufferLimit || currentPos + len > bufferLimit) {
+                if (len > COPY_BUFFER_SIZE) {
+                    // the data to be read is larger then max size of buffer.
+                    // read it directly.
+                    byte[] data = reader.pread(fd, currentPos, len);
+                    System.arraycopy(data, 0, b, off, data.length);
+                    currentPos += data.length;
+                    return data.length;
+                }
+                // fill the buffer first
+                bufferOffset = currentPos;
+                fill();
+            }
+
+            if (currentPos > bufferLimit) {
+                LOG.warn("current pos {} is larger than buffer limit {}."
+                        + " should not happen.", currentPos, bufferLimit);
+                return -1;
+            }
+
+            int start = (int) (currentPos - bufferOffset);
+            int readLen = Math.min(len, (int) (bufferLimit - bufferOffset));
+            System.arraycopy(tmpBuf, start, b, off, readLen);
+            currentPos += readLen;
+            return readLen;
+        } catch (BrokerReader.EOFException e) {
+            return -1;
+        }
+    }
+
+    private void fill() throws IOException, BrokerReader.EOFException {
+        if (bufferOffset == this.fileLength) {
+            throw new BrokerReader.EOFException();
+        }
+        byte[] data = reader.pread(fd, bufferOffset, COPY_BUFFER_SIZE);
+        System.arraycopy(data, 0, tmpBuf, 0, data.length);
+        bufferLimit = bufferOffset + data.length;
+    }
+
+    @Override
+    public long skip(long n) throws IOException {
+        final long left = fileLength - currentPos;
+        long min = Math.min(n, left);
+        currentPos += min;
+        return min;
+    }
+
+    @Override
+    public int available() throws IOException {
+        return 0;
+    }
+
+    @Override
+    public void close() throws IOException {
+        reader.close(fd);
+    }
+
+    @Override
+    public synchronized void mark(int readlimit) {
+        markPos = currentPos;
+    }
+
+    @Override
+    public synchronized void reset() throws IOException {
+        currentPos = markPos;
+    }
+
+    @Override
+    public boolean markSupported() {
+        return true;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/IcebergBrokerIO.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/IcebergBrokerIO.java
new file mode 100644
index 00000000000000..ec3b4f170301b1
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/broker/IcebergBrokerIO.java
@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.iceberg.broker;
+
+import org.apache.doris.analysis.BrokerDesc;
+import org.apache.doris.datasource.HMSExternalCatalog;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.io.OutputFile;
+import org.apache.iceberg.util.SerializableMap;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * FileIO implementation that uses broker to execute Iceberg files IO operation.
+ */
+public class IcebergBrokerIO implements FileIO {
+
+    private SerializableMap<String, String> properties = SerializableMap.copyOf(ImmutableMap.of());
+    private BrokerDesc brokerDesc = null;
+
+    @Override
+    public void initialize(Map<String, String> props) {
+        this.properties = SerializableMap.copyOf(props);
+        if (!properties.containsKey(HMSExternalCatalog.BIND_BROKER_NAME)) {
+            throw new UnsupportedOperationException(String.format("No broker is specified, "
+                    + "try to set '%s' in HMS Catalog", HMSExternalCatalog.BIND_BROKER_NAME));
+        }
+        String brokerName = properties.get(HMSExternalCatalog.BIND_BROKER_NAME);
+        this.brokerDesc = new BrokerDesc(brokerName, properties.immutableMap());
+    }
+
+    @Override
+    public Map<String, String> properties() {
+        return properties.immutableMap();
+    }
+
+    @Override
+    public InputFile newInputFile(String path) {
+        if (brokerDesc == null) {
+            throw new UnsupportedOperationException("IcebergBrokerIO should be initialized first");
+        }
+        try {
+            return BrokerInputFile.create(path, brokerDesc);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    @Override
+    public OutputFile newOutputFile(String path) {
+        throw new UnsupportedOperationException("IcebergBrokerIO does not support writing files");
+    }
+
+    @Override
+    public void deleteFile(String path) {
+        throw new UnsupportedOperationException("IcebergBrokerIO does not support deleting files");
+    }
+
+    @Override
+    public void close() { }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
index 9bae707c2f3ed8..b54b05f47e194a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FileQueryScanNode.java
@@ -34,10 +34,12 @@
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.BrokerUtil;
 import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.HMSExternalCatalog;
 import org.apache.doris.datasource.hive.AcidInfo;
 import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
 import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
 import org.apache.doris.planner.PlanNodeId;
+import org.apache.doris.planner.external.iceberg.IcebergSplit;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.spi.Split;
 import org.apache.doris.statistics.StatisticalType;
@@ -313,7 +315,13 @@ public void createScanRangeLocations() throws UserException {
         List<String> pathPartitionKeys = getPathPartitionKeys();
         for (Split split : inputSplits) {
             FileSplit fileSplit = (FileSplit) split;
-            TFileType locationType = getLocationType(fileSplit.getPath().toString());
+            TFileType locationType;
+            if (fileSplit instanceof IcebergSplit
+                    && ((IcebergSplit) fileSplit).getConfig().containsKey(HMSExternalCatalog.BIND_BROKER_NAME)) {
+                locationType = TFileType.FILE_BROKER;
+            } else {
+                locationType = getLocationType(fileSplit.getPath().toString());
+            }
 
             TScanRangeLocations curLocations = newLocations();
             // If fileSplit has partition values, use the values collected from hive partitions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
index 5f79623ff4ad9a..91a208202d05e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergMetadataCache.java
@@ -201,15 +201,18 @@ private Table createIcebergTable(String uri, Map<String, String> hdfsConf, Strin
         HiveCatalog hiveCatalog = new HiveCatalog();
         hiveCatalog.setConf(conf);
 
-        Map<String, String> catalogProperties = new HashMap<>();
-        catalogProperties.put(HMSProperties.HIVE_METASTORE_URIS, uri);
-        catalogProperties.put("uri", uri);
-        hiveCatalog.initialize("hive", catalogProperties);
-
+        if (props.containsKey(HMSExternalCatalog.BIND_BROKER_NAME)) {
+            props.put(HMSProperties.HIVE_METASTORE_URIS, uri);
+            props.put("uri", uri);
+            hiveCatalog.initialize("hive", props);
+        } else {
+            Map<String, String> catalogProperties = new HashMap<>();
+            catalogProperties.put(HMSProperties.HIVE_METASTORE_URIS, uri);
+            catalogProperties.put("uri", uri);
+            hiveCatalog.initialize("hive", catalogProperties);
+        }
         Table table = HiveMetaStoreClientHelper.ugiDoAs(conf, () -> hiveCatalog.loadTable(TableIdentifier.of(db, tbl)));
-
         initIcebergTableFileIO(table, props);
-
         return table;
     }
 

From fab1a627fcd07303f7bb30c168d3713bd1a6e07a Mon Sep 17 00:00:00 2001
From: xy <whuxingying@163.com>
Date: Wed, 3 Jan 2024 11:44:02 +0800
Subject: [PATCH 277/299] [fix](scan) _insert_data_normal should catch
 exception when BlockReader::_unique_key_next_block (#29426)

Co-authored-by: xingying01 <xingying01@corp.netease.com>
---
 be/src/vec/olap/block_reader.cpp | 23 ++++++++++++++---------
 be/src/vec/olap/block_reader.h   |  2 +-
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp
index 82604ac4083307..7073d880f38cb2 100644
--- a/be/src/vec/olap/block_reader.cpp
+++ b/be/src/vec/olap/block_reader.cpp
@@ -301,8 +301,7 @@ Status BlockReader::_agg_key_next_block(Block* block, bool* eof) {
     auto target_block_row = 0;
     auto merged_row = 0;
     auto target_columns = block->mutate_columns();
-
-    _insert_data_normal(target_columns);
+    RETURN_IF_ERROR(_insert_data_normal(target_columns));
     target_block_row++;
     _append_agg_data(target_columns);
 
@@ -325,7 +324,8 @@ Status BlockReader::_agg_key_next_block(Block* block, bool* eof) {
             _agg_data_counters.push_back(_last_agg_data_counter);
             _last_agg_data_counter = 0;
 
-            _insert_data_normal(target_columns);
+            RETURN_IF_ERROR(_insert_data_normal(target_columns));
+
             target_block_row++;
         } else {
             merged_row++;
@@ -356,7 +356,8 @@ Status BlockReader::_unique_key_next_block(Block* block, bool* eof) {
     }
 
     do {
-        _insert_data_normal(target_columns);
+        RETURN_IF_ERROR(_insert_data_normal(target_columns));
+
         if (UNLIKELY(_reader_context.record_rowids)) {
             _block_row_locations[target_block_row] = _vcollect_iter.current_row_location();
         }
@@ -429,12 +430,16 @@ Status BlockReader::_unique_key_next_block(Block* block, bool* eof) {
     return Status::OK();
 }
 
-void BlockReader::_insert_data_normal(MutableColumns& columns) {
+Status BlockReader::_insert_data_normal(MutableColumns& columns) {
     auto block = _next_row.block.get();
-    for (auto idx : _normal_columns_idx) {
-        columns[_return_columns_loc[idx]]->insert_from(*block->get_by_position(idx).column,
-                                                       _next_row.row_pos);
-    }
+
+    RETURN_IF_CATCH_EXCEPTION({
+        for (auto idx : _normal_columns_idx) {
+            columns[_return_columns_loc[idx]]->insert_from(*block->get_by_position(idx).column,
+                                                           _next_row.row_pos);
+        }
+    });
+    return Status::OK();
 }
 
 void BlockReader::_append_agg_data(MutableColumns& columns) {
diff --git a/be/src/vec/olap/block_reader.h b/be/src/vec/olap/block_reader.h
index ed6d1ba02284e2..f21909315a4724 100644
--- a/be/src/vec/olap/block_reader.h
+++ b/be/src/vec/olap/block_reader.h
@@ -74,7 +74,7 @@ class BlockReader final : public TabletReader {
 
     void _init_agg_state(const ReaderParams& read_params);
 
-    void _insert_data_normal(MutableColumns& columns);
+    Status _insert_data_normal(MutableColumns& columns);
 
     void _append_agg_data(MutableColumns& columns);
 

From 5eb38301ddc87c9ae91c73b444536fe29e910d85 Mon Sep 17 00:00:00 2001
From: Calvin Kirs <kirs@apache.org>
Date: Wed, 3 Jan 2024 11:44:38 +0800
Subject: [PATCH 278/299] [Chore](CI)Re-configure branch-2.0 shell-check as a
 required check (#29448)

https://github.com/apache/doris/pull/29289 done
---
 .asf.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.asf.yaml b/.asf.yaml
index c0501eb0d8592e..17e2d1c406b182 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -94,6 +94,7 @@ github:
           - FE UT (Doris FE UT)
           - BE UT (Doris BE UT)
           - Build Broker
+          - ShellCheck
           - Build Documents
           - clickbench-new (clickbench)
           - Build Third Party Libraries (Linux)

From c0db8533afbf70fbbcc3a4904c34a7dc273ab7c4 Mon Sep 17 00:00:00 2001
From: zclllyybb <zhaochangle@selectdb.com>
Date: Wed, 3 Jan 2024 11:53:09 +0800
Subject: [PATCH 279/299] [fix](load) fix single replica load with auto
 partition

---
 be/src/vec/sink/writer/vtablet_writer.cpp                      | 3 +++
 .../src/main/java/org/apache/doris/planner/OlapTableSink.java  | 1 +
 2 files changed, 4 insertions(+)

diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp
index 89519ad6ca0ff8..7ceb7f16d3dd86 100644
--- a/be/src/vec/sink/writer/vtablet_writer.cpp
+++ b/be/src/vec/sink/writer/vtablet_writer.cpp
@@ -1059,6 +1059,9 @@ Status VTabletWriter::on_partitions_created(TCreatePartitionResult* result) {
     // add new tablet locations. it will use by address. so add to pool
     auto* new_locations = _pool->add(new std::vector<TTabletLocation>(result->tablets));
     _location->add_locations(*new_locations);
+    if (_write_single_replica) {
+        _slave_location->add_locations(*new_locations);
+    }
 
     // update new node info
     _nodes_info->add_nodes(result->nodes);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
index f85a621c967ffa..485fde67fb1de2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java
@@ -527,6 +527,7 @@ private List<TOlapTableLocationParam> createLocation(OlapTable table) throws Use
         // for partition by function expr, there is no any partition firstly, But this is required in thrift struct.
         if (partitionIds.isEmpty()) {
             locationParam.setTablets(new ArrayList<TTabletLocation>());
+            slaveLocationParam.setTablets(new ArrayList<TTabletLocation>());
         }
         // check if disk capacity reach limit
         // this is for load process, so use high water mark to check

From dbf61005dff8b1242ef47b8d8673bbe1c4e35091 Mon Sep 17 00:00:00 2001
From: Kang <kxiao.tiger@gmail.com>
Date: Wed, 3 Jan 2024 12:12:43 +0800
Subject: [PATCH 280/299] change boost thirdparty url to offical
 archives.boost.io (#29401)

---
 thirdparty/vars.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 7e2d5bf9dc680d..4d65efe5b45417 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -174,7 +174,7 @@ RAGEL_SOURCE=ragel-6.10
 RAGEL_MD5SUM="748cae8b50cffe9efcaa5acebc6abf0d"
 
 # boost
-BOOST_DOWNLOAD="https://boostorg.jfrog.io/artifactory/main/release/1.81.0/source/boost_1_81_0.tar.gz"
+BOOST_DOWNLOAD="https://archives.boost.io/release/1.81.0/source/boost_1_81_0.tar.gz"
 BOOST_NAME=boost_1_81_0.tar.gz
 BOOST_SOURCE=boost_1_81_0
 BOOST_MD5SUM="4bf02e84afb56dfdccd1e6aec9911f4b"

From 1fbbff32b216b477867103921c9c611038f108d4 Mon Sep 17 00:00:00 2001
From: Jerry Hu <mrhhsg@gmail.com>
Date: Wed, 3 Jan 2024 12:40:41 +0800
Subject: [PATCH 281/299] [fix](pipelinex) coredump caused by
 VRuntimeFilterSlots::_is_global was not set (#29446)

---
 be/src/pipeline/exec/hashjoin_build_sink.cpp | 17 ++++++++++++++---
 be/src/vec/exec/join/vhash_join_node.h       |  5 +++--
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/be/src/pipeline/exec/hashjoin_build_sink.cpp b/be/src/pipeline/exec/hashjoin_build_sink.cpp
index 3d9fd501917985..98ecafdc5a83a8 100644
--- a/be/src/pipeline/exec/hashjoin_build_sink.cpp
+++ b/be/src/pipeline/exec/hashjoin_build_sink.cpp
@@ -491,6 +491,9 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
                        (*local_state._shared_state->build_block).bytes());
         RETURN_IF_ERROR(
                 local_state.process_build_block(state, (*local_state._shared_state->build_block)));
+
+        const bool use_global_rf =
+                local_state._parent->cast<HashJoinBuildSinkOperatorX>()._use_global_rf;
         auto ret = std::visit(
                 Overload {[&](std::monostate&) -> Status {
                               LOG(FATAL) << "FATAL: uninited hash table";
@@ -498,7 +501,8 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
                           },
                           [&](auto&& arg) -> Status {
                               vectorized::ProcessRuntimeFilterBuild runtime_filter_build_process;
-                              return runtime_filter_build_process(state, arg, &local_state);
+                              return runtime_filter_build_process(state, arg, &local_state,
+                                                                  use_global_rf);
                           }},
                 *local_state._shared_state->hash_table_variants);
         if (!ret.ok()) {
@@ -528,6 +532,10 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
         DCHECK(_shared_hash_table_context != nullptr);
         CHECK(_shared_hash_table_context->signaled);
 
+        if (!_shared_hash_table_context->status.ok()) {
+            return _shared_hash_table_context->status;
+        }
+
         local_state.profile()->add_info_string(
                 "SharedHashTableFrom",
                 print_id(
@@ -547,6 +555,8 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
                         _shared_hash_table_context->hash_table_variants));
 
         local_state._shared_state->build_block = _shared_hash_table_context->block;
+        const bool use_global_rf =
+                local_state._parent->cast<HashJoinBuildSinkOperatorX>()._use_global_rf;
 
         if (!_shared_hash_table_context->runtime_filters.empty()) {
             auto ret = std::visit(
@@ -560,8 +570,9 @@ Status HashJoinBuildSinkOperatorX::sink(RuntimeState* state, vectorized::Block*
                                     return Status::OK();
                                 }
                                 local_state._runtime_filter_slots =
-                                        std::make_shared<VRuntimeFilterSlots>(
-                                                _build_expr_ctxs, _runtime_filter_descs);
+                                        std::make_shared<VRuntimeFilterSlots>(_build_expr_ctxs,
+                                                                              _runtime_filter_descs,
+                                                                              use_global_rf);
 
                                 RETURN_IF_ERROR(local_state._runtime_filter_slots->init(
                                         state, arg.hash_table->size()));
diff --git a/be/src/vec/exec/join/vhash_join_node.h b/be/src/vec/exec/join/vhash_join_node.h
index 8304eedb2901d5..35d97ccb03380d 100644
--- a/be/src/vec/exec/join/vhash_join_node.h
+++ b/be/src/vec/exec/join/vhash_join_node.h
@@ -76,12 +76,13 @@ class HashJoinNode;
 
 struct ProcessRuntimeFilterBuild {
     template <class HashTableContext, typename Parent>
-    Status operator()(RuntimeState* state, HashTableContext& hash_table_ctx, Parent* parent) {
+    Status operator()(RuntimeState* state, HashTableContext& hash_table_ctx, Parent* parent,
+                      bool is_global = false) {
         if (parent->runtime_filter_descs().empty()) {
             return Status::OK();
         }
         parent->_runtime_filter_slots = std::make_shared<VRuntimeFilterSlots>(
-                parent->_build_expr_ctxs, parent->runtime_filter_descs());
+                parent->_build_expr_ctxs, parent->runtime_filter_descs(), is_global);
 
         RETURN_IF_ERROR(
                 parent->_runtime_filter_slots->init(state, hash_table_ctx.hash_table->size()));

From 79eb575d7cc6292f96d9d68c4668ca77f0e7c8fb Mon Sep 17 00:00:00 2001
From: zy-kkk <zhongyk10@gmail.com>
Date: Wed, 3 Jan 2024 12:51:07 +0800
Subject: [PATCH 282/299] [Improvement](nereids)Support ODBC table for new
 planner. (#29129)

---
 .../doris/nereids/cost/CostModelV1.java       |   7 ++
 .../doris/nereids/cost/CostModelV2.java       |   6 +
 .../translator/PhysicalPlanTranslator.java    |  26 +++++
 .../doris/nereids/jobs/executor/Rewriter.java |   2 +
 .../ChildOutputPropertyDeriver.java           |   6 +
 .../apache/doris/nereids/rules/RuleSet.java   |   2 +
 .../apache/doris/nereids/rules/RuleType.java  |   2 +
 .../nereids/rules/analysis/BindRelation.java  |   3 +
 .../LogicalOdbcScanToPhysicalOdbcScan.java    |  42 +++++++
 .../rewrite/PushConjunctsIntoOdbcScan.java    |  39 +++++++
 .../doris/nereids/stats/StatsCalculator.java  |  13 +++
 .../trees/copier/LogicalPlanDeepCopier.java   |  13 +++
 .../doris/nereids/trees/plans/PlanType.java   |   2 +
 .../trees/plans/logical/LogicalOdbcScan.java  | 104 ++++++++++++++++++
 .../plans/physical/PhysicalOdbcScan.java      | 102 +++++++++++++++++
 .../trees/plans/visitor/RelationVisitor.java  |  10 ++
 .../planner/external/odbc/OdbcScanNode.java   |  44 +++++++-
 17 files changed, 417 insertions(+), 6 deletions(-)
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
 create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
index b2fe0f94578113..011afb46fc2809 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java
@@ -35,6 +35,7 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@@ -137,6 +138,12 @@ public Cost visitPhysicalJdbcScan(PhysicalJdbcScan physicalJdbcScan, PlanContext
         return CostV1.ofCpu(context.getSessionVariable(), statistics.getRowCount());
     }
 
+    @Override
+    public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan, PlanContext context) {
+        Statistics statistics = context.getStatisticsWithCheck();
+        return CostV1.ofCpu(context.getSessionVariable(), statistics.getRowCount());
+    }
+
     @Override
     public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext context) {
         Statistics statistics = context.getStatisticsWithCheck();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
index 7f1c6a6c5ac9fc..96afa5123c11c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV2.java
@@ -34,6 +34,7 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalProject;
@@ -142,6 +143,11 @@ public Cost visitPhysicalJdbcScan(PhysicalJdbcScan physicalJdbcScan, PlanContext
         return calculateScanWithoutRF(context.getStatisticsWithCheck());
     }
 
+    @Override
+    public Cost visitPhysicalOdbcScan(PhysicalOdbcScan physicalOdbcScan, PlanContext context) {
+        return calculateScanWithoutRF(context.getStatisticsWithCheck());
+    }
+
     @Override
     public Cost visitPhysicalEsScan(PhysicalEsScan physicalEsScan, PlanContext context) {
         return calculateScanWithoutRF(context.getStatisticsWithCheck());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index f8b23935318dac..da0a58f4f37432 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -41,6 +41,7 @@
 import org.apache.doris.analysis.TupleId;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.Function.NullableMode;
+import org.apache.doris.catalog.OdbcTable;
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Table;
 import org.apache.doris.catalog.TableIf;
@@ -112,6 +113,7 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
@@ -176,6 +178,7 @@
 import org.apache.doris.planner.external.hudi.HudiScanNode;
 import org.apache.doris.planner.external.iceberg.IcebergScanNode;
 import org.apache.doris.planner.external.jdbc.JdbcScanNode;
+import org.apache.doris.planner.external.odbc.OdbcScanNode;
 import org.apache.doris.planner.external.paimon.PaimonScanNode;
 import org.apache.doris.qe.ConnectContext;
 import org.apache.doris.statistics.StatisticConstants;
@@ -577,6 +580,29 @@ public PlanFragment visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, PlanTransla
         return planFragment;
     }
 
+    @Override
+    public PlanFragment visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, PlanTranslatorContext context) {
+        List<Slot> slots = odbcScan.getOutput();
+        TableIf table = odbcScan.getTable();
+        TupleDescriptor tupleDescriptor = generateTupleDesc(slots, table, context);
+        OdbcScanNode odbcScanNode = new OdbcScanNode(odbcScan.translatePlanNodeId(), tupleDescriptor,
+                (OdbcTable) table);
+        odbcScanNode.addConjuncts(translateToLegacyConjuncts(odbcScan.getConjuncts()));
+        Utils.execWithUncheckedException(odbcScanNode::init);
+        context.addScanNode(odbcScanNode);
+        context.getRuntimeTranslator().ifPresent(
+                runtimeFilterGenerator -> runtimeFilterGenerator.getContext().getTargetListByScan(odbcScan).forEach(
+                        expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, odbcScanNode, context)
+                )
+        );
+        Utils.execWithUncheckedException(odbcScanNode::finalizeForNereids);
+        DataPartition dataPartition = DataPartition.RANDOM;
+        PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), odbcScanNode, dataPartition);
+        context.addPlanFragment(planFragment);
+        updateLegacyPlanIdToPhysicalPlan(planFragment.getPlanRoot(), odbcScan);
+        return planFragment;
+    }
+
     @Override
     public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTranslatorContext context) {
         List<Slot> slots = olapScan.getOutput();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index a97bd61e5c611a..e02c299e71fa1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -95,6 +95,7 @@
 import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN;
 import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
 import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
+import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan;
 import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoin;
 import org.apache.doris.nereids.rules.rewrite.PushDownCountThroughJoinOneSide;
 import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin;
@@ -344,6 +345,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
                             new PruneEmptyPartition(),
                             new PruneFileScanPartition(),
                             new PushConjunctsIntoJdbcScan(),
+                            new PushConjunctsIntoOdbcScan(),
                             new PushConjunctsIntoEsScan()
                     )
             ),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
index 3b07f2bbe985b9..bed9f4fe2e47d2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/ChildOutputPropertyDeriver.java
@@ -47,6 +47,7 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@@ -149,6 +150,11 @@ public PhysicalProperties visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, PlanC
         return PhysicalProperties.STORAGE_ANY;
     }
 
+    @Override
+    public PhysicalProperties visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, PlanContext context) {
+        return PhysicalProperties.STORAGE_ANY;
+    }
+
     @Override
     public PhysicalProperties visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanContext context) {
         return new PhysicalProperties(olapScan.getDistributionSpec());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index b7d07aa134df74..92a5bb9ddb15a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -69,6 +69,7 @@
 import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin;
 import org.apache.doris.nereids.rules.implementation.LogicalJoinToNestedLoopJoin;
 import org.apache.doris.nereids.rules.implementation.LogicalLimitToPhysicalLimit;
+import org.apache.doris.nereids.rules.implementation.LogicalOdbcScanToPhysicalOdbcScan;
 import org.apache.doris.nereids.rules.implementation.LogicalOlapScanToPhysicalOlapScan;
 import org.apache.doris.nereids.rules.implementation.LogicalOlapTableSinkToPhysicalOlapTableSink;
 import org.apache.doris.nereids.rules.implementation.LogicalOneRowRelationToPhysicalOneRowRelation;
@@ -165,6 +166,7 @@ public class RuleSet {
             .add(new LogicalSchemaScanToPhysicalSchemaScan())
             .add(new LogicalFileScanToPhysicalFileScan())
             .add(new LogicalJdbcScanToPhysicalJdbcScan())
+            .add(new LogicalOdbcScanToPhysicalOdbcScan())
             .add(new LogicalEsScanToPhysicalEsScan())
             .add(new LogicalProjectToPhysicalProject())
             .add(new LogicalLimitToPhysicalLimit())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 0ffe509d8bc306..cede463c924f6b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -233,6 +233,7 @@ public enum RuleType {
     OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
     FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
     PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE),
+    PUSH_CONJUNCTS_INTO_ODBC_SCAN(RuleTypeClass.REWRITE),
     PUSH_CONJUNCTS_INTO_ES_SCAN(RuleTypeClass.REWRITE),
     OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE),
     PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE),
@@ -368,6 +369,7 @@ public enum RuleType {
     LOGICAL_SCHEMA_SCAN_TO_PHYSICAL_SCHEMA_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
     LOGICAL_FILE_SCAN_TO_PHYSICAL_FILE_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
     LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
+    LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
     LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE(RuleTypeClass.IMPLEMENTATION),
     LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
     LOGICAL_RESULT_SINK_TO_PHYSICAL_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index ef51de8ddc3a82..6274956ccea103 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -53,6 +53,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalSchemaScan;
@@ -242,6 +243,8 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio
             case JDBC_EXTERNAL_TABLE:
             case JDBC:
                 return new LogicalJdbcScan(unboundRelation.getRelationId(), table, tableQualifier);
+            case ODBC:
+                return new LogicalOdbcScan(unboundRelation.getRelationId(), table, tableQualifier);
             case ES_EXTERNAL_TABLE:
                 return new LogicalEsScan(unboundRelation.getRelationId(), (EsExternalTable) table, tableQualifier);
             default:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
new file mode 100644
index 00000000000000..22aea72f736122
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.implementation;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
+
+import java.util.Optional;
+
+/**
+ * Implementation rule that convert logical OdbcScan to physical OdbcScan.
+ */
+public class LogicalOdbcScanToPhysicalOdbcScan extends OneImplementationRuleFactory {
+    @Override
+    public Rule build() {
+        return logicalOdbcScan().then(odbcScan ->
+            new PhysicalOdbcScan(
+                odbcScan.getRelationId(),
+                odbcScan.getTable(),
+                odbcScan.getQualifier(),
+                Optional.empty(),
+                odbcScan.getLogicalProperties(),
+                odbcScan.getConjuncts())
+        ).toRule(RuleType.LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
new file mode 100644
index 00000000000000..8a1ba4d1b6790f
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
+
+/**
+ * Rewrite odbc plan to set the conjuncts.
+ */
+public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory {
+
+    @Override
+    public Rule build() {
+        return logicalFilter(logicalOdbcScan()).thenApply(ctx -> {
+            LogicalFilter<LogicalOdbcScan> filter = ctx.root;
+            LogicalOdbcScan scan = filter.child();
+            LogicalOdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts());
+            return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan);
+        }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN);
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 8495411745941b..a475d716fdb622 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -67,6 +67,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
 import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@@ -98,6 +99,7 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalPartitionTopN;
@@ -317,6 +319,12 @@ public Statistics visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, Void context) {
         return computeCatalogRelation(jdbcScan);
     }
 
+    @Override
+    public Statistics visitLogicalOdbcScan(LogicalOdbcScan odbcScan, Void context) {
+        odbcScan.getExpressions();
+        return computeCatalogRelation(odbcScan);
+    }
+
     @Override
     public Statistics visitLogicalEsScan(LogicalEsScan esScan, Void context) {
         esScan.getExpressions();
@@ -460,6 +468,11 @@ public Statistics visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, Void context)
         return computeCatalogRelation(jdbcScan);
     }
 
+    @Override
+    public Statistics visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, Void context) {
+        return computeCatalogRelation(odbcScan);
+    }
+
     @Override
     public Statistics visitPhysicalEsScan(PhysicalEsScan esScan, Void context) {
         return computeCatalogRelation(esScan);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index 113542218757a8..3ae47c3a3edaa0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -49,6 +49,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
 import org.apache.doris.nereids.trees.plans.logical.LogicalLimit;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN;
@@ -238,6 +239,18 @@ public Plan visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, DeepCopierContext con
         return newJdbcScan;
     }
 
+    @Override
+    public Plan visitLogicalOdbcScan(LogicalOdbcScan odbcScan, DeepCopierContext context) {
+        if (context.getRelationReplaceMap().containsKey(odbcScan.getRelationId())) {
+            return context.getRelationReplaceMap().get(odbcScan.getRelationId());
+        }
+        LogicalOdbcScan newOdbcScan = new LogicalOdbcScan(StatementScopeIdGenerator.newRelationId(),
+                odbcScan.getTable(), odbcScan.getQualifier());
+        updateReplaceMapWithOutput(odbcScan, newOdbcScan, context.exprIdReplaceMap);
+        context.putRelation(odbcScan.getRelationId(), newOdbcScan);
+        return newOdbcScan;
+    }
+
     @Override
     public Plan visitLogicalEsScan(LogicalEsScan esScan, DeepCopierContext context) {
         if (context.getRelationReplaceMap().containsKey(esScan.getRelationId())) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 3fe662b7baff6c..f1eae216630f3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -33,6 +33,7 @@ public enum PlanType {
     LOGICAL_EMPTY_RELATION,
     LOGICAL_ES_SCAN,
     LOGICAL_JDBC_SCAN,
+    LOGICAL_ODBC_SCAN,
     LOGICAL_OLAP_SCAN,
     LOGICAL_ONE_ROW_RELATION,
     LOGICAL_SCHEMA_SCAN,
@@ -83,6 +84,7 @@ public enum PlanType {
     PHYSICAL_ES_SCAN,
     PHYSICAL_FILE_SCAN,
     PHYSICAL_JDBC_SCAN,
+    PHYSICAL_ODBC_SCAN,
     PHYSICAL_ONE_ROW_RELATION,
     PHYSICAL_OLAP_SCAN,
     PHYSICAL_SCHEMA_SCAN,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
new file mode 100644
index 00000000000000..840cb1113a41c0
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.catalog.OdbcTable;
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Logical scan for external odbc table.
+ */
+public class LogicalOdbcScan extends LogicalCatalogRelation {
+
+    private final Set<Expression> conjuncts;
+
+    public LogicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
+            Optional<GroupExpression> groupExpression,
+            Optional<LogicalProperties> logicalProperties,
+            Set<Expression> conjuncts) {
+        super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier,
+                groupExpression, logicalProperties);
+        this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
+    }
+
+    public LogicalOdbcScan(RelationId id, TableIf table, List<String> qualifier) {
+        this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of());
+    }
+
+    @Override
+    public TableIf getTable() {
+        Preconditions.checkArgument(table instanceof OdbcTable,
+                String.format("Table %s is not OdbcTable", table.getName()));
+        return table;
+    }
+
+    @Override
+    public String toString() {
+        return Utils.toSqlString("LogicalOdbcScan",
+                "qualified", qualifiedName(),
+                "output", getOutput()
+        );
+    }
+
+    @Override
+    public LogicalOdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
+        return new LogicalOdbcScan(relationId, table, qualifier, groupExpression,
+                Optional.of(getLogicalProperties()), conjuncts);
+    }
+
+    public LogicalOdbcScan withConjuncts(Set<Expression> conjuncts) {
+        return new LogicalOdbcScan(relationId, table, qualifier, groupExpression,
+                Optional.of(getLogicalProperties()), conjuncts);
+    }
+
+    @Override
+    public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+            Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+        return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts);
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitLogicalOdbcScan(this, context);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        return super.equals(o) && Objects.equals(conjuncts, ((LogicalOdbcScan) o).conjuncts);
+    }
+
+    public Set<Expression> getConjuncts() {
+        return this.conjuncts;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
new file mode 100644
index 00000000000000..f63b5a8d420206
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java
@@ -0,0 +1,102 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
+import org.apache.doris.statistics.Statistics;
+
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Physical odbc scan for odbc table.
+ */
+public class PhysicalOdbcScan extends PhysicalCatalogRelation {
+
+    private final Set<Expression> conjuncts;
+
+    /**
+     * Constructor for PhysicalOdbcScan.
+     */
+    public PhysicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
+            Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, Set<Expression> conjuncts) {
+        this(id, table, qualifier, groupExpression, logicalProperties,
+                null, null, conjuncts);
+    }
+
+    /**
+     * Constructor for PhysicalOdbcScan.
+     */
+    public PhysicalOdbcScan(RelationId id, TableIf table, List<String> qualifier,
+            Optional<GroupExpression> groupExpression,
+            LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics,
+            Set<Expression> conjuncts) {
+        super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier, groupExpression,
+                logicalProperties, physicalProperties, statistics);
+        this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null"));
+    }
+
+    @Override
+    public String toString() {
+        return Utils.toSqlString("PhysicalOdbcScan",
+            "qualified", Utils.qualifiedName(qualifier, table.getName()),
+            "output", getOutput(),
+            "stats", statistics
+        );
+    }
+
+    @Override
+    public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+        return visitor.visitPhysicalOdbcScan(this, context);
+    }
+
+    @Override
+    public PhysicalOdbcScan withGroupExpression(Optional<GroupExpression> groupExpression) {
+        return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts);
+    }
+
+    @Override
+    public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression> groupExpression,
+            Optional<LogicalProperties> logicalProperties, List<Plan> children) {
+        return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts);
+    }
+
+    @Override
+    public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties,
+                                                           Statistics statistics) {
+        return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression,
+                getLogicalProperties(), physicalProperties, statistics, conjuncts);
+    }
+
+    public Set<Expression> getConjuncts() {
+        return this.conjuncts;
+    }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
index 65a03505021629..adfe4991665216 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java
@@ -25,6 +25,7 @@
 import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
@@ -35,6 +36,7 @@
 import org.apache.doris.nereids.trees.plans.physical.PhysicalEsScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalFileScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOdbcScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
 import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation;
@@ -90,6 +92,10 @@ default R visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, C context) {
         return visitLogicalRelation(jdbcScan, context);
     }
 
+    default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) {
+        return visitLogicalRelation(odbcScan, context);
+    }
+
     default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) {
         return visitLogicalRelation(olapScan, context);
     }
@@ -131,6 +137,10 @@ default R visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, C context) {
         return visitPhysicalRelation(jdbcScan, context);
     }
 
+    default R visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, C context) {
+        return visitPhysicalRelation(odbcScan, context);
+    }
+
     default R visitPhysicalOlapScan(PhysicalOlapScan olapScan, C context) {
         return visitPhysicalRelation(olapScan, context);
     }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
index 68950ebab833cc..d827ec4a4eba10 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/odbc/OdbcScanNode.java
@@ -22,6 +22,7 @@
 import org.apache.doris.analysis.ExprSubstitutionMap;
 import org.apache.doris.analysis.FunctionCallExpr;
 import org.apache.doris.analysis.SlotDescriptor;
+import org.apache.doris.analysis.SlotId;
 import org.apache.doris.analysis.SlotRef;
 import org.apache.doris.analysis.TupleDescriptor;
 import org.apache.doris.catalog.Column;
@@ -31,6 +32,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
 import org.apache.doris.planner.PlanNodeId;
 import org.apache.doris.planner.external.ExternalScanNode;
 import org.apache.doris.planner.external.jdbc.JdbcScanNode;
@@ -52,6 +54,7 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 /**
  * Full scan of an ODBC table.
@@ -78,6 +81,22 @@ public OdbcScanNode(PlanNodeId id, TupleDescriptor desc, OdbcTable tbl) {
         this.tbl = tbl;
     }
 
+    @Override
+    public void init(Analyzer analyzer) throws UserException {
+        super.init(analyzer);
+    }
+
+    /**
+     * Used for Nereids. Should NOT use this function in anywhere else.
+     */
+    @Override
+    public void init() throws UserException {
+        super.init();
+        numNodes = numNodes <= 0 ? 1 : numNodes;
+        StatsRecursiveDerive.getStatsRecursiveDerive().statsRecursiveDerive(this);
+        cardinality = (long) statsDeriveResult.getRowCount();
+    }
+
     @Override
     protected String debugString() {
         MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this);
@@ -87,11 +106,24 @@ protected String debugString() {
     @Override
     public void finalize(Analyzer analyzer) throws UserException {
         // Convert predicates to Odbc columns and filters.
-        createOdbcColumns(analyzer);
-        createOdbcFilters(analyzer);
+        createOdbcColumns();
+        createOdbcFilters();
+        createScanRangeLocations();
+    }
+
+    @Override
+    public void finalizeForNereids() throws UserException {
+        createOdbcColumns();
+        createOdbcFilters();
         createScanRangeLocations();
     }
 
+    @Override
+    public void updateRequiredSlots(PlanTranslatorContext context, Set<SlotId> requiredByProjectSlotIdSet)
+            throws UserException {
+        createOdbcColumns();
+    }
+
     @Override
     protected void createScanRangeLocations() throws UserException {
         scanRangeLocations = Lists.newArrayList(createSingleScanRangeLocations(backendPolicy));
@@ -152,7 +184,8 @@ private String getOdbcQueryStr() {
         return sql.toString();
     }
 
-    private void createOdbcColumns(Analyzer analyzer) {
+    private void createOdbcColumns() {
+        columns.clear();
         for (SlotDescriptor slot : desc.getSlots()) {
             if (!slot.isMaterialized()) {
                 continue;
@@ -161,16 +194,15 @@ private void createOdbcColumns(Analyzer analyzer) {
             columns.add(JdbcTable.databaseProperName(odbcType, col.getName()));
         }
         // this happens when count(*)
-        if (0 == columns.size()) {
+        if (columns.isEmpty()) {
             columns.add("*");
         }
     }
 
     // We convert predicates of the form <slotref> op <constant> to Odbc filters
-    private void createOdbcFilters(Analyzer analyzer) {
+    private void createOdbcFilters() {
         if (conjuncts.isEmpty()) {
             return;
-
         }
         List<SlotRef> slotRefs = Lists.newArrayList();
         Expr.collectList(conjuncts, SlotRef.class, slotRefs);

From 2a9b4a0f7686e80af2ead476e7e3f70fb8dab029 Mon Sep 17 00:00:00 2001
From: wuwenchi <wuwenchihdu@hotmail.com>
Date: Wed, 3 Jan 2024 12:53:39 +0800
Subject: [PATCH 283/299] [enhancement](paimon)support predict for null and
 notnull (#29134)

---
 .../paimon/PaimonPredicateConverter.java        | 17 +++++++++++++++--
 .../paimon/test_paimon_catalog.out              |  4 ++++
 .../paimon/test_paimon_catalog.groovy           |  3 +++
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonPredicateConverter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonPredicateConverter.java
index cd35d1093ed084..ac750be2745281 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonPredicateConverter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/paimon/PaimonPredicateConverter.java
@@ -21,6 +21,7 @@
 import org.apache.doris.analysis.CompoundPredicate;
 import org.apache.doris.analysis.Expr;
 import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.IsNullPredicate;
 import org.apache.doris.analysis.LiteralExpr;
 import org.apache.doris.analysis.SlotRef;
 import org.apache.doris.thrift.TExprOpcode;
@@ -70,10 +71,16 @@ public Predicate convertToPaimonExpr(Expr dorisExpr) {
 
             switch (compoundPredicate.getOp()) {
                 case AND: {
-                    return PredicateBuilder.and(left, right);
+                    if (left != null && right != null) {
+                        return PredicateBuilder.and(left, right);
+                    }
+                    return null;
                 }
                 case OR: {
-                    return PredicateBuilder.or(left, right);
+                    if (left != null && right != null) {
+                        return PredicateBuilder.or(left, right);
+                    }
+                    return null;
                 }
                 default:
                     return null;
@@ -120,6 +127,12 @@ private Predicate binaryExprDesc(Expr dorisExpr) {
                     if (name.equals("like") && !s.startsWith("%") && s.endsWith("%")) {
                         return builder.startsWith(idx, BinaryString.fromString(s.substring(0, s.length() - 1)));
                     }
+                } else if (dorisExpr instanceof IsNullPredicate) {
+                    if (((IsNullPredicate) dorisExpr).isNotNull()) {
+                        return builder.isNotNull(idx);
+                    } else {
+                        return builder.isNull(idx);
+                    }
                 }
                 return null;
             default:
diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
index b0c9eff5b0a7f8..9661bf56cf8b81 100644
--- a/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
+++ b/regression-test/data/external_table_p0/paimon/test_paimon_catalog.out
@@ -474,3 +474,7 @@ bbb
 2	[[0, 0, 0], [1, 0, 0], [0, 1, 1]]	[["a", "5", "f"], ["7", "2", "3"], ["1", "b", "f"]]	[["11", "a8", "e3"], ["f4", "ee", "c3"], ["0f", "c6", "05"]]	[["b723869515b24e9fbb54503f8a7584f083479998766213b784a9c530cbc0376bd5035054c657437251b85fa3dd41a0483776", "ee780d81d3e9faa36aad06522a09cf9b18e00614c991d2c079243dcc7190f3dd6559e75e2c1992270272d9a9c01e950c7bd4", "6e825a52cdae65786801caae53182956c80f88a48fa258a90d2c93302023c78b83f1dce758615a74731c9eef993a8c1dc4ad"], ["2ae97869372970f7ad7d0007a1baf32f545ce6b35c07d1036a442f2e158cf27866bff515aa87979e54f912474828aff0a9f0", "7a209d362febaa020046d95188d1c3ceacf7d9395b013deefb46e0f2909a9585b9f082d1f2fcb82089aa8605e4eef1ac0e42", "fcc347c8c9c6a9b234549935ad12e804465f11a163dd056057a18db660724ec45d49956417aada66756ae0a26cf462736e9f"], ["70240ada78e7218dd4e35bc2b0dbde0723550bc12310f9ed5d1db473ac692085ad71356a3047351fff568c88b0d3376cf3ce", "ac03095f9b2fdd51d35d7a4d7e8d5afa504b7c6aa3f276916496e0ca7a3686b63ef416aaeb21012a20cb889acfd820e138d8", "583d85847b103178402b3980802a904f88ab6f3fa06b704a50604d79171ec658cb5050ad30fa5b88819e21b1e88bf8245266"]]	[[27294220.50, 621609.42, 89810366.98], [94120497.72, 52108269.00, 51943703.30], [30485257.66, 9827656.82, 10269787.75]]	[[31, 7, 85], [41, -81, 74], [74, -103, -124]]	[[-22183, 17022, 12520], [5187, 14755, -8284], [70, -502, 11762]]	[[345357276, -845901970, 1807254971], [-1727596054, -1015360238, -1974497384], [-1865141785, 1458802452, -1015598789]]	[[1595618270205702107, 4161236884532086724, 4447316033415055190], [7178945014245655672, 1744975092040033332, -1258393622459157798], [352676403984115467, -413501818079508501, -5172233667710709371]]	[[9.592082e+36, 2.2061998e+38, 3.217547e+37], [1.923472e+38, 2.2883073e+38, 2.7381941e+38], [9.7995851e+37, 1.2078655e+38, 1.3424833e+38]]	[[5.4945052263888254e+307, 2.2120955121044974e+306, 6.4336990418599792e+307], [1.2283455915861409e+307, 1.4133837555064845e+308, 1.2823937308644953e+308], [7.28717488536204e+307, 1.6750732179035529e+308, 1.3720069819113936e+308]]	[["2023-10-26", "2023-10-26", "2023-10-26"], ["2023-10-26", "2023-10-26", "2023-10-26"], ["2023-10-26", "2023-10-26", "2023-10-26"]]	[["2023-10-26 03:16:52.978", "2023-10-26 03:16:52.978", "2023-10-26 03:16:52.978"], ["2023-10-26 03:16:52.978", "2023-10-26 03:16:52.978", "2023-10-26 03:16:52.978"], ["2023-10-26 03:16:52.978", "2023-10-26 03:16:52.978", "2023-10-26 03:16:52.978"]]	[{0:0, 1:1}, {0:1, 1:0}, {0:1, 1:0}]	[{0:"6", 1:"a"}, {1:"d"}, {0:"f", 1:"8"}]	[{0:"8b", 1:"7c"}, {1:"d4"}, {0:"78", 1:"70"}]	[{0:"3373c76adf77290db661a17416237af654e03fa35c19a47c24aaade6d68c3e0ad1eb918cf9ad5a6d26c3a6fd4a354b4280b8", 1:"1834c88752f94635e08b9219b81cdcdfa1486b3576fffe31a417ed7c1e882b3fcc0885523e086ae4ad47ed2439639d7fd578"}, {0:"23eea4f2875e28203eb6b056ab82bc4e399110eecf4b117d61c374a904336b6dc2a1c3c19011063300ed304ef72d1a01decb", 1:"9071be5c97e0957535b30fb7e5b6fc15c28bc58c4f3371a61d461683967cdbb5526d1aed6806dcb6a7a8e634d909022e5c27"}, {0:"121cb5f0e372baf6b8a547e533ecbf91065dc75ffd3d883a4ccba6ed92aab2638671e4f5d112cb64f73f835c54d0d43f0e54", 1:"f14c84918207bd23e2492e45c148a0d4555a76215cc127373322bd8a891200361fdd4140eda03a2ac6560fe0d3db47ae880f"}]	[{0:23164756.95}, {0:57668952.85, 1:45497021.00}, {0:89971113.51, 1:93512132.85}]	[{0:-37, 1:-36}, {0:-9}, {1:-46}]	[{0:2361, 1:-23931}, {0:6558}, {0:-30863, 1:28120}]	[{0:-1497921417, 1:115430027}, {0:-1559227823, 1:168556957}, {0:-132672594}]	[{0:-1049555347411727074, 1:-1121490872336991226}, {0:-2632747659482705401, 1:-7442460162314167230}, {0:7517716583745120627, 1:3205812162045902270}]	[{0:2.0413164e+38, 1:6.4866711e+37}, {0:1.9324323e+38, 1:8.7680928e+37}, {0:2.8165718e+37, 1:1.8986205e+38}]	[{0:1.1095560982466849e+308, 1:1.6655508415857888e+308}, {0:1.6690543045381207e+308, 1:6.4299900705390036e+307}, {1:1.3246574407231653e+308}]	[{0:"2023-10-26", 1:"2023-10-26"}, {0:"2023-10-26", 1:"2023-10-26"}, {0:"2023-10-26", 1:"2023-10-26"}]	[{1:"2023-10-26 03:16:52.979"}, {0:"2023-10-26 03:16:52.979", 1:"2023-10-26 03:16:52.979"}, {0:"2023-10-26 03:16:52.979", 1:"2023-10-26 03:16:52.979"}]	[{"6":1, "2":0, "4":1}, {"8":1, "a":0, "1":1}, {"6":0, "4":0, "0":0}]	[{"7":"3", "7":"3", "b":"e"}, {"d":"5", "1":"6", "f":"b"}, {"7":"0", "6":"6", "f":"0"}]	[{"b":"78", "3":"a4", "e":"90"}, {"b":"ec", "1":"ec", "c":"c5"}, {"5":"04", "2":"48", "8":"cf"}]	[{"a":"158117877f551676582b68677b6ed4e662edc85988595dc01216f227a816d92d7d355aa31f8fa266ee8db96197add9283a1b", "4":"4434955d9a279bfc59fc22ee7334b3d37b3b20ddf58182bfc3d3eb269fc4aac9fb84e7f1d263cd414235e42fba9ee8b9a49e", "0":"9ae1de45315df60e6b4dfd7b90a4f1b49ec2d8383e8f48c501ccafe700a9673f72c0806fe315986582a41f321633bb6e1806"}, {"9":"7f9382e9b670b1c5ed9251a8bdc4bb8188cdb52acb92bb2a1e71ca9190425d1cb2b614cfea1f20c16c0cb05523f9e2ec04a3", "0":"3fd21c9f8e44810bc7144ea9b98e0ff448465e74c70984bf864b567e9caf36ffe8830bed5ff0e739ce144b91a662721f985b", "0":"7c8ed58c3401bd5ca778f1da7303fd5ab95c7ed5baeb44efad0818dd9a84a60e1f4eaf900942e6d91dcc7c30200a55e216e9"}, {"4":"d4e2bb6a1df7dd75d3a87ff3c6c6d3175770818a22199bbbdbc9637db46c29d973faaffcc662ca63179ff05c1910fdd62844", "8":"738ec06b1fa51c03d6245760166b450a0c278fa8ddf8dda0da784c03fff33d365e2f6f718d5d35d452d68593cc6d6ba9d537", "a":"3439c2ca006bd369401d286bad73acaeac4807a916d4459b8d43e4bf69eac5a6c035567d4d19d30b95e871a1c2a06df436f6"}]	[{"3":24392132.85, "4":9487420.19, "d":92095408.83}, {"e":31199693.00, "6":59650399.36, "2":69556596.15}, {"0":53085519.85, "3":66408349.20, "2":15651029.71}]	[{"c":72, "c":31, "0":38}, {"f":52, "f":46, "6":-119}, {"0":42, "c":127, "c":19}]	[{"8":-15264, "c":27729, "3":22048}, {"d":16291, "2":23038, "4":29936}, {"3":15465, "5":-22048, "6":-21629}]	[{"a":-2123437528, "4":-1826547987, "3":1001444635}, {"5":-1245178942, "5":1317449109, "c":2053352594}, {"9":-922174876, "e":1467215952, "7":-71135416}]	[{"3":-4172989025121123140, "6":6538912373819103749, "a":-2230018246838553160}, {"e":7809145645942202126, "9":-3271832743100640484, "2":-5072396908364223483}, {"9":4474832576540322054, "4":-7992269645736114390, "4":-9106565998170115259}]	[{"6":7.6324071e+37, "5":3.0157235e+38, "2":6.5890749e+37}, {"b":2.6171484e+38, "8":2.5065334e+38, "7":2.7708948e+38}, {"b":9.25816e+37, "4":6.1175459e+37, "2":2.8126064e+38}]	[{"3":1.3274524069013966e+308, "e":3.1662877373892025e+307, "a":1.6860993526343602e+308}, {"1":2.922697105821476e+306, "2":4.3928540779994066e+307, "7":6.3813111881043586e+307}, {"7":6.884259875198191e+307, "b":1.1771370481121831e+308, "0":1.6411622519450489e+308}]	[{"2":"2023-10-26", "6":"2023-10-26", "c":"2023-10-26"}, {"b":"2023-10-26", "c":"2023-10-26", "9":"2023-10-26"}, {"5":"2023-10-26", "f":"2023-10-26", "8":"2023-10-26"}]	[{"5":"2023-10-26 03:16:52.979", "3":"2023-10-26 03:16:52.979", "c":"2023-10-26 03:16:52.979"}, {"b":"2023-10-26 03:16:52.979", "9":"2023-10-26 03:16:52.979", "9":"2023-10-26 03:16:52.979"}, {"3":"2023-10-26 03:16:52.979", "6":"2023-10-26 03:16:52.979", "5":"2023-10-26 03:16:52.979"}]	[{"96":1, "4f":0, "90":1}, {"d9":1, "2d":1, "cf":0}, {"91":1, "49":1, "cd":1}]	[{"4f":"5", "03":"e", "12":"4"}, {"4c":"9", "55":"e", "a4":"3"}, {"0f":"2", "d1":"7", "a4":"b"}]	[{"89":"b1", "60":"c2", "dc":"f5"}, {"02":"d6", "0a":"a8", "01":"56"}, {"0a":"f8", "75":"1f", "88":"84"}]	[{"3e":"875fedd73b6d3875af3ba9f506c3a2ce597c942587cbbfbda76963f2412ea34caa0eec719ecbc5d6dd9170489eea65d1d3b2", "a4":"2364e48899c9843035944653b246d54790150c120c2bf6ec5489184fcbd2fca299833d5432db984023dc12a4abb2422dbed4", "a0":"dab5fffb329d544204ce662842183faf54d1e50eb522f2ae4513dac184f0ce15514932be3bac884b4a8e70a472ffccf53139"}, {"68":"1d81598bdd6b81525c5a74570d23f7cfee8780d563be7c5ceee3e46ec673629e72f92657cc80b1d976812119fa4bce7a58ce", "95":"09e9334751fb9a04dc09ef3243fba7e246f0dfc0a44a770563901ae8370b1e2062ad5d0e9d86c1c3810c9080b1867e545965", "5a":"8b6437949fcc7b232c28f1c5316b7ae0b0137c7e62c09e35f64e486cd5cf41486617d5f0d3a0913ab507e3ab49ba23be6b30"}, {"4b":"db9b126b28609fc170279bbdee92d3b66b69a219dfd0e1738755751f623c8562026a81e646fcf6ea974e5340dc6e308281f5", "91":"55a0efee8ce1a17b5b2d6c963bc4380ac73a84a94b22f52d981e333d72408aac94c6c05665b6988b72e88fa33efc1f866565", "25":"c452bb3b483d31c9e419e8a033bd9c250a2dbf5a1cd64d304476eb7f32f41a13e1e3d622e7843c9fa73228487fe7f4648595"}]	[{"5e":21834887.64, "ec":47311086.03, "cc":98208392.34}, {"a3":17925355.33, "fd":70430822.15, "c7":15590338.28}, {"87":23190581.12, "25":76034575.37, "a0":12771556.04}]	[{"0c":-91, "6d":21, "d4":83}, {"f6":-75, "f0":-105, "72":70}, {"79":-25, "ee":-116, "53":-94}]	[{"66":6331, "c0":9597, "ca":22929}, {"9c":-13236, "a7":29400, "30":24088}, {"d4":8214, "12":-9768, "d1":22293}]	[{"7e":72050710, "b7":-463248347, "c1":-1577130460}, {"a5":-843856682, "dd":1743398965, "c3":-876260327}, {"15":1877464138, "4d":1325319144, "62":-845592129}]	[{"d2":-4195164360243115761, "ba":7414503939669271863, "75":1559483977275326633}, {"69":1495332962010830027, "72":8124333721012892662, "6c":7714825807296524135}, {"1d":5691614238331234131, "ec":6274090450804373496, "09":848410483928349751}]	[{"fa":4.5853128e+37, "93":3.5148627e+37, "1b":2.4460823e+38}, {"0e":1.1828208e+38, "dc":1.70649e+38, "a4":3.3094453e+38}, {"99":5.83204e+37, "c4":2.3862326e+38, "9f":1.4772062e+38}]	[{"b2":2.8261672475750253e+307, "24":4.0212734506428018e+307, "42":5.59411330729981e+307}, {"04":1.1257207294480216e+308, "7c":2.8435817813681622e+307, "fe":4.2062716227077134e+306}, {"d9":1.2683186813761259e+308, "c7":1.1136404837688582e+308, "64":1.0955170984905169e+308}]	[{"73":"2023-10-26", "fa":"2023-10-26", "02":"2023-10-26"}, {"bb":"2023-10-26", "9d":"2023-10-26", "fd":"2023-10-26"}, {"ab":"2023-10-26", "7c":"2023-10-26", "26":"2023-10-26"}]	[{"71":"2023-10-26 03:16:52.979", "2a":"2023-10-26 03:16:52.979", "df":"2023-10-26 03:16:52.979"}, {"9c":"2023-10-26 03:16:52.979", "0b":"2023-10-26 03:16:52.979", "a3":"2023-10-26 03:16:52.979"}, {"b4":"2023-10-26 03:16:52.979", "e0":"2023-10-26 03:16:52.979", "cd":"2023-10-26 03:16:52.979"}]	[{"b8e1033538d3ec6e07499a3725d7b80dcc1df256c43ff97d8fd27ae5cd65b1f846b3c3cdd48f145438edb2e0b9312644523b":0, "0ea406c379c0ac021754fcd8053268e3fab25c33a6ccb2f0aca5744c8e4f1230d4745370f9f7955681fa566d99f38febc528":1, "b4123e496a40859006790a63eeb540e000a072a7d00033b6ca2b56211bed232f3530e58add1c70f82cbc6662638f99fc24a2":1}, {"536490b6404dc41ada639a27e70cf5e8a8af9791194287be6520ad08661d01921b08da7e2a297c83aefa30aff3ceeabd75e9":0, "bfea92b8ad24067ea21057ec656c4aa9a3de221fccbc22d30d8d3ee62e595ccdcb06ad4d6659ba6e1dee7a108160ecfa98e9":0, "3a9e38f5eb1b837e9a07892d028bd119b017d24723e0d6c7cbc920a58a5b89602875361d5315ddb1a339252865d4f2f4c69b":0}, {"7203eb495b86da099833cac6d73ab66e582e339c9a514ed04c069a6ef9f1dc6622d9efb316a8dbf71c42577cb0347d620896":0, "d53824f4bed5e8811f1abf63078f2f6b7b386c23e7c8822641ffe19d5e1d45dacaa85ae0b1c39657d32e0a5309d66cd31b07":1, "ebf5cb46ca57d5dac6a727fcbdbfc884c72925b22a7554bee5cbd36586f963978bffd791b7978d331e3780ee7d05caff643f":1}]	[{"8350c1ee4de7c60cc81918920afdc32dceb76e0db8ad6fcad93b817c3b6c0148754a3e1be534887a514cd7f60fadea62197c":"3", "56c05fd9d4f54080108953f5f02d13acbb136558ea70baa2c5e19e5951b75dfbfe190add4c9c6b9486bbe1a508eda1cd34f6":"1", "1f3f3ffc673e7338d8fc8de658f817d3921957c09c9c074ed761fa20d684b5a06d3b9d5f529facdba881874f047b21a75870":"6"}, {"dcf7320548d8d231c1ee79630c1c19f18d535a8d22d0c8ab660e2964243f6fa512b9843bb4e782ded975dedcef5774c233ae":"5", "aba4a8e8d5d3fc2f380e1a2da6231cdd28c35f14821f2ed174abc435689a57b9612a106526b289765d34d222b2aa1685e5ac":"d", "637759f294e0ea2fd917d892506c3ecd82dd33b14d1b1f08c1b829393c88019e71393fbb83c8fdb296f45058a54c553a6c35":"a"}, {"bc08097864e17ab0e7b2953f1afff701ccb3c05dd30697195cd35e013f73f0a8d30740fb1e886497e133b66ff3af999cd539":"6", "84f9fee41af5355d8e7a2da6e81641260e204290e92d8a6d8d6f0d1c050705a084473b7fbdd179bbc524f2b3ecde25870db4":"9", "815e061b83095108b0a66244bbc2fc7c6d8712ebdf029641a7722adfa3314034f49307b7f72335f578cf1515996a4ca04b56":"e"}]	[{"224a513ce220772e3e2269723cd85c770c9cb928fb138380e82d8a8b94f00ce13da7fef889351719e99b3dcde8d793881c62":"47", "9a33cbd2f3f55e0709e59a94f4d2f5a78c71559ccd8097dbbe069765b806783473a09d4c2d727189f94846d56808eef17f8f":"c5", "4af7eb71a3dd4db2b22780ca9fa4d57aaeb519a6873853cc2274a7dca8c3fb8da1480c56ae013003ef02fe7b199cf694199e":"47"}, {"c2f800ab71b436d99ad8717ed0b2e9ec71cd50acd5bac515553f5e776a4eae78e2f6fb42f8c48bd42f391061729f20808578":"63", "0fba442e8e217f53113e6611f0f7eecb1c5665ac70d7af9d660918ef75f0fd3ee27d026b183af0656620a369defc3c8e3e14":"eb", "e3921715c1a475f0a1afeb468aa8c0cbf18f2b4d662d02c9682d54a08fe9d6bef6caa6272573664e7598db0215095dd6a950":"43"}, {"95b489c2257044f910f5e69a4427cfb2b4a82d12e93b8a55194a88bf4c3791a5e43bac091fbe53909ec1a8aaf580ec894813":"80", "9a6496a06c53bc2131d5c5254f2846c659082547369ace717bf1ccc150589e719e9ed590cb114324b8785b824e4632f2e3cc":"63", "892eff4ea70adb9afe3768e5fa5d4ee6a859b015c0dc0a5a42c07cc5d9d1cd25ce648972af5a60edec20eb6054103d7c0c66":"c9"}]	[{"342a9c6663775ae9c264dc93c829c607ed439fd3522644326d7339dc9ab47241121c1f1017f6c42812ba0d4a3e4080419018":"bbf13fb9f34b0323b22812710aa9d829313105ca6f306efad574fb723654a738422fa3d53eb5fd4b1f4fe5744d864cb60eb3", "ec236784aea949dd4b233f2088bf4cf54425da108b7a2012d35ec9000e75b55a4a13493d54bbe38fb2637d64c42b36a20ff1":"df2138bf5f5d994dfeae805c2b0f0246fa403aac136f8ada1f687a96a104f3c20eafcc8b5dddf866d4d66ef510e25950af4e", "9124750f8cfa8147797443f3f18ed6e22ac1d21fe5285f333f45052e4a1751c73a12c3c1b28493f4e474775ae818cbe59330":"cb3e80649b3e4864d0cf829efb7dc06d7bc18fcc9353f00028a37590309c182595d4757243b6175fdd3cbeca9a05b28b6d42"}, {"52d83366d5c54a3b5d2c6708008fae828af5dc9a73158e7c678e627817481e9439c98b9b51d9b9685fe658651babea85c999":"9edd36d96a6f3dc7fbdd7af0f090eb05f15921abd11b58f1f46dc911adb9027880c8e7f6bd1c150cc26c2c7c3bee5f370793", "002370a40af29e10d89dc0cf7634a723547d712401becbf9ed82bd4a560ca7d82e514146808954e9fc301569b56630f9ba3c":"64878178fc827540de4f39f7fda1d615de2605c056d24408cf6762b6cb06f396ab4e856437553117fc690a9f7ed48ebc473d", "6791b7111361ad17127b1b931d325a0502139e87ab59f7af3c35364eb428cf082a9e37345ea1d61d920936962b3d830efe28":"02b56998bcfbb5abec567f9d89f1f578a7e3f265526dfc0014347312dcbdcbea33f48ad5f1251a185257ebbed5a3e3a41991"}, {"b2c478a75bdbb5f2d06ca068b8070aad1fdca2b5f6917c705144755a4d18ccf688c5246e9009892831f7bcaff98d1027366c":"74d3bd5e0a7d45ac290b72ed78597d3886a522924dc5f5e9bbf38351915edcaa9d50e926ff0046c7ae75f89fb7375e385748", "1a96d2b2cf84fa4bc7deb59dd3f0ae449d452037d651354060d14d725a735ceb3dd891d6672377ecff5cdeb44cd19183ce67":"80e9e67e990062ce22d0c5abf20b177f23b9727bb4dedf8b4ddfff4d5092eb587d7ac6e80c95d63c983f9e7391882f8d1d48", "d92089d708b0478df4503504fa95a29166a0e1c59e7a29f8b1250772de600a75fd7f58db6a0ba0a18c1edc92f12b8d576bd1":"1b1832ed78d97bd701c5bbcd7cfeb5154bdd06f56b87e22d62d330722c0f41c5241aa0c0aa242ad94903689000a5ae875583"}]	[{"7b4ccecb204bab6f61d25da2908258d23d102751b4a1bdbac86b6a04dfd908c55b1358e0061b19292f204be55b36aa0112e9":79235422.54, "d3f7cb07dc0735ddc237ee4b1718973176883b9854d19503d619966bcc40b4965e64feeaab50b7b49b70d94dd8f01debbb82":87319263.11, "8ea2a8e98515155d3ece5ce126f0e417d83fb09c7927c2c3370694e1072ed4f5a155e1484ef8f64484c73c3aa86059ba350f":54654978.84}, {"f9fe1c0f935ef4cb821f5b69415f17f494dc3959d3ad26e9237256d86530ff3985acfdb8e25fc6e86d55d00d0b188767350a":1715018.00, "75f5926b05c9abd1a1fd59c35f0fc75b0e06a525403ce966f233676b426b8a94d24e0959e4617430c1f78f5cff45d825cea0":22364174.73, "3a2bba360d6bc3aa68012def81b349365ebd1a0cd5f65cb82990bff7a60ea87fa869827d8eaa25d25a5f9e6caaa2207523d7":44378081.09}, {"8b52600d7a367bdd9777a7d28e51bf391f4d3e1a91f3dcc686dd99b7bce5c45ae1d7dd0c8ddfde30ca58ec66107d0aa2422e":67125327.58, "bda55523ff0c74696daa9a90d655168955c17d1ac1f668211d118596a4fd727b83f5717a87f23d7adde450ecc7c85037ce7d":47497740.16, "136a0d1261bd31d7f41e4f2ac86522ff1d09bfcd383453b8430d53052fade6953aa65efd6f506f25ee7cd531da0f1434452f":90274350.69}]	[{"c4b216cb2ea22c0aadca5f7941d6ccea06ef77334f88bc8bcd71ced981d47bc02923450f453393b1ce47db78572b00e357e4":59, "9c1fc6834b9d993d51dee1f653f0abc6ded4c969f006c4851262f4d3c071196a65507f057236048f280e0519e6caf52032ce":46, "c1d604f0f35a3a7049844a1f7a35d43b64e14b837425f13667a5e1d950cf341afa483f850a834d83104803d4ec0b99835c4a":-35}, {"1431905930ca24cc44173a5b7e7fd2bb6b79a437bb65754658bf7e7ef846f81ed8989f34b761ad9e8564b4bc9f419b0aef2e":72, "d8d342db09032c27ce13df5339c434c021c3f29a5fe21697c6782f700e5f010345218e5da19d36bbaf86ef06f569fb0be60b":-115, "c14c5a6ddfb7a5a57d33f97b0ce3d33a345c12815012ba621f06051d5bdba9fcde377cd57c9c48ba3294ef31ea207878db35":56}, {"245f458fd44df5676dabbe669140e6e86e6088d3b9b72fba6494909d1c5f2d83427785074c4e924694613e0a2dfb2919de64":-83, "4248d46dd0b873e8c7227010fb48c37a920c7f98ec62040b5a7d02c37bdd242519e6ee61ef842dbf0b0c8a44924edb49b8a2":-65, "e39579081abb2d5cd46db2f9f9ad6d15d7b8d046ea097b10850969424303cda79f744d07827491545a6d527090b96edd0be6":8}]	[{"a7478a5ab23fb7b63fa4d1a92e6ae11006462caca2474cf0c61d28a5ed7be45994b05fc597a3db11f9b8e6c359ec77a2e5e3":9172, "754240e072121a13b9cd18cc616377e83dd1a4f3a3212e5dcf8c7888ed6606de525d52872477dce8d33368cdfa9568ce1ebd":28881, "99ba43f8aa07f48b62cef502d4c8f52d1de1d269fe184df38243ad5f225e563b2d1b382d2d57820486b9bffa42af4df33797":-31824}, {"971a55498dc304ce23bc3f6e2c1ee4b1f4836884c19c891302063116b92f0329fa9931bdc3c019f1848928cd6bd3f3460ceb":-7153, "58aa72287a13f7e1d89952fff0c27e894c58c69f0108ce1f753d82f49b62e5dd98bcb75be7cb8e13ba6183b4abe7d2db6600":7819, "262b22a5478c67ffa9b51dde92478902309c99c68cb2017e182e5f386738536287e8400b0c9079f6feceef5dfbdcf5eeac3b":14461}, {"6c3ef76435e3b7683fe1e56df450265d2a52f7b9780e84201f3bc1e3b37158b5a0122a184531ab24f1c49206dfd4588d993a":-19506, "01da859e9e3722845b995f9e5e583fcfef394e0a9e31ce2eac44f2f3a9889f25cdbc0d7dd8bf1739fbf720d0c39691db76f5":-957, "40944bc5437088ece82cf25b626c6787f75d1f037db3f3550eba0fa9725921f4a2418f715cd73b2fad62bcab56e6af1c0975":3288}]	[{"f7196b5e70ae80b7ccd92f5d0fa8c7e67bfdb48e1368a72519d991abd1d1ae9e327f5729b9a2d469624b07c170d53b31e48f":-1230230998, "01498ed59a5d79ac31bb4d6421e25b4dc0b89d72ac50ebaf609b13d6338cdbd38494d594d0acef8b830af0d8e6b56dd1c5ce":2067993311, "283dd12f1fa0613a329d3b19542ba8fa8d56507767903075ea79cfeccc9c0ca31ab6bf8d96c8438d51b439a26e0afb980588":-486532987}, {"e8a40f2531f0120319d4540ec627ea4a9c288e6280a5ea4d51db6e2374b126edc4b27a6de8605476365f18ba9fa70946d09c":-1199846390, "bbcebb20641599c37bb76703eda9e54ee63fe957ff188a88a2cab07d7314dc1a3582378fe1d147934d5e6d743435b3a5ed17":-253311236, "2b6a5c2f29469486c3110f8574946d8c0b7db108f38c268fd368a70646cb68c46dbaa6a4c4bd8ccc314d0d33e28e39b88618":362549692}, {"3581ebf704de709cb1ce6c6d98a1bd888b29eb144275577fc8606acfdd0043ca0922245f31336157d5d4a4c6a151f834f2f9":-993132596, "52565affd7c720bb8842f06d1f8b1cffe0b8ac642ae8ccd863a08be7e59d4b9c949f275c9be9ca6b21478a4ca380c6599ce2":548227681, "1ea2ceffce798ebaefead40e4341e1abcf5e6fb1d61af7e48f9045b8c1fa7a8156412f314b9c10ac67be31e68c14318c8021":-770441557}]	[{"94ece5021c2af808f0d24d65c4aa54286a69890024b8d49485ee924a2fbd14b78e208b0e74d6a6d4200b5906127e10d179cb":-8699746582524763266, "4238da7967bb5241544ea361b42f1496d1efe7830bf3786b63447eb179cc0934af54bc80bf5626ac7ed02e3c55d6047bd140":-5695370085931976199, "a5e67a862a7c977f7561d42a8c876e99547c985a7749eea2631853994ad7502f06b9879049cbdc6fdc75effc02fd03407528":3751245492765213536}, {"dad10807167b5fb16ea1a4514e2294731bd8c5e4e067046b71f0e35b6bd919939f42e5bb3a8bd6e71bec1a295cee1e26ba9c":3236985760469621417, "11c04eb08a94ab2d28d9be9d4d77f3820728eb0ed43a7de4200565fe82a92c6ed720edd38acee9a18e42dfbb4097bccf3e05":4039841294342327135, "57e1edbbb4c5ea340fc9a5da8b01b16db14c92fc8ef9ebc535a5cd2e402e834fcaaa885c6f8d274fe89e2b9283e1108997ad":-6936135020533867640}, {"756b38cd7c242872b4dc8c35c0fa043a06eedb61c895a29c38bcb63eed760dc19042775ea43508201d47dfaa26d418d3d0a0":-5235582976327400696, "bc10801ddf14642ee9a235aa0bb011734879e320ffad51783cc49c736fb39d4b6f0a09ad9e8cc051488febe76ca0c8333fa2":-5639429209357416178, "b810497d98ff852b6c53c8f23f03c5be197159923b935b96208d98a1bad9bc50eb25d6170b09e8718b769041c85e22615678":2713530635143005369}]	[{"757f8a085d1e762c9f75ec0c23fd6868f70158a9cc7de5cdb5c1bfd61f09fae9769de5d9f89dd4b88c308f08592c2f5aca76":2.2035603e+38, "8ee5663149668115f8a98dc441edd44fd35ac5c3968b3d838cce9ced565f1f992f900f7e0f71eb7ca5146d2fc2ff151b4b03":1.0114456e+38, "b18314c8c1abc5f215f19a52d792ea805c054a5f7d9fee3c13f86db86975ca662476802ef5b704624f3579cdf24c99936e3d":1.8999912e+38}, {"cd02a58273ce115cc0ff84b47788e4bc46d77292cf6dd343b4a787f5927e89710a1b4e6b2fcb0d6cc15d544b2aff95cb7a78":3.0883003e+38, "0c3b1c6f6ef30217699b848c4d3719056487df1baddfad85a61b7098c17512668a56f9aa85d02b20f2ab9236575cdf3e1762":2.663354e+38, "cceac347aeef3b8a978e92d94af0f074f49c25743aae694cd713477c89a733408fa28c5a9d7d07279a0e90d2c14a95961125":2.5675032e+38}, {"2a333a90bd3b4697e251c8af9e3c5e5db464998429da52332e3a80ee85e16a93ce98fc20bab8dcf7c430fc8b183a815e99c0":2.3445013e+38, "ff1e09f9f9d058ea7b23360a71344d86157bfb3601d2dafd57713fbfa95244ebe165b40499848437a40a8e65519d866bd8a7":7.2439965e+37, "845e2aeef69d77eb0b78f80cf47295d5e1e8154b321b7c9a962c7cfff41e933d89d9562f58967ba346f7aea6457a62af60ab":1.1417229e+38}]	[{"bb74c1b3c29d3bb630673c12500ceb909e3d934c7c0f73c6309bb16fd4296cc1341039a8ef73cacc06a232b787e5db64f7e8":2.3423512001512156e+307, "cf088f2366a4994c3460e67d3ba5f054474989a521f99c91bd24e64780918f831a5b2d1335e9130e5964ff7ec7bafbb07b09":7.7871799607891731e+307, "4c74dd14befe4385585eef6bf752ba5b23de35a9f1a87f263ec8e414e5011278a0aa73c73c05065ae9faeeba2830b2ed0e5a":1.6792278823089662e+308}, {"39c24fe31693757c99c5ed784434d40ad0e4625b69df248bc9fc940783edcb7091128bdddaf0e0636693f3dc81b66390882d":2.5763679148390038e+307, "4f5f58f49a5dd9f1dcbd3d1031a43f2dd02be95f05b5ea4e4c8384a18d5e884b4a0e36f6288ed197072e369682bdb0a3ba11":1.7556017699783819e+307, "34369351d6a0e2fa81f34dae63617422b56bdefb485165ae7e99a9cfb313a713fb4374cc44df9b891bec344d4570a63f2eda":1.6466637405324768e+308}, {"ab5f6e458a85dd552656ed612a5030af599bd5cc5a983f2557f15abd891c84c8a7f0676ffe98a5775034f72107ec0e28e909":1.0146318446829217e+308, "9cfabd94c15e1b8cbfda6ed24c2eb35bc6a6e902160e1cabb4c8cc879d3efb5be4f30de08711cba7bcd537e61d1fb2287084":4.9538383833294865e+307, "a0f2d61aa57d1679f54fdb059250485932ab557348632ab0a3a9d7988f668dbe94f689e8ae40ffd889e30d97ddf4409c9456":1.4056924496744237e+308}]	[{"5f1af7f5da1e9dc29f40c10f625eb7ece45ffd3977d74e2943d1da2bdc54fd3d946f803e1e2e72f7372a89789af2215dfd1c":"2023-10-26", "10d783869d92dff4981e37aa4aa2a4c5978dc03e47243e592142114edd370f1b6e8d753bddc29067053742a89d7381545e30":"2023-10-26", "da128e8f184a142eeaaf93f4de32b02652e77616be535110674fb71d27a7d1054613da41ec97ff6f06d5ecb726c4fc5795fc":"2023-10-26"}, {"f76a81c309f4315c9b4025a62887d48ac590eff54d3e2789b8ebe1c77dc372ca282fc4f74a8db15b2b7b50969e1121c6d108":"2023-10-26", "c245ed664e66f0bc5f93547dadfd4d23014e32176af5f840b830ff86074fbce04c7176a01a3eb8a56dcef9286e8599173370":"2023-10-26", "36a3d82ac5fd2a4db217c0920f8b45f811eb8170c8d7a299291613383a9cf24b316f25d979b84631ca2cf4d00a567b874bed":"2023-10-26"}, {"b0f5f2448749383bfc9f7ee2c2d19514c539c288c4dc405c0921bde03857485f8a4a14fb7ca3ecb957821d1a658b82d6a295":"2023-10-26", "ab2bca3cce7929d030c1fcecaa91215010d3a62057e857199116c7e6e2c0dcaeaf115998ebc7e61e1a7d4c4ac8eb05b5d662":"2023-10-26", "e4b4db3c61d75e5aba3bc0a04d26d0e2d45308b5f8abdca50aa84f4765c4e1f7f0e0624e02e3c406742c585cf52caca33d0a":"2023-10-26"}]	[{"c3d468704005ddb5400f3178e60e90cbaca5951982e796bba60bbe918fe188b9fe4a42746e81dab2d495fd7e79f17c9955b6":"2023-10-26 03:16:52.980", "f1d43fd657c7c242df7479e02637f6578df34b44b6126774419216be93b2da312e6b1da49488de940ed0dfde7a53df540717":"2023-10-26 03:16:52.980", "89e8518996f5eeadb4b9dfcf75be8d83403ef211715c499d99ee94a9f6e78bb787067088f58cfecd0014fcfb8a142283f8aa":"2023-10-26 03:16:52.980"}, {"9e723e2524431997e22a5b869fb4f701c3cc9f24d7840176a619e6c034316927a39c2d7bd463ef1afb316ca252ece269017c":"2023-10-26 03:16:52.980", "0b0f3e1f7b65e0c5bc6eb7b0188fbc9813276f9940f7b81bd269f5ff7d0cace64e64f9c44926adf54e2db150a9f7c4cb2417":"2023-10-26 03:16:52.980", "86f99a76a47625ede3f07e71920e81edf059c91ad9461e4f9fd31681ccd73a411499e08f0642dc9cd2f518138555aa3ea336":"2023-10-26 03:16:52.980"}, {"f1f3d67c894e3737b3594e5201e070152a07f4f1324bb02d263852f56ca489bf1f70d28e0d5e5510005e649136dcf3dd834a":"2023-10-26 03:16:52.980", "ba9aee45c3e6c8cf5a7163dbc914d960cc056f18c9503a838a40e867787e52350f63e95ed9a1d0b547e7aff9002437ca64c8":"2023-10-26 03:16:52.980", "f648bba9f89ce052e1b54adde09ac49582248e356a8d303b27f48c0a175979a891b29a5440d38b56b815e4674b6e41e87d76":"2023-10-26 03:16:52.980"}]	[{72185873.50:1, 56611371.35:1, 50520633.03:0}, {8870652.78:0, 79840870.96:1, 7497786.81:1}, {49987471.72:1, 68856998.89:0, 89294834.89:0}]	[{23139239.15:"7", 29318768.78:"1", 23653829.82:"a"}, {37812990.67:"3", 12220785.23:"c", 8176324.13:"3"}, {18886259.59:"8", 67798086.92:"3", 88573325.86:"d"}]	[{50104031.94:"3d", 45197369.06:"b8", 93326522.91:"d8"}, {33491020.58:"50", 88467976.54:"f5", 60372025.13:"d7"}, {86649204.90:"26", 39679962.21:"bd", 72245532.19:"43"}]	[{45667372.70:"ba0673959f1118b5cfa574c3d1d72029e57d5b30769e0b8c2ad43e2bd30f12ff46df62d8eebfe0782b7dc78f22a76e240e86", 42855455.13:"f4af554b25284bd0de249eac62745ffc04204774253bbd9622c4f937671ca5e6a0e4c55b80e9bdb7308cf56f8e1755e95ed8", 23702841.03:"d7b0d50e2599a11b58072876219c4423047d788133c98235b0fb61c48969f16d3027945e31132f5c6433369601ab0f005633"}, {451066.74:"d6985245416f1ef077f3c34a45008135e4c06e07848f7a12bcc0208d6161f2844cc42510d78d89a1dc5d82fed1c74ac53087", 82507176.87:"4348a0cbeb648d4fe8c99d35797c3010ee52974c3a7f6f2b7ad1bf0b554f098f900f0ed9dcb791901dcea68d64579f4ed8c4", 97869555.98:"ccb71b2086bf51ccfbb2bdcd3f966030e3d74ebb720d4b6c2d593424bfe5db1d4227441ed069a16d1d2076ef2f415f3cf6a4"}, {48212336.36:"acf9b77114a0d5f02f40bd8fe9a7aae76ba6a88f33d2e6dce0668558b76f14155789bac1e2dc2d9b7a127a4edbd3eaefd844", 3835648.75:"4f6dacddeb5ebf1aeca5be0cff2ce27d1ecef10fe4629205ee63cb96d9b97eb7c116f6ae9281d2f86149e68aa7e260da8d94", 59943266.11:"36f3131bff57638e52440c88461a04dc6a479959a24353845d5672c3fc93e4c9f7b62ce615472023969d643b314c5adc3b23"}]	[{50396766.26:93618969.11, 27023532.15:75688784.95, 54166849.28:75480419.80}, {65613714.71:62863125.19, 77016575.34:25512102.30, 58594410.50:40229678.22}, {86896011.84:30189744.16, 69272130.69:57731948.20, 99857193.62:52898029.21}]	[{10527262.08:-59, 35173844.26:73, 57245681.52:-99}, {1798923.81:10, 21178409.38:94, 13653083.36:29}, {50819571.84:12, 18703371.95:125, 76199979.68:-111}]	[{41128527.43:-26263, 17965937.09:-10188, 45648732.64:-10148}, {16269611.57:10709, 65331804.67:-25636, 48302434.96:6584}, {23495328.62:-20379, 93465759.62:-18819, 47868595.00:28269}]	[{49541268.14:1945722429, 76742059.62:651984414, 19953579.48:-602898770}, {32319055.49:-501604905, 56527981.48:1305999072, 79736796.69:1641809100}, {1455425.52:-1292559844, 14292892.63:-148439227, 31901409.48:286138012}]	[{88873044.38:653914187654872398, 32949593.90:-6380068941370057766, 33768083.29:-1492511479966029819}, {85375548.20:-2097797699569810149, 83169225.53:-2568161106216050077, 16292676.37:2547844327486812488}, {33770469.84:5993134912301194695, 10491869.66:-8226973137941854350, 53003436.87:8376478410406804847}]	[{69746181.85:1.8691526e+38, 4663684.99:1.9598114e+38, 64152971.03:2.7341165e+38}, {88268742.35:1.8826585e+38, 79100214.60:1.7029561e+38, 69780052.81:2.4530895e+38}, {99442027.74:8.3651284e+37, 74149716.13:2.61323e+37, 66348923.09:1.059177e+38}]	[{28640525.18:7.14130921808126e+307, 57072720.10:5.9339355906207411e+307, 63708916.46:6.73322312751925e+307}, {52461019.27:9.5634080232749357e+307, 41204159.50:7.7761911276883984e+307, 32293505.12:1.7876485112547596e+308}, {75167271.25:1.0947889810982528e+308, 59938108.38:9.1079018402691922e+307, 28858577.10:2.1832221539153736e+307}]	[{20614913.03:"2023-10-26", 38982072.89:"2023-10-26", 35068084.57:"2023-10-26"}, {48707081.19:"2023-10-26", 1196542.49:"2023-10-26", 97882962.73:"2023-10-26"}, {31546261.33:"2023-10-26", 9120162.24:"2023-10-26", 16163041.52:"2023-10-26"}]	[{68660942.74:"2023-10-26 03:16:52.980", 11989537.60:"2023-10-26 03:16:52.980", 69807213.12:"2023-10-26 03:16:52.980"}, {23801511.45:"2023-10-26 03:16:52.980", 65291202.38:"2023-10-26 03:16:52.980", 88700274.36:"2023-10-26 03:16:52.980"}, {20857706.58:"2023-10-26 03:16:52.980", 98201246.55:"2023-10-26 03:16:52.980", 37721961.60:"2023-10-26 03:16:52.980"}]	[{-122:0, -111:0, 63:0}, {-115:0, -103:0, -27:1}, {-45:0, -2:0, -28:0}]	[{-49:"5", 25:"e", -71:"5"}, {8:"e", 98:"c", -56:"d"}, {-97:"3", -98:"1", 62:"0"}]	[{98:"e8", -104:"d2", 79:"21"}, {-30:"9f", -58:"16", -54:"94"}, {-109:"2d", 48:"8d", -15:"78"}]	[{-81:"18b29795b68904159aa2b99b0ffad1c766345f8c665cb7464d482b0b2816c0217ae1cc8fc9d4fe468d5a72d5b26a42410f9a", 113:"646e345b49931b63214bb3dac5dceb165ff879cd8c18ebf5a0ac696075b4e45e7e517f9b1ac96c29c37f46ea08dfa12b4e63", -11:"d375773eade002c0faec242e90045d071d4e3bf7247334c1e38fab98aa221350ddcd21d82d159a3550d379a3d66a5f2ade08"}, {-40:"06493b82399543cf68be4adda31d73ddb66a02532f947d9f2a389353aa12e6fccd962f1e623783d30daa6f4f9eb63a906dec", -120:"285b4ea78b16c5c9496461de67176aca5044389dc2b6446baef833842648cc541f1014c598abe4a1ab255644dd3ede4afe64", 119:"2db500821369d84b54912edd87a404184f58dd10de1b347968e1a874b75ade39c553dc9be470a042538d4addcea7f6749092"}, {8:"020b137b335e21cabb94ad084b65878575a436996298db6b8eb3b5f22e07b9177252ce515d337e91712a333efddd2c609485", -17:"9de2bbf406dadfa32a837b29639da7370dcebab18ed31d966fab491ca3a315aef61dfd23d6b3493647ff85bc7392c5021c41", -74:"4e59a7912c5cc5d9aa1a7479ccce55a3dd5c81bd84d09043f04463af45c73bdbe9a914dfe99ac7d318bf660a430e3079238b"}]	[{-65:19490258.53, 120:51265544.45, -44:29697012.72}, {32:32598541.34, 62:69594568.13, -32:63327870.34}, {-65:44263268.84, -25:58210049.89, 69:95701274.70}]	[{-33:85, 5:0, 54:-41}, {-41:116, 21:60, -127:85}, {-66:-79, -60:13, -84:-105}]	[{20:-12611, 78:-27673, 59:-25778}, {56:-30960, -97:-5334, -127:6554}, {-45:26144, 92:-8503}]	[{-85:-803291057, -107:-452904869, -47:1358316357}, {69:1359346610, 106:814581281, 58:1651691160}, {65:-399482975, 18:-491838310, -60:-198488911}]	[{0:-6061559191285871892, 85:4792227183449866329, 62:-2677595349004121540}, {40:-5463340573334895817, -19:2867301247077727211, 123:3415938387537434417}, {69:3644925503017774693, -64:5489787090871154444, 43:5190096168728564253}]	[{-97:1.8318788e+38, 101:3.3108113e+38, 61:2.7126058e+38}, {34:1.9023006e+38, 91:2.0331319e+38, -44:8.8312846e+37}, {-57:1.1150573e+38, 70:4.8228741e+37, 123:1.9317418e+37}]	[{-29:1.2675165035256152e+308, 79:1.727672663130933e+308, 103:1.0179986459504677e+307}, {-30:1.7776523450181598e+308, 34:3.372203332235452e+307, 78:1.6160032599513208e+308}, {68:5.6897333798058888e+307, -123:2.8203019773914149e+307, -7:9.335967074809405e+307}]	[{-59:"2023-10-26", 122:"2023-10-26", 38:"2023-10-26"}, {-110:"2023-10-26", 85:"2023-10-26", -44:"2023-10-26"}, {-106:"2023-10-26", -79:"2023-10-26", -75:"2023-10-26"}]	[{-69:"2023-10-26 03:16:52.980", 115:"2023-10-26 03:16:52.980", 31:"2023-10-26 03:16:52.980"}, {-65:"2023-10-26 03:16:52.980", -22:"2023-10-26 03:16:52.980", 75:"2023-10-26 03:16:52.980"}, {-110:"2023-10-26 03:16:52.980", 45:"2023-10-26 03:16:52.980", -100:"2023-10-26 03:16:52.980"}]	[{-30690:1, 25934:1, 26038:1}, {-447:0, -8903:0, -1024:1}, {31272:1, 23026:1, 32679:0}]	[{12697:"5", 9186:"7", -27868:"f"}, {-2229:"8", 840:"7", 15233:"c"}, {-12941:"0", -30243:"5", -20023:"7"}]	[{-11370:"e1", -13106:"14", 4765:"6a"}, {-866:"7e", -25223:"48", 24055:"94"}, {6584:"bd", -11270:"9b", -5112:"a8"}]	[{28804:"2f4d89d610f9928ba115c834ad8448d2bdd5b8aa8ad62de6a3e4b72e0645f5cb27b88bc52a63e2b4270d9e89b114e142df18", -15779:"d1f375335e17d5be02ed76b41b16d4821a2d9730a75a4c7690f30c60caf9782d0c912640397869e785552038489a60100033", -19468:"3672ba97ab20c3b2ffcdf6ac1c70512df868a3cc0ceb12e7b6de82311ac2ad27ba6cb573c8cbd4ef713b510d1cc018cf0f09"}, {-26206:"258a1a5bdc4a58118a52beffa5574945c62d54a5c835d0166567b85e4775548858d0f2842d93a801ad286295f9effe89cf85", -5951:"592e9df3313c1a283cdc665efea28fa442b6188d8768efa9c6a954828ac9dfb09624f37921645dad9506dc1af29636ed7418", -26403:"8503f5269ca5ff3ce9a91857d8988ff3e18d63d86fe16a7a733bfc301cd356234fc5a2762fac0fbdde9bdb8ce1bad8f330f4"}, {7804:"f3436ff0e42f97dc6ab30b98455e45dcf35b3dd0a69f9a2947b9ebc8d9f769d5eec98457f28193cf2e8893198796fe79536c", 13464:"52dd48238eb279b017049c3d3dc72f4edccae6accbfd0b5e8287b3fcc88235da2b60a821aead5f3d1bcf76fc72e7a6f4eb9f", 18055:"9f0b6023f0d4cbc781886e5a6689239c7838a0c52961baaccf059aeceda0afffaae5c879d328278dc5f55a05fb47ad761856"}]	[{-27051:97033247.58, -20679:69549581.73, 4043:72288478.30}, {-5121:69934054.31, -30211:53829228.41, -25735:41796208.87}, {4224:14370892.37, 1232:462863.26, -10943:65031138.32}]	[{-10987:-32, -16595:68, 11879:-61}, {5120:42, -12081:108, 32564:-34}, {-26241:-100, -19667:-113, -23271:-63}]	[{-3525:32491, 23112:-8382, 28574:26023}, {1386:-26663, -17247:-16509, -28168:-11710}, {-6510:-17545, -6239:-7325, -3004:31363}]	[{-26637:-2084519745, 12833:2139842758, 9141:-1099682669}, {-14494:-563996778, 32058:393113919, 19478:1514500317}, {21760:-1736927877, 6875:974956350, 20931:186878002}]	[{-1793:-1395849241629461850, -3933:-196117327172458301, 30367:4782987890347027879}, {17380:-8180842525045112936, 3494:-1476331741758210456, -21368:-9195609603979583463}, {-6169:-430980561794573776, 26377:-5441049689093378744, 17545:-2101381969171187264}]	[{-18033:2.5412914e+38, 25217:3.234819e+38, 23851:7.0312122e+37}, {-22185:1.4724157e+38, -4394:2.4248854e+38, 18839:2.268062e+38}, {-30798:7.9186421e+37, 23930:8.3008271e+37, 1991:6.2777417e+36}]	[{-31851:1.1258883376501757e+308, 12835:1.203014568237875e+308, 31551:1.1534308103525269e+308}, {-26157:8.3581295118491514e+307, 15151:3.57053797051619e+305, -6284:3.592857809517692e+307}, {-29473:1.3230090243507821e+308, 20116:3.1694090688964317e+307, -10776:9.0800605887954416e+307}]	[{-13897:"2023-10-26", -1949:"2023-10-26", -20441:"2023-10-26"}, {-502:"2023-10-26", -7207:"2023-10-26", 5590:"2023-10-26"}, {-17485:"2023-10-26", 17436:"2023-10-26", -32004:"2023-10-26"}]	[{232:"2023-10-26 03:16:52.980", 9050:"2023-10-26 03:16:52.980", -8284:"2023-10-26 03:16:52.980"}, {-3149:"2023-10-26 03:16:52.980", -29553:"2023-10-26 03:16:52.980", 24294:"2023-10-26 03:16:52.980"}, {-31558:"2023-10-26 03:16:52.980", 7569:"2023-10-26 03:16:52.980", -10122:"2023-10-26 03:16:52.980"}]	[{853818152:1, 1507320381:1, 2011801263:0}, {-2006788694:0, 1324595437:1, 580154894:0}, {791195772:0, -1607475598:1, 567847275:0}]	[{-598702671:"8", 1948828617:"1", -736696891:"e"}, {1994658440:"c", 1856315075:"7", -773964752:"4"}, {2097687445:"5", 743770504:"0", -1275777674:"1"}]	[{-2088352934:"4a", 170421737:"ee", -2107308309:"4a"}, {2104593465:"e3", -908932671:"c2", 790010713:"7b"}, {697649641:"d4", 946214823:"93", 1236663139:"7f"}]	[{891706202:"240bf7f44c5a8ba124deda7585ee6896644405cc6ebc85a3881d11ed1d14852801a994060a7bb47674fe6ded29ec4c12f925", -331324107:"a94718e43ae334a3ce68caeaf2f8e50f22f10a50fe39519c3825e9311bda40fb24ef84ca58f2230a31f1574b7bf9240c1531", -99657294:"6bfa0dca2a55d8d3699e38d099db69b401b2939d17b14eeca076725a2c0f0a5cd2f9c54362622a7f7d3f5a54bee986bc23ac"}, {1903931149:"1b2fd537c78cd7676597994e957d8b07c21f73ff90380b82348df13476310666ce8cf1b74f231738ce83df281ba04bb58a7a", 432969684:"d4e4e473cbbe410c1961876c3b6fa3e560017b170ab6887fbd716b66d79b9b0ec5d7e5609101dd329ec8aa31b7a680e0d2da", 90887837:"ba13423528ee857157e555707ed2e1cfb8f61a61e5f11ce17479faa2813a196b68dc5609df04e93668e5f275724d4e2415f1"}, {-1207732243:"478a3e77971910c2d5eac17a5b3dee752731e3a4e8d842c693141cb87e06ff1cd13e1f88d65c0c67c9c36a07db34966061d1", 456724011:"7aa82f64eeb483ff00e05e72d51efaa6455bb982c09b3587fa2101520b11bfc056102f4832845f27e338279b129a1cf9fa9d", 1144851654:"4d8e5e5a0c51e4838de3f8d2079af1ebd08b6ac70e1e7f893022e85b48f0c12af03dc5d9df0aafa196584d02005d115aeabb"}]	[{2012432178:18818664.05, 1351589882:86412006.80, 86706745:71037816.26}, {2071264695:40425249.78, 1099550982:21845567.36, -1304189052:68405979.47}, {-494929353:33276030.64, -1537033813:70473508.50, 795204578:56482105.02}]	[{2088854977:120, -1466859008:33, -1753132446:49}, {1278179798:54, 1147901960:41, 1874122347:31}, {1522553764:125, 475518976:83, -937600374:-8}]	[{972707719:29707, 434993511:-18329, -686425502:-18858}, {-182822314:12671, -238681689:-12629, 1226751480:16497}, {-1439957061:20300, -1142304443:22248, 1187438487:-8276}]	[{1553326257:1203297569, 2043892618:-35993383, -207334862:-73797461}, {384339080:1780394651, -508192228:1660345850, -858683972:-570795968}, {839318931:-1591802550, -2033356680:-432126037, -410725890:18125134}]	[{-1830399321:663106583550997062, -855216044:-111800205402470356, -1199104991:-2175022965048987103}, {-1289038822:686966301900628900, -2104007229:4187308566611935250, 984104624:8539268123326068389}, {-2047302608:338671647489149425, -1257417350:-3923588938911195670, -248798948:2883481690187447650}]	[{-1465176773:2.2085554e+37, 1951182072:2.1516651e+38, -1285262258:1.9614737e+38}, {-1544390172:2.206411e+38, -835334393:1.7531382e+37, 2022345585:1.6319622e+38}, {-41369940:5.3088046e+37, 1112082552:3.106548e+38, 1602184445:2.37337e+38}]	[{-775002234:1.2901969630904464e+308, 1525169252:1.5927780078240161e+308, -1394810560:1.089357010370962e+308}, {-1079178952:1.499074375861764e+308, -702804538:1.4828783545207568e+308, -2108148702:5.7944199540920075e+307}, {-1822608923:3.5022682058155822e+307, 1064800078:1.2040276169662617e+308, -754592104:8.4260066309246122e+307}]	[{1622805502:"2023-10-26", -896889825:"2023-10-26", 1633229720:"2023-10-26"}, {-413182594:"2023-10-26", 526311685:"2023-10-26", -790586249:"2023-10-26"}, {-520950394:"2023-10-26", 709840088:"2023-10-26", 1174693003:"2023-10-26"}]	[{-456758430:"2023-10-26 03:16:52.980", 171452530:"2023-10-26 03:16:52.980", 1172193709:"2023-10-26 03:16:52.980"}, {-247497446:"2023-10-26 03:16:52.980", 1862760938:"2023-10-26 03:16:52.980", 811872354:"2023-10-26 03:16:52.980"}, {1998721926:"2023-10-26 03:16:52.980", 2109462570:"2023-10-26 03:16:52.980", 261734312:"2023-10-26 03:16:52.980"}]	[{-7951063950642344721:0, -4205088131239668274:1, -1289908268654437950:1}, {-5990293059267038297:0, 7621084711018783451:0, 1802825137395660123:0}, {4039155025278899671:0, -2583997383452376449:0, 5113615318009121443:1}]	[{7069317866900674585:"7", -8429640772271548904:"f", 3166223379358144889:"0"}, {9084567314846474965:"8", 3293896716828330676:"2", -9132537244839602775:"8"}, {1378239105042325208:"0", 2752072243919781983:"6", 8968884231084686368:"f"}]	[{-2565200316837887779:"b0", -218935404095841249:"eb", 6620250675400934148:"ec"}, {-984537824429103143:"71", -2109588646221313310:"3b", 6561941739911285829:"40"}, {4913717839523112356:"ca", 7378059115177931362:"d4", -4093188018424154837:"64"}]	[{-398620380031457696:"d33c39444a6fdaffd57320dc422fd5828fefddc85a78dd924fbcc713e0f95730649162b017bc92852a70c5bc54c117397276", 5903883581737791086:"83da42f13ddcbb61228c376003fdae5b3228716c27d5cd4ba081252963f8ff1a76c698ab05342995e9f3754704176c0d5389", -6698033796431496492:"014a102e742ad41f657eb3819932cefad2dd4124e1f9b59a211c1cb7c9b4a2b054a051a1b349c54e0334794f01775e01f5ee"}, {3862727925141571466:"61d4e150ff329859f7ebd972a56dbf29734935fe4ccb5fcb8df3984a07edc1fbb24f8ed224985a1a2592e439033a89e9e65d", 5878890496704105124:"9730c49d58e78a4bfec6e68b3901af659af9b97ab10e600f7c4b31298276e1cda71703d7495c0f8541bf072a7a81f7bdf382", 2384212248578055560:"e018f61caa90e8a7fb113f30df5f61aee4da157d4199bf96806079cf603ed9aaf087e9623b6d5105b1f97c30ddee92451862"}, {-2217790992814326890:"3faa7de0e77af0e731021942395f3a25d62f151d7e4b2e81d0eb11deb69dda164200a269bd71ee8625ae6348f356e7d175dc", 8502600453513099230:"0807ba39ffeb29ed6b493c0a0e442d3bd6919e1eda242a7e99f0a7619b83c431ae2c1ea15bc436898a507e8da137cb339ab3", -7566905961493617588:"b1c41740dbce2745f381188cd596a11bc976da7127a1399e1ddebb0e532e7844b304dbc013741421aab87aa9bfaba7f29bc1"}]	[{5832815832395750039:56923709.07, -7399665648947090644:30701670.30, -1205087189401004746:99087907.80}, {-6680308343285880454:94808365.00, 7863988011359101127:71166819.54, 655870306373515251:56589637.78}, {-4308900787885521325:84585932.76, 578379343029977014:74680843.05, -276230549733320310:6378960.40}]	[{-3447908128437695508:-88, 7274377443403350333:-2, 5472981513836591606:-50}, {4237575771521331368:-7, 880871537639887798:-42, -7703288721822530335:13}, {-2070928763433972479:39, 4833757679143367182:107, 887708792917147263:-106}]	[{-8155635399790741320:-8266, -8740868533151583751:19018, -1992989487961866096:-23207}, {4624132081015071440:-675, -6661539545404629013:-6220, 3207922963124335753:5175}, {2408681240214746281:-347, 538209966159822549:-12012, 6513602497691698143:-11192}]	[{5412877616041027616:496144670, 8423256971956392242:-1277594312, 4972793681188677299:826917163}, {3150485209265965818:1988994698, -5835782063869726346:1980656834, 8461250796401496898:442119907}, {3221838457426299760:1159279813, 8360156165490788075:-1289116902, -6077682763540152493:-515066376}]	[{-5317695961552654710:8794978360416930273, -4937180357533956041:6472034488557761280, -7843019386357747945:-8632532147868907646}, {9157572555209733130:-5519987918095083857, 2920366025044543224:-1521544401229175128, -6056352465624227252:-308960335417192223}, {-4508491037053447746:-721855790875949069, -1321203143411470555:-3103960197907234827, 1481747476507598760:168771129351337045}]	[{-7876447156323612353:9.052511e+37, 7886495108522604865:7.5443434e+37, -3303510749232471903:2.1553356e+38}, {-780863273862928842:9.7634966e+37, -1940634396973422140:4.9195329e+37, -3777769701699849445:3.3390499e+38}, {-219626155432476293:1.2851074e+38, 7581268539226783859:3.0940548e+38, -2411712615205683336:3.2963854e+38}]	[{7717530884521091972:8.387143011646801e+307, 2385189084227887797:5.111675624969553e+307, -3484129392081615713:7.2447652736252758e+307}, {-3148282736343199735:2.2569450380229393e+307, 6204696402794912888:1.4981004590100919e+308, -4297188139423083998:1.6968669952322079e+307}, {-4384932973492808466:1.0192939908811789e+308, -395409044675382707:6.195814801513859e+307, -6371673861533194071:1.6322067834914874e+308}]	[{7646494356971077925:"2023-10-26", 2160034688503809029:"2023-10-26", -3575219561512307138:"2023-10-26"}, {2280810562333467117:"2023-10-26", -7068375455117127088:"2023-10-26", 19293144593505224:"2023-10-26"}, {9207056097176522983:"2023-10-26", 1462871400671785516:"2023-10-26", -5370823582650949144:"2023-10-26"}]	[{8379130174692217380:"2023-10-26 03:16:52.980", 7021483882380587848:"2023-10-26 03:16:52.980", -2682385881848843999:"2023-10-26 03:16:52.980"}, {-1591313391848357526:"2023-10-26 03:16:52.980", 1573548364544965008:"2023-10-26 03:16:52.980", 8906222880182589887:"2023-10-26 03:16:52.980"}, {-2500641467155471495:"2023-10-26 03:16:52.980", -5677223362201873077:"2023-10-26 03:16:52.980", -1493375525942895085:"2023-10-26 03:16:52.980"}]	[{1.9157734e+38:1, 1.2430983e+38:0, 9.537576e+37:1}, {1.7119694e+38:1, 2.6868459e+38:1, 3.0860106e+38:0}, {8.4719559e+37:0, 1.0640104e+37:0, 6.9223514e+37:0}]	[{1.0261944e+38:"c", 3.261662e+38:"1", 3.4095631e+37:"e"}, {2.5638566e+38:"a", 3.3626454e+38:"8", 1.5036742e+38:"c"}, {2.8036874e+38:"0", 1.8246544e+38:"4", 2.5642286e+38:"c"}]	[{1.3446665e+38:"27", 7.6941387e+37:"76", 1.9918403e+38:"b0"}, {1.5698505e+38:"ba", 2.8841645e+38:"f5", 2.4765528e+38:"9f"}, {2.137655e+38:"27", 7.5707826e+37:"d3", 1.000346e+38:"22"}]	[{2.0788677e+38:"a265ca07c3cc6e06be60575713ce688ca1347e929d7b82fc567968581f2feed53444dc2239c733711b1b48111f73a29315f4", 2.0846297e+38:"d22a7b3bf7184f43f52fad5f7fb1f25a154c4c3aea3d045087aab874c409592f70443e4e8e83e692917158b90265a2d14988", 1.2844366e+38:"e2b005499bd2a6ed8707582e4f081d38e13dcaa391db966dd1118daa16c32acd9fae4072312609562161e6e5408beabf2b0b"}, {1.7883558e+38:"06c45a373f261325aa73295ff4efbaeab41f9d64ea3479607f9b5be2e21cfe17c0dc82a7c8afdb424c356e454a0d5da5cb34", 2.0218054e+38:"0a14ea89627d90a52bac0064e39315b788e240dd71cab188ace0b4b721c935120f54ab161f2a0c943c16e66a8a3d0d87784a", 1.920469e+38:"39d248698a1a328db81004878683de3117077475cecbba0d8710551ea38d6e22ef62f6f27228dff028ccd7a88650deb276c4"}, {2.8222002e+37:"1d1a62b2b5caade6f48f30ea71d1c36eb8eb7940b9cc3dee38491c47dcb7c3865a24e0961b426d9f3e072a712aa47629bc2b", 2.455654e+38:"8f785d92262c6b15b091e7750a0c6972374bf11e7756f905b38ea98ae578fe60973503ac1e98d0cf4b86b532727496f5dc8b", 7.8942089e+37:"604d84c4ef2c8395fa1767954d30f39452188f3171bf753562c6bc251cb7b55d217c55053e83432d09c7d00e7c0bb7434061"}]	[{2.5934718e+38:29646773.78, 2.5814159e+38:29803106.53, 3.0384289e+37:91129554.69}, {2.715942e+38:82104677.03, 2.2915312e+38:18292995.75, 2.157406e+38:53245813.24}, {1.1959878e+38:47584744.67, 2.9528081e+38:65719906.50, 2.6165028e+38:65454333.65}]	[{2.3657972e+38:-59, 1.7845582e+38:-58, 3.0001487e+38:65}, {2.3066034e+38:93, 7.9458053e+37:-61, 2.7675021e+38:120}, {2.0211991e+38:-3, 3.3949235e+38:-33, 2.0230694e+38:58}]	[{2.569701e+38:31450, 1.4225671e+38:-1019, 3.3605693e+38:12569}, {3.0033883e+37:22645, 1.2471552e+37:-30106, 6.0843943e+37:16625}, {2.4588673e+38:902, 1.7168792e+38:-23807, 2.0429765e+37:23637}]	[{2.5139854e+38:-1647331648, 1.285296e+38:18839579, 2.8907468e+38:-517201505}, {2.7549579e+38:-1138804563, 6.6291104e+37:879399867, 1.0059823e+38:-1025095053}, {3.1690661e+38:135030662, 2.6074407e+38:-624810353, 3.2564526e+38:553212556}]	[{1.6013742e+38:3654547239021094563, 2.882746e+38:9083703347518843755, 3.0452127e+38:8378878058262552506}, {3.3299104e+38:8184697417895064998, 1.0202054e+37:2931893902138833791, 2.9324476e+38:-4336163723247495033}, {4.5727123e+37:7427465937514938973, 1.6488885e+38:2876334764201694278, 1.8685819e+38:-9205563431378939356}]	[{2.1802803e+38:1.1972118e+38, 1.72696e+38:3.0303936e+38, 2.2761404e+37:8.0007504e+37}, {1.6292909e+38:2.2281168e+38, 3.3842535e+38:3.0520972e+38, 9.397675e+37:6.2307542e+37}, {1.7391272e+38:1.0020658e+38, 7.2805261e+37:3.1956697e+38, 2.6613503e+38:1.2921451e+38}]	[{3.368691e+38:1.3193211104106196e+308, 5.1853978e+37:7.896041819783945e+307, 1.326064e+38:2.2592467311266441e+307}, {3.1143636e+38:9.989893815245082e+307, 3.34551e+38:7.0845381491861e+307, 2.3331553e+38:9.3328406765365473e+307}, {2.7970112e+37:8.755019191250077e+307, 2.3195053e+38:1.600355866650953e+308, 3.0494007e+38:1.7303263097083202e+308}]	[{2.7671837e+38:"2023-10-26", 9.9575719e+37:"2023-10-26", 2.6933249e+38:"2023-10-26"}, {1.1102931e+38:"2023-10-26", 8.83295e+37:"2023-10-26", 2.403772e+38:"2023-10-26"}, {6.965985e+37:"2023-10-26", 2.3738997e+38:"2023-10-26", 1.1589004e+37:"2023-10-26"}]	[{2.1477127e+38:"2023-10-26 03:16:52.981", 3.3272149e+38:"2023-10-26 03:16:52.981", 2.4077425e+38:"2023-10-26 03:16:52.981"}, {1.5131882e+38:"2023-10-26 03:16:52.981", 1.7886296e+38:"2023-10-26 03:16:52.981", 1.9836614e+38:"2023-10-26 03:16:52.981"}, {3.1079923e+38:"2023-10-26 03:16:52.981", 3.112939e+38:"2023-10-26 03:16:52.981", 3.2935023e+38:"2023-10-26 03:16:52.981"}]	[{1.7863783445216499e+308:0, 5.4063420261769146e+307:0, 2.2212508662010354e+306:1}, {6.10268991219079e+307:1, 2.0400457681234079e+307:1, 1.6651882713315521e+308:1}, {6.5994166208473578e+307:1, 8.2049239246473131e+307:0, 1.5581302064633035e+308:1}]	[{1.1293276942669707e+308:"6", 2.5991854569599518e+307:"8", 7.8965606603313346e+307:"f"}, {1.1193766512121369e+308:"b", 1.1522016897324584e+308:"b", 7.4881082095466555e+307:"b"}, {1.2369326697095564e+308:"1", 1.5124719455136983e+308:"4", 1.1995496407758467e+308:"1"}]	[{9.9844533909283229e+307:"80", 6.8658926727721936e+307:"9c", 1.4206507750874213e+308:"73"}, {1.2747412509463522e+308:"de", 1.4499694599164859e+308:"33", 1.5414153073225206e+308:"57"}, {1.202962191706013e+308:"d9", 6.8514503504120549e+307:"3b", 6.8372102942468532e+307:"64"}]	[{1.350661771380991e+308:"126ce22720b98a53929e279039a06f709d944f59283daf896d8b11da93a325c3a6f4c481ef15cc164554a27a304fffa292a4", 1.3386591115088859e+308:"98d275afc194e440d4b14dd450632396133d8b0d7177624df10c2456481a48bdc048bf44392b031534ca76d8c26df7dcfc1c", 1.1746528946925264e+308:"20ff948bcedf44bbd28eeb7b91786b7f2c5cb3dd7649385df1878854dd206495460b5bc9dee264dd8d5ad6f2dac19067ef11"}, {9.3064069863527452e+307:"e0f90a26775712a7bd1b7625f0c6b18da127f1ada78941d83cdedfdecaf3847f383182d5b31c77d1d7408db9a1df9f174716", 3.2007256225980254e+307:"9d74b7a6d1ef35642cb53fd9e19f739df9e2d079def27189129686c7306e3bd52d16416fd7b79725bb2e1ecc9dd685e5eccc", 1.5880026291551707e+307:"b61b24aa47dd492e3e0e20729bb6dd5a8a8c619784552c203e75b0f8e2cf1571c4739d3414ca75b7001663f3f729b686ec0c"}, {9.88639751590524e+307:"96207eb11d1eb8a6447057a693561ac043f48d85be1a8fc619c5340f4ce491befa7fbc178803f17169aa87fa1bd34ba6dfee", 1.0545627405320881e+308:"b13e2f06e481c945aba394a0beec921beec9d73c1211f0967a8b1c2295a34888450f4d2dfefa5972da78a6b5506cf57c20bb", 4.3441429713934472e+307:"94915b787b248e5287faba6761a731622aee2769051705e955c89e6b907bbbd412cd14b7f2457c5a350660b652b12885cfe2"}]	[{5.082754843697191e+307:38206509.46, 8.3722588757387765e+307:39216365.10, 5.1794442931899773e+307:66588625.34}, {1.5998912486398083e+308:55682572.67, 9.571924139572229e+307:60943131.36, 7.6478818760623081e+307:43239042.72}, {1.5845210986426648e+308:30234940.14, 1.6026908779327157e+308:60702753.07, 4.8012790723166816e+307:7703647.33}]	[{7.9704762794663063e+307:-71, 3.5931177058304696e+307:96, 1.446683125773332e+308:-4}, {1.7905443893640787e+308:35, 1.1675147296063708e+307:8, 1.7817187828467568e+308:48}, {1.76317010501145e+308:13, 8.548604160915873e+307:59, 1.1353726461612988e+308:-54}]	[{4.79805358335142e+307:12155, 1.467583048646221e+308:21011, 4.03696256442736e+306:17488}, {3.9964428726574267e+307:28579, 6.2922304557375505e+307:30477, 5.1641076418015888e+307:-32622}, {7.8620577006681163e+307:-14034, 2.6081612173565721e+307:12699, 7.3959479330848731e+307:2426}]	[{7.4568293907690876e+307:-1536430449, 1.4965611082560973e+308:156772662, 9.8302099782458634e+307:217438687}, {9.6888194456747278e+307:-1509913359, 1.7034976710679524e+308:-209133351, 1.88595895566057e+307:1495021176}, {9.6291874356496854e+307:239456003, 1.4887527348251017e+308:-106552782, 1.6645969037654707e+308:-1291299586}]	[{5.0063235099434368e+307:1285836839268046027, 8.6237418063060823e+306:-9171578578237511683, 1.2124782117970968e+307:-2481101727181623778}, {6.6202058770786324e+307:1684624107492982992, 9.3110785663587234e+307:-8960471988781501676, 9.0255983550721274e+307:4972395459337776317}, {1.1914113091689633e+308:4396478642246881748, 6.0515919909674147e+307:-5410146590534036768, 6.8396504590259689e+307:2292359843368183391}]	[{7.709641451373475e+307:2.132393e+38, 1.3227149299916771e+308:3.2686091e+38, 4.67766076568587e+307:1.5228753e+38}, {1.1256826073418275e+308:1.3389857e+37, 1.4815413336686926e+308:9.1959573e+37, 5.0611915028815888e+307:8.5569275e+37}, {8.4812197823888737e+307:2.5425814e+38, 1.0889264191447482e+308:3.3152495e+38, 2.9169649920811492e+307:2.4279304e+38}]	[{1.3730695089175998e+308:3.3334777557431834e+307, 1.3468792819565234e+308:1.4787194056808138e+307, 1.602767242110148e+307:2.9680539206615153e+307}, {1.2757724372632169e+307:2.2256305797459039e+307, 1.7317568997231886e+308:1.3579877859164e+307, 1.368674028125795e+308:2.978860967098416e+307}, {1.5574326164942272e+308:1.1051672294984759e+308, 1.4560316404336756e+308:6.9040844739900243e+305, 6.5855990042915688e+306:1.1754522539075596e+308}]	[{1.6177278939041744e+308:"2023-10-26", 9.9515829733210311e+307:"2023-10-26", 5.0748256998318917e+307:"2023-10-26"}, {2.657286502672462e+307:"2023-10-26", 1.6125133890611584e+308:"2023-10-26", 1.3777627372474191e+308:"2023-10-26"}, {1.14336175615532e+308:"2023-10-26", 1.1985345361489119e+308:"2023-10-26", 1.2781530004076846e+308:"2023-10-26"}]	[{1.0765430448756355e+308:"2023-10-26 03:16:52.981", 1.5974813193981796e+308:"2023-10-26 03:16:52.981", 6.5749328664346954e+307:"2023-10-26 03:16:52.981"}, {1.2766677013851353e+307:"2023-10-26 03:16:52.981", 7.0422683565593667e+307:"2023-10-26 03:16:52.981", 1.12134087442788e+307:"2023-10-26 03:16:52.981"}, {6.8265702744340863e+307:"2023-10-26 03:16:52.981", 2.1046354875197386e+306:"2023-10-26 03:16:52.981", 2.0444907227444578e+306:"2023-10-26 03:16:52.981"}]	[{"2023-10-26":0}, {"2023-10-26":1}, {"2023-10-26":0}]	[{"2023-10-26":"7"}, {"2023-10-26":"0"}, {"2023-10-26":"3"}]	[{"2023-10-26":"0b"}, {"2023-10-26":"60"}, {"2023-10-26":"6c"}]	[{"2023-10-26":"35dfa4d5c8e8acaf991f8a9bbf5629ef734d6585441013f6126c977d39500bb4adf9d901381c5db374f85dbd191aac5325fb"}, {"2023-10-26":"11d2537db633a133af82a82e93a278553f71fe0431ce050343eefa7adab6cbad253b0952dc8e8ddf3190c78fbbc900d0a6f9"}, {"2023-10-26":"92a34f3e8ac7a0e0bd2564255c2ddfae84f0f2b76dc78bae5a5593f4b90af1f02d0582605cec9752a4d80318ad51bfa3c433"}]	[{"2023-10-26":41350220.93}, {"2023-10-26":43687922.56}, {"2023-10-26":60102562.51}]	[{"2023-10-26":85}, {"2023-10-26":-96}, {"2023-10-26":33}]	[{"2023-10-26":22381}, {"2023-10-26":-26388}, {"2023-10-26":8529}]	[{"2023-10-26":-607158792}, {"2023-10-26":-1555496414}, {"2023-10-26":607980365}]	[{"2023-10-26":7486933745350704840}, {"2023-10-26":-4010486314599977506}, {"2023-10-26":9220261474956750599}]	[{"2023-10-26":2.9687927e+38}, {"2023-10-26":8.3410634e+37}, {"2023-10-26":6.2639428e+37}]	[{"2023-10-26":1.267224790184319e+308}, {"2023-10-26":6.4014297140659457e+307}, {"2023-10-26":9.5071675599688732e+307}]	[{"2023-10-26":"2023-10-26"}, {"2023-10-26":"2023-10-26"}, {"2023-10-26":"2023-10-26"}]	[{"2023-10-26":"2023-10-26 03:16:52.981"}, {"2023-10-26":"2023-10-26 03:16:52.981"}, {"2023-10-26":"2023-10-26 03:16:52.981"}]	[{"2023-10-26 03:16:52.981":0}, {"2023-10-26 03:16:52.981":0}, {"2023-10-26 03:16:52.981":0}]	[{"2023-10-26 03:16:52.981":"8"}, {"2023-10-26 03:16:52.981":"8"}, {"2023-10-26 03:16:52.981":"3"}]	[{"2023-10-26 03:16:52.981":"d4"}, {"2023-10-26 03:16:52.981":"bb"}, {"2023-10-26 03:16:52.981":"51"}]	[{"2023-10-26 03:16:52.981":"9fe175ee0dbfeb26aebdb9cef3c481a847e0117f3b625727cb33e2aa02a09e83bdb45809ddb775e1363e66cfe1d011ec86b9"}, {"2023-10-26 03:16:52.981":"de9eb335a8505e578e2a6d6700e84c559b37c896459eabe61170312c43fcabd233e45ff139dca12609e7bb02b742dd8ee6df"}, {"2023-10-26 03:16:52.981":"2e2c6d1cec6b97f520924f3427d3345c3f18958c61f37cad1ec908e38dc3148fbe7499cb93854288384fde6f03e6ab48c981"}]	[{"2023-10-26 03:16:52.981":94497939.74}, {"2023-10-26 03:16:52.981":22111680.34}, {"2023-10-26 03:16:52.981":58323905.64}]	[{"2023-10-26 03:16:52.981":-25}, {"2023-10-26 03:16:52.981":80}, {"2023-10-26 03:16:52.981":46}]	[{"2023-10-26 03:16:52.981":16807}, {"2023-10-26 03:16:52.981":-8091}, {"2023-10-26 03:16:52.981":-9373}]	[{"2023-10-26 03:16:52.981":79710347}, {"2023-10-26 03:16:52.981":-2022250327}, {"2023-10-26 03:16:52.981":1091021962}]	[{"2023-10-26 03:16:52.981":1687715012832559350}, {"2023-10-26 03:16:52.981":-4894414771471660987}, {"2023-10-26 03:16:52.981":91609642581411284}]	[{"2023-10-26 03:16:52.981":3.1441367e+38}, {"2023-10-26 03:16:52.981":2.4318627e+37}, {"2023-10-26 03:16:52.981":1.7095067e+38}]	[{"2023-10-26 03:16:52.981":1.7942753750955034e+307}, {"2023-10-26 03:16:52.981":1.1413258779645193e+308}, {"2023-10-26 03:16:52.981":8.3555619080300626e+307}]	[{"2023-10-26 03:16:52.981":"2023-10-26"}, {"2023-10-26 03:16:52.981":"2023-10-26"}, {"2023-10-26 03:16:52.981":"2023-10-26"}]	[{"2023-10-26 03:16:52.981":"2023-10-26 03:16:52.981"}, {"2023-10-26 03:16:52.981":"2023-10-26 03:16:52.981"}, {"2023-10-26 03:16:52.981":"2023-10-26 03:16:52.981"}]
 3	[[1, 0, 0], [1, 0, 1], [1, 0, 1]]	[["c", "6", "4"], ["1", "e", "7"], ["7", "1", "d"]]	[["50", "a9", "b0"], ["02", "bf", "3a"], ["0a", "1d", "9a"]]	[["2df05ba6ce8661f0fb9bc88386a1ba67188e3e99a4142a0703e1cd8bdf041fbc20131e50bea2a9891498c638ebac842d3d46", "f6fecf3a4263ed8d4c5b63e4b3d9f4084b83835d9fba2046bf48d0a8068f2044c48271a1e9726741a09badea72c37cf18de3", "d23af3266db4eda12673f5c451d36343ba1cea00fbfeeff2165de40e834778eb96a1199cb523dd394b4f08824f6af2a7d894"], ["dc00e9b27b9540e170caf938058b547e8e21c7bb49ce5fa0fe929f18f4822e630dc26d4816d5f3d97e5dab8ebd4080b4f42a", "5d592c5c0464c800deeabbef436323571b3c0e57fcd12752895815e6c51e614ed6c32189f5cb5693327fb7c165cf9fa505ab", "7f56a4f147563724c01f3517a998d513943490b7d1186755bcfe559ab21095ea38151024166fe1c4bd79f0475f93d8a19eef"], ["ac298bd533c5eb30b7088f1c53ca7f3cd15bdcb831b0a658b5f533d9f666bf6a7af3237babe99cfb329ee06d52764f22476f", "c81b46d418692b6f9fd7376aa9ee3af8877c77f536696b40ac8e61dd928d30250a404ebf8617789527affeb5fa1524dc1357", "3ef1b08be0753d746743e97e88bd3870731a83e55b4004ec40e59bc12682ca23b03f87a11933600f01e257dde681e3c5262f"]]	[[5003753.23, 73570529.65, 14282493.86], [87800482.55, 8557989.22, 22459017.96], [30093982.75, 53169540.20, 90906195.89]]	[[-6, 40, -38], [-47, 89, -57], [24, -117, 32]]	[[15374, 26151, -20338], [15006, 32692, 7665], [12425, -30752, 15615]]	[[1796619104, 1023314741, -376282550], [156272155, 335095467, 15466270], [-252662041, -986864039, 2010198139]]	[[3055378941426239704, 5426937350987735345, 1844608114473848829], [-8415676173577023370, -6300775517173435538, -4056129460641262219], [-6450128590773189755, 5495688587826492233, 8080965061759130501]]	[[5.608788e+37, 3.1603907e+38, 8.4842156e+37], [1.9740397e+38, 2.4502789e+37, 3.0691853e+38], [1.8005423e+37, 5.2711493e+37, 2.0173921e+38]]	[[1.4813014163717616e+308, 1.2400728092921882e+308, 1.1860913398849248e+308], [1.0249154562992275e+308, 1.0560964131224829e+308, 8.0240375497340952e+307], [1.6458534692996229e+307, 2.9017044792708546e+307, 1.253587607736931e+308]]	[["2023-10-26", "2023-10-26", "2023-10-26"], ["2023-10-26", "2023-10-26", "2023-10-26"], ["2023-10-26", "2023-10-26", "2023-10-26"]]	[["2023-10-26 03:16:52.984", "2023-10-26 03:16:52.984", "2023-10-26 03:16:52.984"], ["2023-10-26 03:16:52.984", "2023-10-26 03:16:52.984", "2023-10-26 03:16:52.984"], ["2023-10-26 03:16:52.984", "2023-10-26 03:16:52.984", "2023-10-26 03:16:52.984"]]	[{1:1}, {1:0}, {0:1, 1:1}]	[{0:"9", 1:"8"}, {0:"2", 1:"e"}, {1:"4"}]	[{0:"91", 1:"10"}, {0:"bf", 1:"6b"}, {1:"be"}]	[{1:"1f071fcffde93ecbb256e1a78ee2482abbd35ffbf92266fe1d1a090f1d36a62b341f8b3b7b66a8e282a483c213db1f529261"}, {0:"9b931b1329fd10d8764749709584ab69229b622571f3ad7bc1f038d9ab8b253cf63c8b57f01de5c212ba2bd3e32d5d312c22"}, {0:"7ebf35c4f1581d34234ac5fe44edcf466fbc9be9a4511bb550e55801905d59cb17e2548d542928594ea68d7e3d6a9c6fe014", 1:"a2ba28a58b5e0b83b285b5b55ed6bac4a011e45fb7f9be164bc25943003b5a77058c8af819ff06bc2411c28b4f724dcfbc1b"}]	[{0:19839754.05, 1:89600377.82}, {0:71402873.04, 1:43020241.84}, {0:33521290.44}]	[{0:68, 1:-18}, {1:-15}, {0:110, 1:43}]	[{0:22055, 1:15480}, {0:756}, {0:32569, 1:30857}]	[{0:-1097067759, 1:1537489392}, {0:1186620471}, {0:-515408435, 1:-1591697674}]	[{0:-252631566868171671, 1:-810351861392731413}, {1:-2777491830756081712}, {0:7329068750388563594, 1:-1211236243710043924}]	[{0:2.5761776e+38, 1:4.3408788e+37}, {0:2.2513081e+38}, {0:3.9311245e+37, 1:2.7582459e+38}]	[{1:7.1991443245270281e+307}, {0:5.6962910203346541e+307}, {0:1.3739388307135712e+308, 1:1.5518702176738732e+308}]	[{0:"2023-10-26", 1:"2023-10-26"}, {1:"2023-10-26"}, {0:"2023-10-26", 1:"2023-10-26"}]	[{0:"2023-10-26 03:16:52.984", 1:"2023-10-26 03:16:52.984"}, {0:"2023-10-26 03:16:52.984", 1:"2023-10-26 03:16:52.984"}, {0:"2023-10-26 03:16:52.984", 1:"2023-10-26 03:16:52.984"}]	[{"d":1, "e":1, "a":0}, {"8":0, "d":1, "c":1}, {"9":0, "e":1, "d":1}]	[{"d":"d", "b":"d", "f":"9"}, {"d":"2", "0":"d", "6":"f"}, {"f":"9", "4":"0", "a":"b"}]	[{"f":"bd", "3":"82", "0":"2b"}, {"2":"8c", "7":"78", "4":"42"}, {"1":"45", "c":"e0", "d":"45"}]	[{"d":"559469bc832f2c592527d56f3269f963025706c3042554dc85e6469b71d06c5e68ae7893c0ca453ad4ee2f30b2412dc7a139", "7":"c0a84d74cce1ede358b7a8d4dc58c3c49df9289e77fc2964504700d9dea63344b9ae70bf798472f229042c3b71faaaf98303", "2":"c888ad2986f919a14e127de52f86ca06d88678a510e46e7e1b7a4af77ed08c2ea094c893e8c5f6663ceab1258bff655656ba"}, {"2":"5aaa2ac95ec3e5c05e8216d9d38dce81908f61b827dfecfcf2ea8738f375162fe059c76aacce650cf7216e4777c226092e69", "2":"b4a98a580f42ab1519b6957c2056bb5fa3ee574742e8d9aa3f451f68f8770a7b74001942f4151f6567cfbdfc3e902242d393", "7":"ccc907e9c6fdec74712ac8c763fc40bdf1a8c743ecedb48f5748008db034bfecfb2f3655d3df99faa950109b9a49ec9e1011"}, {"2":"c31a804695d5ebb66b8496c9b38528b9bf37f7db8d1acf09ccde8fa83b00ed0b036d533e36e82a3552395dce378721a800d1", "e":"1724fc9a583027d7f11cd3efeca5ab437769d0267a939e0fd04936f3d2712502f1b4350090c730bf6b476ee8c6c14da31f60", "f":"57607195e62b78a3b0c578661c59b85f3867dc7e91c72cb142919b4fe17d77315825d41cd2d6a75887fcda100b8038bf4621"}]	[{"b":76553637.24, "b":47833250.44, "c":84313986.40}, {"9":59618808.73, "f":5129841.99, "2":28334633.75}, {"4":5138048.17, "e":42396061.69, "e":783279.24}]	[{"e":42, "d":-41, "2":63}, {"0":-114, "a":-106, "4":-70}, {"7":117, "8":62, "3":75}]	[{"3":-14362, "4":5735, "2":6901}, {"4":-28862, "a":5282, "9":24752}, {"c":-16971, "1":28151, "e":28833}]	[{"7":-384669457, "6":133341996, "7":-1788558557}, {"7":-1082793097, "5":-1348390022, "7":39540163}, {"4":952566277, "5":299721249, "9":-130464146}]	[{"c":-645918960611412183, "9":8614726590096659219, "a":-4811786611451268605}, {"9":-4803426571288018750, "0":7937988270821110422, "3":-8552779640056569770}, {"d":-6011898865730652301, "8":7094587763368443709, "c":-6346167729377060318}]	[{"f":2.7666876e+38, "3":3.3488213e+38, "3":3.2617023e+38}, {"b":1.0549502e+38, "d":7.311188e+37, "e":3.1130286e+38}, {"5":1.4492938e+38, "e":8.9189087e+37, "a":2.7725249e+38}]	[{"5":3.6669447710342061e+307, "f":3.6333552809806644e+306, "4":1.0680168131382127e+308}, {"c":9.50457206759392e+306, "c":7.92261880030454e+307, "5":1.0455456973639938e+308}, {"2":1.1054050778372218e+307, "a":9.0907638583019416e+307, "7":1.0811585934507172e+308}]	[{"d":"2023-10-26", "e":"2023-10-26", "a":"2023-10-26"}, {"9":"2023-10-26", "0":"2023-10-26", "c":"2023-10-26"}, {"4":"2023-10-26", "7":"2023-10-26", "c":"2023-10-26"}]	[{"3":"2023-10-26 03:16:52.984", "f":"2023-10-26 03:16:52.984", "b":"2023-10-26 03:16:52.984"}, {"3":"2023-10-26 03:16:52.985", "6":"2023-10-26 03:16:52.984", "4":"2023-10-26 03:16:52.984"}, {"b":"2023-10-26 03:16:52.985", "9":"2023-10-26 03:16:52.985", "0":"2023-10-26 03:16:52.985"}]	[{"aa":0, "c9":0, "ad":1}, {"8d":1, "41":1, "ab":0}, {"c8":1, "b1":0, "1a":1}]	[{"0f":"e", "77":"5", "6d":"8"}, {"dc":"f", "f6":"4", "5a":"1"}, {"88":"d", "1f":"f", "01":"2"}]	[{"4a":"99", "0e":"7f", "5b":"05"}, {"71":"df", "e6":"0d", "72":"7c"}, {"6b":"f2", "72":"6f", "70":"97"}]	[{"f8":"fdfae432c68a3eeb5f5bae4a2040ff8f162aa2c71ccdef6c0bca7578961ebbe1cf1399307e55e005e4a52e64b423d9dccde9", "81":"e9c8352dff50b7ce08e1a6d8f5c41a21865a1028d2a7c0a52f972c1678e3504958e1cf0a5b9ac8b6b22ea904e8ca0deaee2a", "a2":"f0865b0086dd2c29bfc2b34e3a4d2873ad8fc6d2d27fff1a20c28baa42a175b6abc237e72b16196a10d864cdcd9dded342a1"}, {"16":"d8ced66fbfd05b848088a1cf253b3d775a05d6c1df4860c46609de574f2404b52ae10e9b2c5ced7a6a689294cacc597cf4fb", "76":"9b40006c890b57fdd32ef53719c93eb4163ad1b2255966620fe46e7c51f109ca56188c99b6ac4c6b57544d06376139fed4e7", "da":"750091647e21cf3f1a9ce1c8291e5fc2b0284b78c7bc02cf719ce19c50229b72aa59a0f6e2a0828bbc2f0aa54f7cb3ce8a90"}, {"18":"ae94afbb7cb711c1b5375842000f16bd812d88acadb74d4e63bcb3c1f44d6e8b57faf214641ac22f5feaab5ea68b08defe54", "c9":"7b92d9de5ac9ed64fab32fad0e92af79deddec23c97bb1fe6380761576bf28ffe34fe3405a8a4b74b059c54f10a9a36c4f2c", "10":"3d63fbebd3c5bfa50212aca5aeec7d9d2b5788ea8b09a5d7873bd3dd29d07e58d12a85673f3b70adc0e23126dd5b30b85841"}]	[{"f5":92472285.08, "c9":16515468.97, "90":80306144.06}, {"cb":77785434.65, "fc":51830427.17, "95":93907933.88}, {"f8":25371963.60, "fc":64226710.19, "18":5326712.20}]	[{"08":-112, "10":-120, "1d":-6}, {"8a":1, "fa":-14, "90":93}, {"a9":-31, "43":-39, "a1":-43}]	[{"b0":6897, "dc":-15993, "2d":-16602}, {"95":-6178, "8e":24863, "3e":-13460}, {"5c":-22981, "a4":-23773, "9d":-23930}]	[{"d4":-1321130084, "e6":-1500927045, "2b":-455607275}, {"9a":-1289540032, "70":-1996773624, "3e":256067347}, {"9a":1381550835, "a3":-1116577372, "76":2022998110}]	[{"fb":-608184207064354145, "ff":3494598411731077659, "5f":-8222350788773010574}, {"91":-1737097448609502969, "ab":8510899815098152879, "e2":-1268475157311976784}, {"06":2357523518723177529, "ed":8086699582886906415, "b5":8314635499142654791}]	[{"3c":3.1540135e+38, "f9":2.3966902e+38, "81":1.3954178e+38}, {"df":1.4702904e+38, "f0":1.9401873e+37, "bc":2.1363367e+38}, {"3b":1.6773348e+38, "9a":2.0904364e+38, "e2":5.7570352e+37}]	[{"2c":1.1205062274338892e+308, "32":1.2369010198246723e+307, "5a":1.7803785231148744e+308}, {"05":3.5857871839466424e+307, "08":1.0428645758077941e+306, "87":8.3481834231163591e+307}, {"b5":7.4056059990491561e+306, "20":1.0426197328991832e+308, "e4":4.1637764990124554e+307}]	[{"5c":"2023-10-26", "3e":"2023-10-26", "f7":"2023-10-26"}, {"c8":"2023-10-26", "cf":"2023-10-26", "c5":"2023-10-26"}, {"ca":"2023-10-26", "bd":"2023-10-26", "3a":"2023-10-26"}]	[{"f2":"2023-10-26 03:16:52.985", "99":"2023-10-26 03:16:52.985", "b4":"2023-10-26 03:16:52.985"}, {"cd":"2023-10-26 03:16:52.985", "f9":"2023-10-26 03:16:52.985", "0d":"2023-10-26 03:16:52.985"}, {"4e":"2023-10-26 03:16:52.985", "b4":"2023-10-26 03:16:52.985", "39":"2023-10-26 03:16:52.985"}]	[{"6d03b441521bc76d32675ded7fc3651f0b22c22204fd80daea3f45720c4866dd9938601920b484db882ba24c80228daadc4a":1, "76ac0fcfe09c559ce5dd3bd033afc13036f910129dc3ef202d4cfa5dce6dcfcd04d59cb039e6cd3a3df267bc75020b88edfa":0, "f6f4fddff33988497c7bceb9ea7f0b146e50854ba58e992ffff68f1281cf5cd9972d89b1aa6d2dddcdc8dcf98217bee44d13":1}, {"f5b45e7e6fc27230fa705a704f7f8ccc388b00b64bda787a294e87a3ed6b862f215f10b2c8f6700eedd0e044d2918e734591":0, "c6e5164ee9fe855c18635a516c8a6b493878e32ee768416d18a2258f3a749f0e150ee418f2889b29bc957f901706055361e7":0, "7e70383a81d19661ff3b7fa2d8b9d22e9a96908ef2aa3749e1cf9c6243501ce5ca5054901d231096fc2e0ac5e056582a8a8d":1}, {"a7fe72d0f5f86b832b7c73e6e0de3bb7f74be747a3bf818e1262809c0ae66ee11eda875f1c2451666e4031ef1256be323b06":0, "7ebae5cda9655ca047ec3c98ecad9ec142f3d1bb0d46c074f604a0b8a8ba9d38f8dee45ef40c1c31b69a0e56a307ac99904b":1, "8e23b51f8e04c7e11139bf06ba550ba7ceef467790ea2c353851aeb2f878a9928999627f876b263bed4f3a0aa066e914a033":1}]	[{"b4a72d1e491856c0669a5e86aa184e5e431e8ab12fc8e035c3090ca987503b6a84fb4b49c58c5f9185a21327277ae5ece5fa":"2", "3df4b400ecfb9c892702a35c0d901ed858735fc5088c13ecc6b1bf3a408af58b16c213445c99933c8ae43de160a78f553536":"b", "5b9a6483c347f7d29c162f5deac494e51b93f0f8636d467b0c6ef5805a589710d929ca714f7f3abb6f48186025362786299b":"9"}, {"84d703145ce6b8c1b00c5b3d8c4dfd30435613b8c8f4de8f43925e7eb83f47aefb450b7cd094ef7b99ab5afad577c7b4b700":"a", "dcfd108ef6bc3b98f8023f2ce995bd332e91bb832f7bdbeb6cf086a743f21a26c3c0e9de365788df2ab4902e952fd89de70a":"0", "c911e46a1e05fee5702c4122f73c1c3670dd001db64079c9a94e1e0226210553185808d3f2daf46f08ad65b4f85eb2d253f3":"1"}, {"929a056c34bc030f284344886cab10ac50cd17d8f056a1354f8e6c2b16c26a845deda5138f6e9fec449d163a952d3731ac44":"a", "a56c9753cd88d4521788569e0d52a52e4f6133e0c92fccf61972d816eb5a648c3deb38e34742203e86a5735aaf400dde907d":"6", "44174b85018833bd5f669b2c686e414d576f100a62e167f6c889f473467a6d63532de3e6c437d925ff8dd959ad548f843006":"4"}]	[{"ac47c92898f102464c5a7f09583bc653b14f884d873aedb3b77fa64aeefaa624047f5e50c9cde50a0883f24ceeef72eafa97":"03", "1d381e353bc03696febdc7630aa1751ad115de4591c3a6ac655d777bb847a9c1ce76264e2786c5faf7329d52f9f5bfe1694b":"88", "5dabf208daa7f524b18dcadab776a52d37b0a9353b69649dbca9639e568b4a9cf930e7edbcecd50a2dc73e31cf936894780f":"72"}, {"3636bddd9a41998957997bfc9fdf2e52105b2b91b8ca7a2306a5bdaec5b02b052ea07ae1b4a84b3c22efa4a51aa11a625b2b":"a2", "9f6bdb1606975a934fc58fc9cacd08f927f22e6bee36c070d03be7ea5984d9088e15a474c3ca81993f965a93b7960063cd3a":"5a", "c33748b749866054dcec141c7faa72f9748b34e6e872dc0e46a9fae92cf3c7c570e29b015e38ad6b53d38aff2c67bd521243":"91"}, {"bb34390928ee05060e7889e0831acd63694a908de1782765f3c47342f52ecb5a848c91d48d858ab7e2a122227919aca736bf":"76", "46d565b1987a7471986a42898d55d0fa15e693e4b89aa137c75fc9ef1d7c827bdd662f04dc2d9ee46470d83def62dfc52996":"0f", "77a33d59616f59f025aea7b78dd7641e7edc3a2ff3326ec4a304c80f283d775516579a3ddf5e48556e0823ec2c0f0477546d":"2a"}]	[{"9ca01a116e2963222055d39aaeb307fa0ca35fa2f9325145d732ea3d0d0f91ab97a2b40ca123306ccc28e310ee7dfc39d45d":"aefe29a082d3e3beb997ca4c64704b024f4b1dee0d651715ba13304c8ea5fe12414eb4b644ecab0b936e301a2da7136b0c65", "46af35f99d5876a9ab465f781ba7cba2921a3dfc25f379864f71d9c89551bebd66f0b767cff1a4d6c78735cdfe584cf55b91":"a4ce8fb2af3ce081120976961368f83611c1c1ac67d6aa7905d5bb194650a220a602d9f530303b84ee3d59e26da0ea373b6b", "430ba28215445e0af10df01fdb242b22a78a10e6280fee944e96b0463848759468410e4c9129a013ecc4c236f098e9a93bfc":"bab6d6afbb64b28234875d90cd2cfca05f00bd3d95124e4cc63db5c6533f20e6d7fa0a1cefbf5ac203831942b5a99dad4725"}, {"5ed5dd2c2b6b67b7a0bfb3d81a41b53ceb5114c578aa3ec3e61fca7816ad737fc86c67d3dc8d658055fae93ed8dbe61c29e5":"6e54af5581e950237c33fb57ba5ed118caddbedd70e834abe75b525fe1531942be0d970450b753a5308175bd037b7f586731", "d943ac739b0df976db367f76e115a30d8ddd3da1492c25338d752050440f08a46f026e79e157c5180cf0d88f2d2a05a477dd":"c5eb8f1756dd3135171751cbd6bac71ec1d1f752e8a8d70da209baabadfac037c84df039092dcef8302bac2d10f146ca2ade", "84b547a3ef71f8c253d0a587d0c2f65518e36cd5454aa5a490fc25efc047bd43f20f81f18bf42702d38b68afc90ffb56a3e3":"660c90e47575b4a2b487c97595d112db56f35ea1d2ca3bf4e086ca5e4fd36bb3beeb9f9b8eec57ed7a07966a1e21d5a23d2a"}, {"f19fec4d95f88df5d2fca890ea554b9fe93bf6ca2987e2bcb0b143eabb04b5706872942176f0d40121e4f18624e3d11e1566":"aee8be5f9e4598931f653765c92dbd4c0986dec6800e10c49f59985a5a8b1e1d658af7719f339cd4f9e280807d72a1f90d31", "7bf57ac0a7c2b23fa5cba4b5fbedca20bb4212e5042de2ae8241ed4ae6d9c23b3085be41bd461f4ed1ecdedd1f5fee2f4844":"086750dd1d29c4a7f0880037eff59a90752a08f0e92f1d097aa828716753a6a77f9a9e5776ae6c90301e9167ce6faf1a01d4", "56e4209346c6b6155319909eae1cbb450ea903331334c1b988c69cfbd863d83886c083f1941a38e12188068f72b01738e6a9":"10aa69e4272f32a12b26863a2159666b3cc97071abfe8bd5598eee36391665ac42b82db7dfe695f9ce22c13faff6e96ed800"}]	[{"133b6956d9af07ab12202545424bf0ac48b218098ad6911ed3715ba27d115f481ea29c0edd70508a7581e0eb9d3eb1661523":36677207.77, "13de0989cd0be6715f6c7f891769907b634f873e8a3ab5aecced6b5dccf742853923f3f23ad5f6705a24f7180edd9abfe504":8510328.74, "35981245f8f07c4ce893fab6bde28c9ff0f3058d9c70d4b860b05c4a514496359d9967e70c6c68f9c045f9dd21baa259318a":35365816.89}, {"485a4e37c73c36dba9ef5fa6e9c0f17e4180eaaabcc6e89cc74b46481583323f03f08c440f824067bb68c51019257b0522f6":76755283.04, "ec7d2877b667827bc7a2cf4f778e0aec60572c630c62c21f4a91c266ec4d3d6063d5a0a89f1f5d612af094aaf950a6263a3d":9825650.93, "978cf4596747d30d790643664aa8d5233fb62245021406ed82edcb597bbaa3664111cacaa64dbaefd7d96ed369d08d94fbeb":72149432.40}, {"02b225bb41bf5bcb89e0bb6e81a61c3a001bb78c06686eb85e0144608965219c9d395c380ad9ad673f71a0bb9c7908d0fe05":9901010.39, "3d50822b5182b42dce839aedc4e6f41989b59ee3e2a1d546a4a2aea8557308b33d2382272879d681576eb948ddbefd429d57":40618693.07, "d37716a8b12985c581460afeb0b29b405e351495201cb9002582ae98ba847ca9d435391eba692084866861e1f3d76ff86cb0":39336421.28}]	[{"bbfe359184a41bde7431e5a617571c482987295f4ad421edcd74a421718b685eecc4b26d078ce747047153c49027dac0d12c":75, "b3e557849463b93ffa5b93f3f308aee04bbe73cb364cbc9af664ee50494eb21c6a882d09f5eee07745e027e9734187b1608e":-112, "788c338573aa310f403157ad9629ac8404c26daf341a9f654a536c01838da685f5924affdefbdec0b43fce3d6c529e398a3b":-59}, {"f9c1330c1b70165cb01496f46cffd0244b463ce6a97d65468c3ab58fd36dbfc7b6a8812e5f1ccaea82c73cc48e5ccdf8ac42":-118, "06eae6706b5484f85ae9cb6cebb1d4514d9e7c50cd1407be8c662993e5da8d42389152d3951794f5edee87bfe798ea9812a9":-3, "3694461166318ac4b96822967c61169a2bfcb41de9531cd4ee4bad5ad43bbe5e173eca791f1b87528010117b22cfc2938aff":-124}, {"978df7c86b4e373f9b8e97e60f4efcc66c99b8e232243dcaac7d203117a3c3d280e86e940016996c794be66291149aad4859":-64, "34f3638a75090afc45b32b944494c78a5852c4eb9d03b74fdd630d254b94bbe6afffc210cea635df3a513c67f27294a427e8":-82, "90c9153100168ea7a9fbd379359ed0ddb7d4cdca417bc6c93acb7b6e39ec0c863e30871b0f8f9c3db11eefcc4ca942209c1a":20}]	[{"23d7f18d422807cfe00b4b1d4218eef270e51f193ac987b895a77f2e2031cf1d3e7a121cc9773f70b56ff76c1d8bfbb8fa4c":12564, "22af05473c8307eab37b2d09c01e78aab3945871469d7cebdea9d09782eb939c8f72413b01fdd69275a7932962dce38acca0":24000, "ccdbfc81b15084a0939d38c45a90fb049eb13349c6a97788a66b2e2c7e2bdab05495962f6288c1f084983c3003fd6fd80454":-28955}, {"1cbca3d307bf65d4d1e39273b82997d9ff0a42633b88fe13955b3dec0e271858516080bfd116e41b52dce0fc00f30d58200d":-3339, "f3435eee4b49a91751aafabc0db198f1c2cb4531c9e737b1ae2e2ab5d6f42e2afd8ebaaf595e97d59e5324a2be21ffed14a3":28437, "c88d6eb295c3739ab16bcb7806c73a3c3e9ce2d75c11fe92fbb48505e69e5f1217e69db4a3bf01fce44b66d4cd48248346fe":18442}, {"432f3b73009c40f07fb8fd27c23dce03bfe1055fc8baa9c57801c54b0efe6cb20ab41f523d03201b5a7ddecff8e095cabc70":-31746, "b212c532e8123fea1e64891c579bb321369edf69b56455a71d31e2495245fce2e1bfdacf88387427e5f3a1eee3bf5aceed47":-1259, "61e5fd0d988bc9ac7871a7f34326fcd7e6525223b75e392eb5537245ce40ac4234283714cf5de267951be98550d0e702b8b0":12002}]	[{"4b274d8f56edf0e03e4bc8375f79b1708c98cbbf4569a04004bf7de4f25483253375a500a29c4a5addd24019240413997145":-1192648923, "db650e630a12d14b43fc178f7ad8282ff5d0d7ab26b141ae51178a0abc62911ef4dba0cd56142686e7ecf5b77a27ad0ca930":1124565633, "72da24f6b7fe895bb84898aaf316d02998dce5e3209dae930c5800702baa464b34a76bd42765d7bc84d39c6a95f9c77d1b79":-573762681}, {"a6657202fdd16b0f75fd7774d3b2243a662defc3336c4072747b1639ab5e3358b0f548b3d0bfdd4ad500261287fa2bea40b9":697772338, "2b35c3dfd764f62e5950497bddc130fb8a0335961fb7769ef6c5fadd2f7de57e3f925a66847d6ec08b01dab1ea387047fea6":2093310859, "36c2c5dfd0b55d98bfacb85e1ee5985bf402875176d1b05a990c3a15300623f1de259660f817fd11cbc445b22f9467dfffa3":-1421925028}, {"a299e18577ddc5de6a86b1b8a79d8a1555fd2f1c4dddb0973db15f13a9ffcd8cd63f4229ef98c1424f76ca922590fddc3d25":-892535469, "e42c8354b29b0e7010e7aba349f817685cdee8fd80de3ca232284d84b6dcd0236a51574a05cc63e0cf52d61d0205dc4e6b20":1688349931, "8fe5176a80280577dbc88b463294542085d71d521e5932662b6219fd735e53d7485867842ea9dbb7a2830b04f1685a5f4d15":1267219382}]	[{"70014ab48570d90b12f928a7920d23db7cb6f96ac4a21aa37931205a5c1962a11ec178d8a4adf6a2bbf82adf46c3654a15e2":-5950742631241271482, "19640c415a78275accb41fb91bd93a091ff7112fb102fe9f3a6ca7bc7e93f930386dc36311af4345dbcdc2d0f60161f86c77":3490627268062090718, "1ab000adb5e8033957d2107a62fa0cf623c69232b38283f639d1f08c936f1989369d9a9ab91cb15a607a10e7fb83c816cf0c":5960361510265206703}, {"64332073565c17b7071b9a037a62d978a2c6a12d254af65c9185af1d337fa7539485f996eaf7362b054ccff7e2f1b27f62ae":7552640314076087414, "67ef7d4c905042f1bae532c2dc3c5d765d4032e3e6130e1aae53e4a2997b4b94c75de7efe98b01ac8945ca49098a92df0483":-4586467585305815660, "e1602d2edec610e0a9cf158391bdf73a6de61c3aba1d32f2d0bc47e091e1ce60526c4ce155f4c91a7bec5505720922791c56":-1556604219310661752}, {"a58ba24c559ec9162f801dbe8cb6c86aacca03635da2f24d5294ad2ebe19f7c1441a6b4de72410854607ac1db34c504018e0":8185816873556859569, "f8b4a829bbdf3698aa4b9b3ebd38dbb433d1e562b0f026ac7a93943008b6e01c225ea6c22fa1e3a8142b38e789c69e98409e":2354512397123676805, "38ab811189b5788ccad597c185d9b4309027a21c72ab5c452e5131a6601d1542376510caec0c7b5260cac8d6c9c037f4cb6d":4194647391995991705}]	[{"bbe9db6012010ecd236bd8a9ef3cf60a323b24cd72a0c649847552906d926bf7417628fd805d650e6de7cddf8c078ec045a2":1.0888399e+38, "dfbc82812524edb0b6abd410ae7e8742cf3fe96fb97f51cb8a706ba13eae07cfed79d580c1518262211f2c3c6a7d555992e0":3.3393484e+37, "14fde2d3bb668ea307defb6ec50f49ed42c72369b1a22dd8131c849a39442e40f40a62a022e1ec3909c244c6db3995c620d8":9.542224e+36}, {"e7768685c538e23b47cb8c391a2cc8b381df550c732e4491a17f3ff7ea05675fa56037b6fe286b1f526bc26ef464b4de5edf":1.842277e+38, "29c6db57fed052c453edbfcc178465dbe2d3d304f8195ef7def5eb49fa3ae813a4f937e149ce25ba57063f7d062c7422edf0":2.2982194e+37, "7f7dcded8a9c7155aef8041723304566724f845a3db1debe31e07a1f907a570a8f7c0776b4faeee6aa197c1cda1d3033f0dd":9.4293949e+36}, {"661aa7d96730476eb75afee490e394711c10be3d78cc55506f052339d148508619401725e0b2495fae7d3ab348d9289a66bb":2.1364529e+38, "d4aabe08b69a3bccbf620959dea42901cfcc7a7c3dce2bcb13cbe5f2d9a28be4fc6ecbfbb1bf0f9a405436d7f902e4bcd98c":2.662585e+36, "b8ba54e6d040134f6ff790dd72544eebaf351f2812a0055c20b9a4c344468ce8f1434fa0065939a2d018d23b0b1bb66c1a17":1.8345417e+38}]	[{"70c20873c0602262c98dce606c627f92936ab4babd37d133b04af21afde8f72db4cf32e9ab4f4a0052f005ea54f09e6444fc":1.7900983236166278e+308, "edcb799889a4b17eaa395b44486f1c33f86e3da09f83a14780a8e018aea0895a668eb36a81dd3d9dc0c5218209bcd3650c2f":1.1268973687037591e+308, "9ddae0ea0a1cc65878f039e7026e0cf49c9546c597710efd1ef0ef3b59dc0daac2de9d1d84d0cc784714e73e6df06221cc19":1.0480588805783237e+308}, {"42bbbed63b57fc198c0af258b4ed4b8c9bc08123cf26856a4b7a1266f13aabc3ccec6f3aed13c4b2a21a82e53a41b0151e44":9.1775546455049035e+307, "ee5c0b4da9bcb4b044377e7e14089d0a186056153cb5216dc7984b0632538c6c366f77c54704758cf4a05bfa3fdac3d4a9eb":1.0735122940557393e+308, "81a7a4ff4546f65465fee10070739448285a4c7699993a2870c92c5fc9ac117b338f6dcfe012c562520e2cc6e9bea44aa895":1.2517900361791551e+308}, {"7e3099d104abb78e4906ef47da58ee7f25495e8fa38136deaf2356212a3f8f899dbad73c3647af8a226489dff38d5e06df38":1.3691151822793197e+308, "28c751cf443f04afde6dfcc1eac0fccc2b857cfa30fe0067959acb285eda90312a29f7edf4678e4e3569bc62edf441317cb5":3.2132892998460623e+307, "f82bcf82da89d51287ac352a950da239c5bd4e808b7e80d13b47ce14378d6dce5271acb718e8d2688513dd092fb630075a58":5.0389567286009372e+306}]	[{"5be9708dc59aecbe9dd4effb81d18bd7c338619ba306b502b7f9f59dde1f4b47e7ef1bb12e154099abc929efc4ad911b33d3":"2023-10-26", "64504e6d1241985e9e8379b38a767e05202ec583f98569fb895e4599ee04eacaa50383906cd7ab4e5c43cb47f15cead43bb5":"2023-10-26", "ac53cba478401094d41a89788ea8b4df8483dd1101db9daee2d26b25ce90fde88d99453820551effbd8f75c24bc6817d5134":"2023-10-26"}, {"f2e10c2efe9cc5cef099660589d1c1b7f8b60366d520141d719f6a3c6310d0f9081ad45bac87f4c8df706cc8759b133898e5":"2023-10-26", "f39477b56d32f9558eada981e826205f5573247c27916a9cef285c82a6b5522f76032b3c31f70cc61134c0bbe67b9b45d32c":"2023-10-26", "af605a723664086a66f1db78c25e1eef1816b724968d6ddac4ca1dbd4f2f6bcdf60997c840c96ac2b5a6ed2d90515b795643":"2023-10-26"}, {"1de84d8a6cd7df5c868f99f6cd6c9eef3e36a9a017d69ef842ee7da5a85a47cd66e0cc100cda3600eb18f19b0ab1e8e9b3b2":"2023-10-26", "efe16a69969ed6bb3984dca0fcb7ac87aa5a17107e635bb112fd0755f09c6a49b5037f5b153f928877b1169f8b8f5d44c610":"2023-10-26", "1901737e6e84e04bd96dd4092ffe7b8ca0eda3fb6fd065400e33c90043c065a84c8b3613a9bbee973b89f1502b5d146e82ca":"2023-10-26"}]	[{"237b3af9f2670d10f23322cabce5e9665dacefe254e3086777eb963ad867836a0cd49ee7eba49cbefeae32f6c8296c99f334":"2023-10-26 03:16:52.986", "e054ff38281ac463b01b03ad40c804b635d8fab910a3c4b7bd110783ec1324894ed6948623d3c23bacacbcc4ad55d54001e1":"2023-10-26 03:16:52.986", "4dce633458f86ae31bf4f23f73d94c45ef7d937b106670c06008ce0d6cb3367e2d129b158abeb2c8b0f26f65f02fb00a16f7":"2023-10-26 03:16:52.986"}, {"aade785e783bf9494bb551f6f7c1fe17abf543cca900f896eb057a6eda9148cd5f81f64e3a4fa9929105802fc1d96a2df335":"2023-10-26 03:16:52.986", "acfad66e63d67104c6bd5a00d6bea693848a0912639a5b1987ba8605a1be97a7d6116b01f5ed4cf00a57fb18e3f9b064c201":"2023-10-26 03:16:52.986", "f0e6f8b1b86ce19222a3eb7148bd3bac67fcb25c4a6bbd3d211e1665e8f4b359a69e7a126aa2cf2652551d5dc03a2ba2ecb5":"2023-10-26 03:16:52.986"}, {"aac1382a9247e1b64cbef663536564f449bf399be4e90740085c94f5cdbc9d8e6a50189ee86f22278a86aa07130ba584b3d4":"2023-10-26 03:16:52.986", "a8fd5933e9cfa1b478a788731648ef82e6e5931f49216375407b975993510288bec28a988989aa8469e07795b1446141c675":"2023-10-26 03:16:52.986", "413fd67f896584e2ec84349709e174ae04ba155f989587fd29f030144638601f8c3de22160d6ab951e582c024c0321f3c5a3":"2023-10-26 03:16:52.986"}]	[{8520103.02:1, 33150200.74:1, 64649148.92:0}, {51631274.89:1, 7665359.29:1, 74154921.82:0}, {71907927.47:0, 56084186.53:1, 31494018.64:1}]	[{5336148.97:"5", 26725167.11:"d", 13600113.74:"5"}, {38997940.87:"6", 19527932.05:"1", 31385141.45:"f"}, {66323600.16:"b", 41248379.50:"4", 72679263.13:"7"}]	[{48443164.77:"56", 82400542.84:"02", 98049726.33:"b7"}, {80657142.08:"9d", 48405401.90:"98", 46364070.08:"be"}, {33702681.98:"3d", 47690856.30:"17", 24479400.81:"30"}]	[{14042759.11:"4e683834ac5b6d20825355812da724a2cb4f7b98a80883c213c2e3f9fb83cd503a38d968bbe6fe30dcc4acdab8a4e82c94c0", 63264142.85:"baedd01ce4a34217f33085d68fc70b1c539921ec2df527bcbaeba0e961740dc38023bd068c21b63ab2d3d255cca7dfd92ee1", 46070761.42:"5ebd78416ee5ebdca2c53f5b67d8f918b18ccf60d18992c99ac127e1ffef5bcdb1a67f5bfee00c881efb453300c72c0460b1"}, {63380351.74:"9aa41c068c6bd13ec3968e8d3f74b5a47092f0fb8372af70c72e6184400663c24101ee33d36165cd36624378e3946b5f34dd", 82133511.80:"22c0e53ebccd359752ddcefb53cd9aa9ebca57eda964886a69851165a3f4b97438662a6b346a20c47385ce2c8a6d26cf4e47", 33791888.61:"45391355e0021e37f4bfe16d5c67deba662770bfe733812da444d636e57236e73592eb5db165a9f260291e057e0efc5f86c1"}, {14967538.75:"638a314ee816921d09a89807768c43691374baac6b5bce936f15468b03e0d103a6eb459a10ee0be08c92e3e719dbf149a059", 59427645.95:"9546c625ae7e16470a7ffe56d45287350df80353cb78e9fe54685f397ed44e214f1d908552ee64f0cfc81fe7621d4ec36d93", 2899872.81:"d20eb7dc287c90b656ba6067e8e814b70467173008dd876011cb38cdaf6613871729efa92bd314f1a1581328c85634deafe5"}]	[{9313831.01:1735726.03, 81403060.05:99773767.76, 43635744.84:95901731.93}, {87353883.54:65663935.89, 51135625.07:612161.18, 19739848.64:7527457.45}, {99425828.71:98097524.38, 10470185.37:29911594.37, 44741404.60:77062364.29}]	[{3595691.52:-70, 66940420.41:-53, 13993955.43:-70}, {47204475.14:-39, 80932815.88:14, 30404573.27:-70}, {27276469.64:-55, 65101288.88:-103, 30219469.92:82}]	[{29652008.79:12324, 59112817.73:-5050, 31518931.55:-5605}, {44764201.08:-7075, 34191039.74:25558, 98502040.88:-18485}, {9856886.42:-26913, 5446933.12:17864, 6653154.84:25006}]	[{92620322.48:305170953, 64372127.91:-2085751959, 48308396.22:97356289}, {63190718.36:1553515184, 97015821.73:1656123681, 49939729.71:-2023510759}, {46726131.96:-1129044688, 12865964.57:-59243749, 14103690.56:-340019991}]	[{90102715.52:6218132939653208622, 62451212.28:-531521974849018784, 21944067.54:-5131323368421038817}, {91426871.76:3407725703630855070, 67501104.98:-2187961339337430148, 76836353.44:-7113923712276511813}, {42041991.24:-5790911582408512237, 98966670.05:9138304141769356305, 24142688.10:-523206532579812793}]	[{5048737.60:6.7502303e+37, 28196906.87:6.597664e+37, 54093861.80:1.1929905e+38}, {82557955.27:3.1013628e+38, 3004023.15:1.357968e+38, 54439379.28:1.9437927e+38}, {15256342.27:2.2020819e+38, 11934884.97:5.0453036e+37, 10587577.71:3.285984e+38}]	[{39094956.93:6.4688305033989194e+307, 55425630.25:4.0875347255490105e+307, 97579684.07:1.9461749915532969e+307}, {45760454.91:1.2615967120157324e+308, 87750775.16:3.742552076755832e+307, 85468435.58:7.9466048404010963e+307}, {25026327.05:4.0870821149814692e+307, 38091922.82:1.7203246936389238e+308, 7449296.72:3.8352612268680871e+307}]	[{9091735.23:"2023-10-26", 24894768.54:"2023-10-26", 35488182.29:"2023-10-26"}, {5922617.53:"2023-10-26", 97873265.35:"2023-10-26", 52499649.21:"2023-10-26"}, {82100319.07:"2023-10-26", 89593712.06:"2023-10-26", 16070877.80:"2023-10-26"}]	[{98846355.74:"2023-10-26 03:16:52.986", 91979360.36:"2023-10-26 03:16:52.986", 27415606.25:"2023-10-26 03:16:52.986"}, {25908470.13:"2023-10-26 03:16:52.986", 46857009.16:"2023-10-26 03:16:52.986", 31839705.83:"2023-10-26 03:16:52.986"}, {28227918.65:"2023-10-26 03:16:52.986", 85142863.76:"2023-10-26 03:16:52.986", 73036366.64:"2023-10-26 03:16:52.986"}]	[{33:0, 90:1, 86:1}, {52:0, -34:1, 91:1}, {1:1, -118:0, -107:1}]	[{22:"c", -3:"b", 3:"e"}, {18:"3", -76:"9", -72:"3"}, {-2:"a", -83:"9", -24:"6"}]	[{-121:"3c", -70:"b6", -8:"0c"}, {16:"61", 0:"4a", 83:"41"}, {100:"c0", -111:"39", 43:"c2"}]	[{80:"acce851f2953daab534a9538d2de993393ebc0770f000bd67fc119570dabc4725aae41c1357d62c53f6bf9346396de96cb07", -71:"49c27636dd21d2e3a7fbf24492068158ee0a69569ee0ec630e6cfaade4e2398c7dd4bb1b055da68d24d574850ee86bae4aba", -88:"66a26e6e7e638944124b7794806cbd7bd7a70ba7fb576520c6268229785ed307aca0d534d0e830d3a094248a26eddf19c595"}, {-30:"ee1cd1c51a6b2c4ec5ab374ef87487d628ea20878e5a453b9dd80fd50f15869ca2dfc876b2210ba8ba03da996733fe80903f", -103:"989ab1db7ba57bb261ee6845d98cf8f18c5afc1820895e646dea72ea5db5968ba9354f40d41ecfbb5381acd8d8621166ac56", -47:"55f890949b873c43b0f6e036c6018ee66d469a3879b89833fbd114a8c3e2b5c6bae03ccd4cd6b554b9cefa84d1ba9578d14d"}, {-75:"16310112498e149727bbaac04a81f11a4f81031e3eadcf25f25561d733f8961e9d22112abb26dd4b1832021d4cfa9719e341", -20:"07492559d23f0bd44650fd79f1c1f0028a6be48cf738bba1436c1bb0ff2b477ea5f85688930857ce6b187995746abfb22806", -96:"ec6af58b07f102db442ffbedba58b7f297819df60af4384d1518267547f0467936f22aa070dbd7df53ba07ead1707b71e43f"}]	[{-110:67712748.98, -124:46220854.18, 63:33495650.15}, {-22:86964636.84, 57:12062450.33, -20:94376190.17}, {59:2920430.00, -80:89992773.79, 71:99247729.39}]	[{-33:-92, 113:-98, 126:-101}, {45:-107, -52:72, 63:-95}, {60:-39, -49:-20, 99:27}]	[{56:6538, -29:-18696, 59:-20200}, {84:18388, -82:2027, -34:19058}, {-46:-19989, 57:-15524, -24:26067}]	[{-21:1774301461, 117:-1299931121, -47:-1481799633}, {12:-1217929424, 85:-1276418163, -120:1325283594}, {-81:784242435, -98:706166876, -116:-1668021208}]	[{77:-3927223138828377131, 125:3693748221639404854, -35:-738092288975151942}, {36:-2367727016212520427, -25:7022757294133569050, 93:1879001990359631862}, {-102:810963393936201432, -59:-8864541744320524200, 67:-7329710472402329495}]	[{36:3.9183765e+37, -98:2.9836885e+38, 66:2.4713974e+38}, {-47:3.003101e+38, 14:1.5627054e+38, -80:3.3529407e+38}, {-116:7.3803987e+37, -68:1.0566693e+38, -48:3.084806e+38}]	[{78:1.7582338828817097e+308, 94:1.6198169187760943e+308, -68:1.0543864125565194e+307}, {-33:1.1998484617110846e+308, -118:8.0668831697278285e+307, 26:1.4900130013342143e+308}, {88:5.7061682406856587e+307, 101:6.8297590256302121e+307, 27:1.4773307736201595e+308}]	[{0:"2023-10-26", 34:"2023-10-26", -39:"2023-10-26"}, {-77:"2023-10-26", 40:"2023-10-26", -73:"2023-10-26"}, {-61:"2023-10-26", 48:"2023-10-26", -62:"2023-10-26"}]	[{-69:"2023-10-26 03:16:52.986", 93:"2023-10-26 03:16:52.986", 86:"2023-10-26 03:16:52.986"}, {124:"2023-10-26 03:16:52.986", 68:"2023-10-26 03:16:52.986", 4:"2023-10-26 03:16:52.986"}, {41:"2023-10-26 03:16:52.986", -7:"2023-10-26 03:16:52.986", 27:"2023-10-26 03:16:52.986"}]	[{-938:1, -14702:1, -555:1}, {-27074:0, -20323:0, -8216:0}, {6330:0, 15183:1, -5356:0}]	[{25284:"8", -10403:"5", 22571:"1"}, {-23981:"6", 10013:"e", 7766:"2"}, {24090:"9", 28447:"7", 16839:"b"}]	[{-6289:"01", -25847:"6b", 11818:"ca"}, {4472:"1e", 23017:"60", 26161:"ae"}, {-31963:"e2", -23120:"5b", 18527:"1b"}]	[{-12041:"659216014248a4358f93c896488745462ea56cd388647a5443be2d165365b6d915716b3f5c797549ef82f8ca562732c5ebef", 23194:"2305da27c751389854e7d86355e551a3eb7d132998190e03f98c819ecf9e92850af598f67a66849702a199b0b484c7d0b03d", -26687:"98b74e190018919f4f9257771213ad3469bd98be85d7abd052e1d9d63fa139e552c45e3070b10f6d800055c8a03a7d8ff726"}, {-20141:"b62db0d5711df7122f0d1a1d1621d513251b6755e65e759cea518522bb4a02577efacdb551e1bebdb70751863e3bffc45291", -14870:"e25f5d25c7deb2049f1f1d0216bd9a4e954b283a281b81e925fd5fced4c0687c6da0743de47cab29d3d2e435c7181c88f353", 30101:"650c2270db140c8d587cddb4ec2a6cc79a1e97752ecc9a8f66200dc174c23786a26836ccde9ea36ad62a03f915f9e9c1dabd"}, {2968:"8363d882c8a17927f4c907051513482c7412b2ec5a3e790223b8cd5f3ddfa918b9ed16161bfbad610e0e83bd6782d130856f", -6767:"64e1eea2aadfe1d352ebfe0dc5e2197a2735e37d7bea233cd8393f8819d88f2551c9f9ac93f3069060b9395f3df0a472c824", -30431:"bfe256cd06ad22bf345c33c93b2ca380cd058b797a01f16fe9a02ea6bcddc3bf0b30e0596304d2ef19916a2a4cdb34242090"}]	[{14418:47422864.83, 6722:52974625.45, -2492:4806535.72}, {13077:44273088.54, -2322:39452352.79, 9895:32637704.63}, {-30558:33558011.78, -16359:17857571.67, -29056:6982144.13}]	[{-16249:9, 16210:127, 27430:38}, {-5989:74, 28740:-21, 3021:-57}, {-16481:111, 22671:-2, -20860:17}]	[{12924:13339, 26174:-29694, 3023:-2400}, {31544:18040, -5635:5211, -22796:28931}, {19493:13468, 9538:-27879, 8827:-31362}]	[{-7705:-1548292944, -12482:624994557, 28517:-893683680}, {23000:-171355841, 32711:161781665, 5587:-872491216}, {-6545:1659108101, 18798:-947095038, -14648:-480003018}]	[{2106:-2439183381828831690, -29320:-3010909727210773603, -25008:-4138315238991813428}, {25964:1791516789950945213, 13494:-1449049425031652460, -25419:6186364633373951140}, {-15358:-7684372786688112969, 27901:-7824869410062469510, -29848:2417774707291778273}]	[{16398:1.5755516e+38, 18019:1.6403663e+38, 6191:2.3612485e+38}, {9229:2.1707249e+38, 11790:2.7049803e+37, -9203:1.6202249e+38}, {5152:1.8797194e+38, -28591:3.2235965e+38, -27807:2.9189653e+38}]	[{2384:1.041271466145199e+308, 4266:9.6504740124719085e+307, -22620:1.5958506697896049e+308}, {18224:6.6619422341446274e+307, 6969:5.2477521420572547e+307, -4776:1.4787276240609435e+308}, {30689:8.134196893884338e+307, -1359:1.2637680122540544e+308, -30332:1.5290841066825333e+308}]	[{29804:"2023-10-26", 4281:"2023-10-26", 690:"2023-10-26"}, {15569:"2023-10-26", -2395:"2023-10-26", -11112:"2023-10-26"}, {-1157:"2023-10-26", 19102:"2023-10-26", 1650:"2023-10-26"}]	[{14090:"2023-10-26 03:16:52.986", 28394:"2023-10-26 03:16:52.986", -11712:"2023-10-26 03:16:52.986"}, {-10330:"2023-10-26 03:16:52.986", 14214:"2023-10-26 03:16:52.986", 16702:"2023-10-26 03:16:52.986"}, {6852:"2023-10-26 03:16:52.986", -5986:"2023-10-26 03:16:52.986", 22985:"2023-10-26 03:16:52.986"}]	[{1648625797:0, -1916745273:1, -1796110526:0}, {397864882:1, 133576600:0, -922934822:0}, {-1558716061:0, -1589821319:0, -1984466136:0}]	[{-587449708:"9", -771269706:"3", -220744203:"c"}, {237173603:"6", 1883587076:"1", 1784007616:"9"}, {245340026:"6", 492322125:"7", 1776389268:"e"}]	[{-1652453736:"d5", 525223664:"a9", -1188333267:"1e"}, {-1348144674:"10", 817514952:"7e", -1760384991:"37"}, {-1712431878:"80", 69010803:"d5", 1267886252:"e8"}]	[{-1314140134:"1a835b85e57c8fe3c8f8f2be69d1d000f453803fb589228af76ed00e36156a7cb3a4a4b308958af4554301d2d854defaa85d", -900785463:"fc012c3ce853e9087f518c9a00fbe4c6bde8d733a4d65203f18f03439b447925a38a7c1fb415fc34b107b2845c0271ef1359", 1584765254:"e960ffd4a974b2203fe2fa69c5e82abc2e7cd07a918c33f684edb129c7f1f79340d9bcf8664ad8defb2af1f46010e27a8536"}, {1265239574:"523cc1309337b7b1f4d6029b266c637ddb4e8f7104d354205d724bf637bf17595eab59e57a5d4c17e9c56320d5e93f70f69b", 1014613598:"b1b510e15a3767d920f41cb13fbe79e6995452e0cbfb3b6fb1968dec546e7725298a5ddf08ec77c7d3220aa83bcad12d25e1", -1084544152:"bd89ef54b7a54aabf17907373b22cb9a3979a728e48ffd36cac23efbead071ae503f00d5e51e7c3f70e88a2867108cf19fb3"}, {2023117026:"a3b0096d1d8e7e13d9328e8ed6568344f78b3b006156c52a173c9f5cdf6730eb6541216e3caf63da035fed1835e6021b0625", -922826946:"390f30bd3d79efc91f46c7a5e0bce1848710641afda0165d6d592d74c2b01eb204111ea132de6da752ce59948b3f071ce885", 1510390575:"ab340f0580254f8b4792a8e13d59ce0023847d801d756a0b364df65c137d7e1418c092e6ddc91d5baec30669c3d62c6948fb"}]	[{-782038093:5035393.61, 116206468:78331998.02, -1240231639:65542123.19}, {-860997521:85802752.63, 1983588242:46461344.26, -2113975148:20500503.29}, {1140349268:54585129.59, 388183282:83944724.73, 1651646629:7788231.06}]	[{2076663790:33, -569016177:36, 205269127:24}, {-1513070588:25, 1844166896:-60, -1797593663:30}, {-1891145685:56, 812635439:-47, 1141343750:52}]	[{-1397566712:29517, 1631787309:8908, -927803951:14907}, {625271908:20854, -1367784987:3555, 1945750295:-1538}, {-819242049:1949, -1367394061:15558, -1887083681:-17956}]	[{-1877888392:-1802321330, 1936364760:1463263317, 1714272858:1394651830}, {2121924866:-1262513593, -411888927:-1500275863, 1318199760:-1377795828}, {-1703236488:-1499741613, -1848925556:-39055776, -567806216:-1686298125}]	[{-1359982700:-8406507371565433806, 1805050963:2998036703112045478, 2106222880:-52741285402987305}, {-1162434002:4949182737260266951, 917836179:-8569073056407769060, -1426967323:6326256075088223795}, {1704873194:5998206389660526257, -336214204:-6509825477235419681, 1175690120:1292219765955055806}]	[{-235088567:2.9335702e+38, 1716132823:2.6714661e+37, -1193336447:1.5043472e+38}, {1019340904:2.3975581e+38, 1810096649:2.3723925e+38, -1907255970:2.0726907e+38}, {-707083028:1.08617e+38, -1236478977:2.9087249e+38, 972844419:3.2613417e+37}]	[{1415179393:1.6736592164557544e+308, -1836634395:1.6740150610752804e+308, 1046333122:8.6224476981514341e+307}, {61396437:1.1939040278004224e+308, -1080148932:3.623794336266382e+306, -2063709668:5.1439204138308237e+305}, {-401423982:1.6888146187004249e+308, 1698217221:1.7029949539113577e+308, 1253070314:1.0927911804886686e+308}]	[{858829916:"2023-10-26", 1247366810:"2023-10-26", -923423906:"2023-10-26"}, {1974603354:"2023-10-26", -709589026:"2023-10-26", -1724207175:"2023-10-26"}, {-1534836328:"2023-10-26", 1989849895:"2023-10-26", -2112866669:"2023-10-26"}]	[{114775671:"2023-10-26 03:16:52.986", -902521882:"2023-10-26 03:16:52.986", -17884440:"2023-10-26 03:16:52.986"}, {1128469562:"2023-10-26 03:16:52.986", -203825040:"2023-10-26 03:16:52.986", 2085796436:"2023-10-26 03:16:52.986"}, {-1989144471:"2023-10-26 03:16:52.986", -2019673991:"2023-10-26 03:16:52.986", 1092210742:"2023-10-26 03:16:52.986"}]	[{-7366105307915772026:1, 2599324890260512919:1, -8079262002787734728:1}, {1402880447171107056:0, -4532245612963177394:0, -1600052487520804574:1}, {2690562969261551410:1, -5835513996215584310:0, 6670010146206520193:0}]	[{4413453244095551452:"a", -1211818538289992070:"4", -5430397343810900483:"8"}, {2594403108707658672:"e", -8042723108869254233:"f", 7464455733411419448:"a"}, {-4343956155279332465:"c", 5852015903816164465:"5", 6452319468353669171:"9"}]	[{975791611952226652:"af", 608721917240828401:"be", -3628839568527813485:"51"}, {-8175267199516343951:"13", -8858809582261080536:"d5", 6633702528417962174:"42"}, {-2697209353496795211:"2a", 2048774271313901023:"13", 6505180960883661120:"cf"}]	[{-1346745244214907279:"d1c4cc52af6f36457ba67ced7650e784a98b30da68c1bf0e3c1c1d0f126364ad7511069610a538ac2d2283f841b8054801ce", -6314874609008667437:"fd940f00dbef91cfb7b78af8e8dffbc214e2954db6ebdebd8f514da361885df7abdc5cbf3698aa37ec46c69ffb14a2f9dc09", 3463843154118313620:"bb2c4939bc70fdb5134c9362d018c5a0224a26c65b8778cc8c61c0e9bc2a1c13cffd63b4372b3fa7d3868a6183a08313d8fe"}, {5288868401956481522:"2f2d7c39f7fb6cb084c7230195f220d1e2ff5d852af0d4f4ce62c6ed4b4865d7a888c97220a76558475aa9139b44fca76684", -6832896243406509788:"abf314f42e3f900ef20d6f4fdb97b85522012baa3d0bee244838872b09e34dbc5d67a615217663f95376cec176866dfc55c7", -4503954307342471764:"13b5f4692405b495116a896189c5bfcee8f9b59067ea3d413c01de66ce12b710bd406a311d16b5938414ac993185ad9a6178"}, {-1667578948785987366:"8d0e968a468a8ca454704ec6327a2f98df9007650eb187128aa47fc174816ef35d7c1e274f2dbd02350b8868f7b97d804208", -6727608163596017846:"eb2cfb53bde4642b2ceb3751f4a6c318c1be98131302d87f2366adf3deb46de007d33f29f799c2efb19b6628d987f64d03d3", -5583643715655600222:"0ad7524dc3317fcbb0a7ecbfe79bdd23986aac3dfa9f07b1f73dce23c129a74417e61379b92cc5c01897eef0b5647b192a6d"}]	[{159908411859734456:9136722.50, -1111551349683810310:37867116.36, -322328980980475177:23420646.81}, {1207184303147983890:58519011.18, 3449004235823923149:79248862.07, 6340455063061808783:3196097.05}, {-6094643256257790754:60087541.51, 5584086872468129009:17097793.63, -7148850621247198039:13939457.86}]	[{-8396208680374024065:52, -4729477683018986072:18, 2508437656819186409:6}, {7341980552633333003:-48, 1089073202521184641:-101, -2765550091107076085:31}, {-1517174732906476365:-109, -5180776839495860623:-108, -8548707263218640010:65}]	[{-5414574279395557642:30842, -7303476451795664957:-7667, 9204891015781125495:-24798}, {-7883554104486220914:18500, -1928436990280083319:-7624, -3593041641777088040:-4464}, {3648669436381121324:-13061, -3011901398593878520:9444, 4108783407109816471:5524}]	[{7196929626441064801:-101044284, -6985575674038337450:989100904, 5990726155536940173:766938129}, {-6810959414650930899:-1045104694, -5924416194564136020:-2127034062, 5902534287823570109:1799714157}, {-1726879291727452367:985596995, -5493624492888742149:-1008396645, 697260709149987221:212960358}]	[{5189859692953071815:-31913493257957933, -304305469384580292:-3134829775142285973, 729516411968549980:-2710187328973948624}, {2307866299777804834:5284506100663519715, -2485329502409325451:-8899641637886853684, -2457376949059429391:6116488331130445175}, {-8970057904137155598:-589746610635219992, -6870514015972289926:6028218678212929701, -5399766036281995100:6669865074577741746}]	[{8656794698150164371:1.2630847e+38, -2883993939281508319:6.7064862e+37, 8469241797230858595:2.3226955e+38}, {-6305580096134117582:7.9023e+37, 2394292456341801388:1.4050514e+38, -2559448774591566890:3.2110985e+38}, {4748990774415988933:2.8153907e+38, 5201809383293658248:1.2668793e+38, 6145402410742887824:2.2867342e+38}]	[{7941586766285252916:4.74689766514415e+307, -755009563146551618:1.5726645043868444e+308, -4551728424472363511:1.4556121608026074e+308}, {5190516698883241093:1.5505562446643624e+308, 5517850752777374917:1.5741733381897408e+308, -7946875212385751990:1.3159078744754623e+308}, {-5720858564776536983:1.3873483225290794e+308, 7397656217005837466:1.3729902854493187e+308, -5098501786445789178:1.0484470893634214e+308}]	[{-8105850773343240448:"2023-10-26", 665260583989298139:"2023-10-26", 9087851294756900337:"2023-10-26"}, {-7264010240187051851:"2023-10-26", 1499332189592232697:"2023-10-26", -3651267312532415178:"2023-10-26"}, {4013035271743652142:"2023-10-26", 8775996522750339414:"2023-10-26", -6934135974188719206:"2023-10-26"}]	[{-2161417055945133227:"2023-10-26 03:16:52.986", 755811567489341660:"2023-10-26 03:16:52.986", 4216426814030631499:"2023-10-26 03:16:52.986"}, {-5156268857120190977:"2023-10-26 03:16:52.986", 5043844330898652676:"2023-10-26 03:16:52.986", 5165219018259762219:"2023-10-26 03:16:52.986"}, {8310801222166953064:"2023-10-26 03:16:52.986", 6777390833640654906:"2023-10-26 03:16:52.986", -8540639044781263974:"2023-10-26 03:16:52.986"}]	[{1.7518221e+38:0, 2.9463551e+38:1, 3.3414461e+38:1}, {2.0305574e+38:0, 2.0838355e+38:0, 2.1534986e+38:0}, {3.3044071e+37:0, 2.0463959e+38:1, 2.7637387e+38:1}]	[{3.2852821e+37:"f", 6.4530443e+37:"f", 1.9574195e+37:"7"}, {7.3086034e+36:"8", 3.3466361e+38:"b", 1.4495752e+38:"a"}, {7.1481854e+36:"7", 6.1522562e+37:"c", 9.3936986e+37:"8"}]	[{1.3678543e+38:"32", 2.5697841e+38:"db", 8.9975284e+37:"68"}, {2.0849159e+38:"be", 8.2203318e+37:"c7", 3.2548104e+38:"fd"}, {2.8628242e+38:"80", 1.5211348e+38:"cf", 3.3900605e+37:"93"}]	[{2.8009811e+38:"636623589bb9becfe8f46bb8d665e57b4516969c28df257be32705738c1c559d628a1d601e36ef7eb898645e04b193807cf9", 2.6066722e+38:"ff1e1ab4320cc8dc7c85efa07a57cde62bf4944f7f48276aaa98b0292cb3c0a5092339b1a519415c77503d561e0f6e6301a9", 6.3048428e+37:"8a9ecde4c299e66d41cdff2a938fa1177f98d0b00b5474a588a75683a8dc9fdba4a39592a7531a630b26c7eaa4201915ef0b"}, {2.9687936e+37:"945d78cbde1ef8911ed181a4414af1e83135e1ed6c04c0da2e9120bc15f9f1a259387eae95a807e9529003402b8b6c2e978b", 1.9850851e+38:"ea146e2d146fdf32c57f1339eb0d58765cd5511b3b15c65806a2289f78e4130c76aefb0a0ed138a760643469e06bcc792678", 1.9457554e+38:"9eaf0d8be26b2c0c7b0afc0277110e299bfcb647e75710d6cb3beb9d6c0e07375ba1f51765c945893d093f7ae7032d18c514"}, {1.2534964e+38:"2be8cf07182de86f61868fcd7cb2e4b12125537d46d791c8773822acb3c0d90727dccf914274430e166c6358f25705987427", 1.8730493e+38:"cfd9f13edd58a1fa9f79e5186aadcf4c341fdf53a7d24a2b9b321cd3f3eb9400473eb052ef6b0d41e25275ca879ae7acdfd2", 2.439195e+38:"e993b22dea6729fb7da9e62b0927d2402d989304975251a9f28b78ab9a20bf0310ff1d89e07ed2fa5fc65a7aa3145d10c4f8"}]	[{1.7993635e+38:82046329.98, 9.0320531e+37:69050750.48, 1.1245292e+38:95527951.33}, {2.9125627e+38:20817805.64, 1.1691811e+38:13227959.41, 9.2198774e+37:34297834.54}, {1.3829097e+38:15644319.48, 6.9981273e+36:69384957.41, 6.4033489e+37:99120783.59}]	[{5.5470864e+37:77, 2.1589432e+38:62, 1.0122624e+38:78}, {1.3793198e+38:87, 1.3348865e+38:-21, 3.8251851e+37:79}, {2.9102098e+38:1, 1.205493e+37:-79, 1.4579308e+38:-47}]	[{5.1127031e+37:-26850, 1.6311614e+38:-1118, 1.2600384e+38:24864}, {5.0833418e+37:-31518, 9.2248567e+37:10582, 1.1032294e+38:21269}, {2.6317002e+38:-31727, 2.2466257e+38:32023, 2.3830073e+38:-23815}]	[{2.4685356e+38:-1162263708, 1.4748677e+38:1200207862, 2.9596816e+38:579593546}, {6.251459e+37:-1444887010, 2.8935648e+38:26851854, 3.0352598e+38:-116565992}, {1.0357336e+38:250884268, 3.0524911e+38:-1929859043, 3.0465775e+38:1962433460}]	[{3.214866e+38:5737796259439971568, 2.6620981e+38:-1620834117999910231, 2.4601812e+38:-7700913896791875817}, {1.3225473e+38:3170094435332391129, 2.2596444e+38:-3520748004397558592, 3.2590745e+38:6436283103070667842}, {1.4001315e+38:1223700250225204298, 3.2190439e+38:5617609752086895667, 4.5102866e+37:-2594255970026495278}]	[{4.0467279e+37:2.2421306e+37, 1.4734602e+38:1.8301668e+38, 2.783976e+37:8.1052641e+37}, {4.0733369e+37:5.2819421e+37, 1.4999969e+37:3.3161411e+38, 1.8830723e+38:1.5928712e+38}, {8.3220948e+37:2.9007054e+38, 2.0762411e+38:9.5059912e+37, 2.1421498e+38:2.9553098e+37}]	[{2.3146355e+38:1.7083590718843164e+308, 1.9005457e+38:2.9569638035197616e+307, 1.1105389e+38:5.8719636167502007e+307}, {1.4665541e+38:1.1060346654452101e+308, 8.98433e+37:4.4751544864085193e+307, 2.8684933e+38:1.4079002398012361e+308}, {1.8532074e+38:4.6785843034490426e+307, 2.6331476e+38:1.3460887152316149e+308, 2.4709528e+38:1.4334768316844933e+308}]	[{2.274238e+38:"2023-10-26", 7.8158113e+37:"2023-10-26", 1.0720517e+38:"2023-10-26"}, {1.134687e+38:"2023-10-26", 1.7763916e+38:"2023-10-26", 3.0358084e+38:"2023-10-26"}, {2.4241613e+38:"2023-10-26", 2.4720347e+38:"2023-10-26", 1.1518704e+38:"2023-10-26"}]	[{8.7803038e+37:"2023-10-26 03:16:52.987", 1.5468421e+38:"2023-10-26 03:16:52.987", 1.040932e+38:"2023-10-26 03:16:52.987"}, {2.5066716e+38:"2023-10-26 03:16:52.987", 2.9127078e+38:"2023-10-26 03:16:52.987", 1.1711671e+38:"2023-10-26 03:16:52.987"}, {1.2017717e+38:"2023-10-26 03:16:52.987", 2.3821676e+38:"2023-10-26 03:16:52.987", 2.2182281e+38:"2023-10-26 03:16:52.987"}]	[{1.6791023976855424e+308:0, 2.5299067273199039e+306:1, 1.4057482298674958e+307:1}, {1.7081851733159151e+308:0, 1.5239357146144377e+308:1, 1.7699262046915312e+308:0}, {1.5268939031167621e+308:0, 9.0655072243850677e+307:0, 8.5874615477001466e+307:0}]	[{2.088956493227517e+307:"5", 1.0674323380560357e+308:"b", 9.00369769102035e+307:"f"}, {1.4761628616481553e+308:"b", 1.1088452742179012e+308:"7", 3.0104395312877223e+307:"0"}, {1.4249806162156349e+308:"f", 1.7122823675967652e+308:"b", 4.84828850430845e+307:"f"}]	[{4.7936080861709133e+307:"b4", 1.440192573791489e+308:"d5", 1.1749931059026829e+308:"97"}, {5.53723292220953e+307:"9c", 1.7766361830419384e+308:"66", 4.5610522301884063e+307:"33"}, {1.2570054571915617e+308:"a5", 8.6369148060174211e+307:"52", 1.2766632325188215e+308:"93"}]	[{2.20542192647443e+307:"813dd727fd83957973a650ea44b26ae59fe9ad85e304b9466327764a21ae73c18d7b2fbc5ff496c7b8cc4be991caf9935cb1", 1.7880726683240837e+308:"fa2f47e030e3ecca28df60ba30a73fd2b3435ed3fbf9f5fdbc0968464491eee96d6b855a0fd5307777c851fcc8f68fba0301", 1.3595608280129958e+308:"f4522f434683e8a7738d6d872bec8975442c4d75459bed17313d245b09a80567729fb94928a498ef15258c06246a8bb85820"}, {8.9690172595858888e+307:"84e0981ed42a41c1644eb70cc57d7805cc019e978c3e1f919cb0121b8064f147f74bff110e167f1d7d7cff6be1eb4b28933b", 1.6746958659903706e+307:"6114e39f2a717b073dad37b87a04e430090a507c52f3480e196ecd8c8beec2f4c3bbc442d90b52e7252fb7dc6d1204602eda", 1.3976460095327371e+308:"7a8c8c89d892854b0104e5121e6816fb2ed1f1cc83926f0a5163747bd200f98d045a6d17bd335b53bf92c6743c024819328c"}, {1.2832573531975194e+307:"c0d032d7eb52b26fe1dc81fcab684603c013453e38dcbd23a7150e9f5d119c9bc24110b415bb5c7505d0de4c030ac36b360d", 9.9775727419234367e+307:"2b6c594cbfd044937904bd94d751116e2c140af1e50a98a82cf8f4451cb4bcd7a372a8f7427b2689ae31d3c92bf937dd0738", 1.6162183669341029e+308:"7d16d873a5d0eed913596172da0bdb949cb92abd666178e03cdc7352049e1cfa3f4921004c3a8709f0b37a47e20c84595f19"}]	[{1.1210112586833211e+308:99613311.48, 5.0337701212881811e+307:66538034.68, 1.1880462167228515e+308:96619066.79}, {7.6914193399411777e+307:41995003.32, 4.2576493833990175e+307:35638144.39, 1.2946285683633852e+308:91149882.21}, {3.10465393939426e+307:31356699.90, 8.437698380594389e+307:17755106.80, 5.22140997575902e+307:56961478.40}]	[{3.1664436609054078e+307:-14, 6.6498023884519536e+307:48, 1.3569396968987324e+308:-11}, {1.6343096883653655e+308:-31, 3.347239268016444e+307:121, 3.6037659832301909e+307:106}, {3.6940442103318012e+307:-35, 1.386813639272129e+308:118, 1.2801794351856471e+307:-45}]	[{1.4555669676436808e+308:8312, 1.6914250334690539e+308:-7631, 1.6195643629756671e+306:31618}, {1.4361596693307541e+308:11360, 1.0127597308237579e+308:3984, 1.6890292133620608e+308:794}, {1.462727301621658e+308:13706, 2.3977706090250962e+307:-29955, 9.6156624455307e+307:2020}]	[{8.3758273625487968e+307:1592290309, 9.8553790300156457e+306:1783905994, 1.0174492289376822e+308:1555825176}, {3.9753775350227687e+307:268528214, 3.2907011617244692e+307:1805986498, 8.7802500926021239e+307:-1395751015}, {8.49508852940846e+307:1955051099, 1.7500226830733431e+308:892553294, 7.0539533755719646e+307:258000687}]	[{8.9391628824500816e+307:-4238501403164926611, 7.4814696584589869e+306:5520190430717696350, 2.1905204880439089e+307:-8680240056253701121}, {1.7758679661525579e+308:5775440171943789240, 5.8574032509629684e+307:-3904457361041348258, 1.4842637913547878e+308:2354683699006584272}, {1.5833944041219624e+308:-1508753137464543061, 9.5431442547440273e+307:-8642396304262043373, 9.4695068251045637e+307:-4241183732039493845}]	[{1.3471920098351424e+308:1.5401668e+38, 7.1620448706916651e+307:1.9967957e+38, 3.1584436147620188e+307:2.901348e+38}, {2.9788253567304409e+307:1.5356359e+37, 5.4442372158516444e+307:3.2200493e+38, 1.7054871912325912e+308:2.0526821e+38}, {7.9056343122684356e+307:2.4205804e+38, 1.4254388539804705e+308:2.3927e+38, 8.5466020755408349e+307:4.6546036e+37}]	[{1.7270285909493177e+308:7.7908410093353058e+307, 6.715207215166598e+307:7.5636834019824456e+307, 1.2587289640929043e+308:7.9148809156520181e+307}, {1.7035192386264395e+308:1.5470219640066283e+308, 1.2942207285255918e+308:7.8123133848184191e+307, 1.2364692716057604e+308:1.476367298743881e+308}, {1.7273342538276163e+308:1.3372076292046648e+308, 4.8489217377481329e+307:1.9796330111934133e+307, 6.4221398801883356e+306:1.42376541531052e+308}]	[{1.3152089352277706e+308:"2023-10-26", 7.0469634736428227e+307:"2023-10-26", 7.00844337893011e+307:"2023-10-26"}, {7.9434039795711751e+307:"2023-10-26", 7.5721133224886512e+307:"2023-10-26", 1.5575847721158946e+308:"2023-10-26"}, {1.3999887293411846e+308:"2023-10-26", 1.0823556247834667e+308:"2023-10-26", 8.4926168440603082e+307:"2023-10-26"}]	[{7.9643788980223435e+307:"2023-10-26 03:16:52.987", 1.2664623508771269e+308:"2023-10-26 03:16:52.987", 3.2928577460803168e+307:"2023-10-26 03:16:52.987"}, {9.8286522971053418e+307:"2023-10-26 03:16:52.987", 9.1973817611438941e+307:"2023-10-26 03:16:52.987", 1.2391271890721865e+308:"2023-10-26 03:16:52.987"}, {1.3823857186844162e+308:"2023-10-26 03:16:52.987", 1.6882013803571224e+308:"2023-10-26 03:16:52.987", 1.2426454915645717e+307:"2023-10-26 03:16:52.987"}]	[{"2023-10-26":1}, {"2023-10-26":1}, {"2023-10-26":1}]	[{"2023-10-26":"7"}, {"2023-10-26":"c"}, {"2023-10-26":"1"}]	[{"2023-10-26":"30"}, {"2023-10-26":"5d"}, {"2023-10-26":"40"}]	[{"2023-10-26":"1290d69923cf6bb99c9ccd03f49dc8a6d9d14551bc5473f30511e5365f38e594d4368677ec34a365ccdba3dfdb4c53d05efb"}, {"2023-10-26":"d6ccf534c43e08520e09c5aea0b417af556f371c94cdfcf5099c7829cfd2e8dfb87eaf8db61a9c0f70a645bd2a2b2159d2f3"}, {"2023-10-26":"d59ab72761bc84d6f013c0539cc821ec7328d87cf369f4af56640e469534fd0afa7af4f32b8822d2d19814c4cd813c80e3cc"}]	[{"2023-10-26":45997691.70}, {"2023-10-26":21424348.87}, {"2023-10-26":91359356.44}]	[{"2023-10-26":110}, {"2023-10-26":-75}, {"2023-10-26":-99}]	[{"2023-10-26":-25694}, {"2023-10-26":6198}, {"2023-10-26":23718}]	[{"2023-10-26":-776923703}, {"2023-10-26":1451914003}, {"2023-10-26":323923385}]	[{"2023-10-26":-6626506271166747754}, {"2023-10-26":1101161460355326549}, {"2023-10-26":5325684255116417875}]	[{"2023-10-26":2.5789021e+38}, {"2023-10-26":2.8164312e+38}, {"2023-10-26":1.1726865e+38}]	[{"2023-10-26":1.6734069950357059e+308}, {"2023-10-26":1.1768835185850233e+307}, {"2023-10-26":4.9638178175643781e+307}]	[{"2023-10-26":"2023-10-26"}, {"2023-10-26":"2023-10-26"}, {"2023-10-26":"2023-10-26"}]	[{"2023-10-26":"2023-10-26 03:16:52.987"}, {"2023-10-26":"2023-10-26 03:16:52.987"}, {"2023-10-26":"2023-10-26 03:16:52.987"}]	[{"2023-10-26 03:16:52.987":0}, {"2023-10-26 03:16:52.987":0}, {"2023-10-26 03:16:52.987":0}]	[{"2023-10-26 03:16:52.987":"3"}, {"2023-10-26 03:16:52.987":"f"}, {"2023-10-26 03:16:52.987":"c"}]	[{"2023-10-26 03:16:52.987":"45"}, {"2023-10-26 03:16:52.987":"e4"}, {"2023-10-26 03:16:52.987":"97"}]	[{"2023-10-26 03:16:52.987":"050aeddc100e3ba0212999f4c2062dcc0115bee7beb5e478b2515c4875e5f43edc3a5ed0a1fae5d515caadbbdad82bfd1623"}, {"2023-10-26 03:16:52.987":"21d4932074cdff9f0a2e1f458e0eee7b3686efcd21520b730abfd94ea23c43710be78c8a4bb54f3c71ca2abf53e9e5e0aaef"}, {"2023-10-26 03:16:52.987":"0e9f9bd46b3c446a022cc593662afff927c7f32487c298f2278cd1a932f00371e8049abdc2adab5fdfab530da51f9530c85b"}]	[{"2023-10-26 03:16:52.987":61673823.67}, {"2023-10-26 03:16:52.987":54860070.29}, {"2023-10-26 03:16:52.987":50203646.01}]	[{"2023-10-26 03:16:52.987":66}, {"2023-10-26 03:16:52.987":71}, {"2023-10-26 03:16:52.987":-75}]	[{"2023-10-26 03:16:52.987":26076}, {"2023-10-26 03:16:52.987":-8292}, {"2023-10-26 03:16:52.987":30309}]	[{"2023-10-26 03:16:52.987":-1342327171}, {"2023-10-26 03:16:52.987":1797049642}, {"2023-10-26 03:16:52.987":-2134381346}]	[{"2023-10-26 03:16:52.987":4621394428141713482}, {"2023-10-26 03:16:52.987":-2376792531548299094}, {"2023-10-26 03:16:52.987":-5285583889220779304}]	[{"2023-10-26 03:16:52.987":2.0516586e+38}, {"2023-10-26 03:16:52.987":7.3252346e+37}, {"2023-10-26 03:16:52.987":4.59661e+37}]	[{"2023-10-26 03:16:52.987":4.8756373571788081e+307}, {"2023-10-26 03:16:52.987":9.3750657455984325e+307}, {"2023-10-26 03:16:52.987":1.1561060812088368e+308}]	[{"2023-10-26 03:16:52.987":"2023-10-26"}, {"2023-10-26 03:16:52.987":"2023-10-26"}, {"2023-10-26 03:16:52.987":"2023-10-26"}]	[{"2023-10-26 03:16:52.987":"2023-10-26 03:16:52.987"}, {"2023-10-26 03:16:52.987":"2023-10-26 03:16:52.987"}, {"2023-10-26 03:16:52.987":"2023-10-26 03:16:52.987"}]
 
+-- !c101 --
+1	2	3	4	5	6	7	8	9.1	10.1	11.10	2020-02-02	13str	14varchar	a	true	aaaa	2023-08-13T09:32:38.530
+10	20	30	40	50	60	70	80	90.1	100.1	110.10	2020-03-02	130str	140varchar	b	false	bbbb	2023-08-14T08:32:52.821
+
diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
index ecf3263f82506a..87ea14ad2fdb02 100644
--- a/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
+++ b/regression-test/suites/external_table_p0/paimon/test_paimon_catalog.groovy
@@ -166,6 +166,8 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_
 
             def c100= """select * from array_nested order by c1;"""
 
+            def c101="""select * from all_table where c1 is not null or c2 is not null order by c1"""
+
             String hdfs_port = context.config.otherConfigs.get("hdfs_port")
             String catalog_name = "paimon1"
             String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
@@ -284,6 +286,7 @@ suite("test_paimon_catalog", "p0,external,doris,external_docker,external_docker_
             qt_c98 c98
             qt_c99 c99
             qt_c100 c100
+            qt_c101 c101
 
             // test view from jion paimon
             sql """ switch internal """

From 12286f0a63d37426ae941efda443a5af7ba576d0 Mon Sep 17 00:00:00 2001
From: Qi Chen <kaka11.chen@gmail.com>
Date: Wed, 3 Jan 2024 12:57:20 +0800
Subject: [PATCH 284/299] [docs][hive-transactional-tables] Add hive
 transactional tables documents. (#29369)

---
 docs/en/docs/lakehouse/multi-catalog/hive.md    | 14 ++++++++++++++
 docs/zh-CN/docs/lakehouse/multi-catalog/hive.md | 12 ++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/docs/en/docs/lakehouse/multi-catalog/hive.md b/docs/en/docs/lakehouse/multi-catalog/hive.md
index 01c78d3599f8f4..90db7fe3543171 100644
--- a/docs/en/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/en/docs/lakehouse/multi-catalog/hive.md
@@ -555,3 +555,17 @@ CREATE CATALOG hive PROPERTIES (
     'yarn.resourcemanager.principal' = 'your-rm-principal'
 );
 ```
+## Hive Transactional Tables
+
+Hive transactional tables are tables in Hive that support ACID (Atomicity, Consistency, Isolation, Durability) semantics. For more details, you can refer to: [Hive Transactions](https://cwiki.apache.org/confluence/display/Hive/Hive+Transactions).
+
+### Supported Operations for Hive Transactional Tables:
+|Transactional Table Type|Supported Operations in Hive|Hive Table Properties|Supported Hive Versions|
+|---|---|---|---|
+|Full-ACID Transactional Table |Supports insert, update, delete operations|'transactional'='true', 'transactional_properties'='insert_only'|3.x, 2.x (requires major compaction in Hive before loading)|
+|Insert-Only Transactional Table|Supports only Insert operations|'transactional'='true'|3.x, 2.x|
+
+### Current Limitations:
+Currently, it does not support scenarios involving Original Files.
+When a table is transformed into a transactional table, subsequent newly written data files will use the schema of the Hive transactional table. However, existing data files will not be converted to the schema of the transactional table. These existing files are referred to as Original Files.
+
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
index b4efb291f89ed6..52e40fc1d0b729 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/hive.md
@@ -534,3 +534,15 @@ CREATE CATALOG hive_krb_ha PROPERTIES (
     'dfs.client.failover.proxy.provider.your-nameservice'='org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider'
 );
 ```
+## Hive Transactional 表
+Hive transactional 表是 Hive 中支持 ACID 语义的表。详情可见：https://cwiki.apache.org/confluence/display/Hive/Hive+Transactions
+
+### Hive Transactional 表支持情况：
+|表类型|在 Hive 中支持的操作|Hive 表属性|支持的Hive 版本|
+|---|---|---|---|
+|Full-ACID Transactional Table |支持 Insert, Update, Delete 操作|'transactional'='true', 'transactional_properties'='insert_only'|3.x，2.x，其中 2.x 需要在 Hive 中执行完 major compaction 才可以加载|
+|Insert-Only Transactional Table|只支持 Insert 操作|'transactional'='true'|3.x，2.x|
+
+### 当前限制：
+目前不支持 Original Files 的场景。
+当一个表转换成 Transactional 表之后，后续新写的数据文件会使用 Hive Transactional 表的 schema，但是已经存在的数据文件是不会转化成 Transactional 表的 schema，这样的文件称为 Original Files。

From 193b7518aba951dd4e8604bb18b782309ed845d0 Mon Sep 17 00:00:00 2001
From: starocean999 <40539150+starocean999@users.noreply.github.com>
Date: Wed, 3 Jan 2024 12:59:32 +0800
Subject: [PATCH 285/299] [enhancement](nereids)throw readable exception when
 meet missing column in agg's output (#29243)

---
 .../rules/analysis/CheckAfterRewrite.java     | 20 +++++++++++--------
 .../nereids_p0/aggregate/aggregate.groovy     |  8 ++++++++
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
index c43d794035a87a..56d5273e62300d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/CheckAfterRewrite.java
@@ -106,14 +106,18 @@ private void checkAllSlotReferenceFromChildren(Plan plan) {
                 .collect(Collectors.toSet());
         notFromChildren = removeValidSlotsNotFromChildren(notFromChildren, childrenOutput);
         if (!notFromChildren.isEmpty()) {
-            throw new AnalysisException(String.format("Input slot(s) not in child's output: %s in plan: %s,"
-                            + " child output is: %s\n" + "plan tree:\n" + plan.treeString(),
-                    StringUtils.join(notFromChildren.stream()
-                            .map(ExpressionTrait::toString)
-                            .collect(Collectors.toSet()), ", "), plan,
-                    plan.children().stream()
-                            .flatMap(child -> child.getOutput().stream())
-                            .collect(Collectors.toSet())));
+            if (plan.child(0) instanceof LogicalAggregate) {
+                throw new AnalysisException(String.format("%s not in agg's output", notFromChildren
+                        .stream().map(slot -> slot.getName()).collect(Collectors.joining(", "))));
+            } else {
+                throw new AnalysisException(String.format(
+                        "Input slot(s) not in child's output: %s in plan: %s,"
+                                + " child output is: %s\n" + "plan tree:\n" + plan.treeString(),
+                        StringUtils.join(notFromChildren.stream().map(ExpressionTrait::toString)
+                                .collect(Collectors.toSet()), ", "),
+                        plan, plan.children().stream().flatMap(child -> child.getOutput().stream())
+                                .collect(Collectors.toSet())));
+            }
         }
     }
 
diff --git a/regression-test/suites/nereids_p0/aggregate/aggregate.groovy b/regression-test/suites/nereids_p0/aggregate/aggregate.groovy
index f0ba091e045911..6f63b146cb5837 100644
--- a/regression-test/suites/nereids_p0/aggregate/aggregate.groovy
+++ b/regression-test/suites/nereids_p0/aggregate/aggregate.groovy
@@ -328,4 +328,12 @@ suite("aggregate") {
     assertTrue(plan.contains("max(c_bigint"))
     assertTrue(plan.contains("min(c_bigint"))
     assertTrue(plan.contains("any_value(c_bigint"))
+
+    test {
+        sql """
+              SELECT k1, k2 FROM tempbaseall
+              GROUP BY k1;
+            """
+        exception "java.sql.SQLException: errCode = 2, detailMessage = k2 not in agg's output"
+    }
 }

From 28ff349381cec4a2ca4df930bc088e0e75a8babc Mon Sep 17 00:00:00 2001
From: Rohit Satardekar <rohitrs1983@yahoo.com>
Date: Wed, 3 Jan 2024 10:29:59 +0530
Subject: [PATCH 286/299] =?UTF-8?q?[doc](fix)invalid=20character=20?=
 =?UTF-8?q?=E3=80=82=20in=20en=20docs=20(#29355)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Rohit Satardekar <rohitrs1983@gmail.com>
---
 docs/en/community/developer-guide/be-clion-dev.md      |  4 ++--
 docs/en/community/developer-guide/be-vscode-dev.md     |  2 +-
 docs/en/community/developer-guide/benchmark-tool.md    |  8 ++++----
 .../admin-manual/cluster-management/load-balancing.md  |  4 ++--
 .../http-actions/be/tablet-distribution.md             |  2 +-
 .../admin-manual/http-actions/fe/debug-point-action.md |  2 +-
 .../http-actions/fe/query-profile-action.md            |  2 +-
 .../admin-manual/http-actions/fe/statistic-action.md   |  2 +-
 docs/en/docs/admin-manual/privilege-ldap/ldap.md       |  2 +-
 docs/en/docs/advanced/cold-hot-separation.md           |  2 +-
 .../import/import-way/binlog-load-manual.md            |  2 +-
 .../import/import-way/spark-load-manual.md             |  2 +-
 docs/en/docs/ecosystem/flink-doris-connector.md        |  2 +-
 docs/en/docs/ecosystem/kyuubi.md                       |  2 +-
 .../udaf/remote-user-defined-aggregation-function.md   |  6 +++---
 .../docs/ecosystem/udf/remote-user-defined-function.md |  8 ++++----
 docs/en/docs/get-starting/quick-start.md               | 10 +++++-----
 docs/en/docs/install/source-install/compilation-mac.md |  2 +-
 docs/en/docs/lakehouse/file.md                         |  2 +-
 .../join-optimization/bucket-shuffle-join.md           |  4 ++--
 .../query-acceleration/pipeline-execution-engine.md    |  2 +-
 docs/en/docs/releasenotes/release-1.2.2.md             |  2 +-
 .../sql-functions/bitmap-functions/bitmap-count.md     |  2 +-
 .../Data-Definition-Statements/Create/CREATE-TABLE.md  |  4 ++--
 24 files changed, 40 insertions(+), 40 deletions(-)

diff --git a/docs/en/community/developer-guide/be-clion-dev.md b/docs/en/community/developer-guide/be-clion-dev.md
index d7de660e25d78c..6634f0d314d5b9 100644
--- a/docs/en/community/developer-guide/be-clion-dev.md
+++ b/docs/en/community/developer-guide/be-clion-dev.md
@@ -25,7 +25,7 @@ under the License.
 
 ## Downloading and Compiling Code on Remote Server
 
-1. Download Doris source code on the remote server, such as in the directory `/mnt/datadisk0/chenqi/doris`。
+1. Download Doris source code on the remote server, such as in the directory `/mnt/datadisk0/chenqi/doris`.
 
 ```
 git clone https://github.com/apache/doris.git
@@ -34,7 +34,7 @@ git clone https://github.com/apache/doris.git
 2. Modify the `env.sh` file located in the root directory of the Doris code on the remote server.
 Add the configuration for `DORIS_HOME` at the beginning, for example, `DORIS_HOME=/mnt/datadisk0/chenqi/doris.`
 
-3. Execute commands to compile the code. The detailed compilation process [compilation-with-ldb-toolchain](https://doris.apache.org/zh-CN/docs/dev/install/source-install/compilation-with-ldb-toolchain)。
+3. Execute commands to compile the code. The detailed compilation process [compilation-with-ldb-toolchain](https://doris.apache.org/zh-CN/docs/dev/install/source-install/compilation-with-ldb-toolchain).
 
 ```
 cd /mnt/datadisk0/chenqi/doris
diff --git a/docs/en/community/developer-guide/be-vscode-dev.md b/docs/en/community/developer-guide/be-vscode-dev.md
index e7964e2a3f7bb3..7cedc2a0257e5b 100644
--- a/docs/en/community/developer-guide/be-vscode-dev.md
+++ b/docs/en/community/developer-guide/be-vscode-dev.md
@@ -309,7 +309,7 @@ lldb's attach mode is faster than gdb，and the usage is similar to gdb. we shou
 }
 ```
 
-It should be noted that this method requires the system `glibc` version to be `2.18+`. you can refer [Get VSCode CodeLLDB plugin work on CentOS 7](https://gist.github.com/JaySon-Huang/63dcc6c011feb5bd6deb1ef0cf1a9b96) to make plugin work。
+It should be noted that this method requires the system `glibc` version to be `2.18+`. you can refer [Get VSCode CodeLLDB plugin work on CentOS 7](https://gist.github.com/JaySon-Huang/63dcc6c011feb5bd6deb1ef0cf1a9b96) to make plugin work.
 
 ## Debugging core dump files
 
diff --git a/docs/en/community/developer-guide/benchmark-tool.md b/docs/en/community/developer-guide/benchmark-tool.md
index a6ed29e12a516e..eb8277384a29c8 100644
--- a/docs/en/community/developer-guide/benchmark-tool.md
+++ b/docs/en/community/developer-guide/benchmark-tool.md
@@ -33,7 +33,7 @@ It can be used to test the performance of some parts of the BE storage layer (fo
 
 ## Compilation
 
-1. To ensure that the environment has been able to successfully compile the Doris ontology, you can refer to [Installation and deployment](/docs/install/source-install/compilation)。
+1. To ensure that the environment has been able to successfully compile the Doris ontology, you can refer to [Installation and deployment](/docs/install/source-install/compilation).
 
 2. Execute`run-be-ut.sh`
 
@@ -53,9 +53,9 @@ The data set is generated according to the following rules.
 >int: Random in [1,1000000]. 
 
 The data character set of string type is uppercase and lowercase English letters, and the length varies according to the type. 
-> char: Length random in [1,8]。
-> varchar: Length random in [1,128]。 
-> string: Length random in [1,100000]。
+> char: Length random in [1,8].
+> varchar: Length random in [1,128]. 
+> string: Length random in [1,100000].
 
 `rows_number` indicates the number of rows of data, the default value is `10000`. 
 
diff --git a/docs/en/docs/admin-manual/cluster-management/load-balancing.md b/docs/en/docs/admin-manual/cluster-management/load-balancing.md
index b655d57e0da3da..1933fa4f1a9ed9 100644
--- a/docs/en/docs/admin-manual/cluster-management/load-balancing.md
+++ b/docs/en/docs/admin-manual/cluster-management/load-balancing.md
@@ -294,7 +294,7 @@ Query OK, 94 rows affected (0.079 sec)
  
 Verify the monitoring results: The indicators of the ProxySQL monitoring module are stored in the log table of the monitor library.
 The following is the monitoring of whether the connection is normal (monitoring of connect indicators):
-Note: There may be many connect_errors, this is because there is an error when the monitoring information is not configured. After the configuration, if the result of connect_error is NULL, it means normal。
+Note: There may be many connect_errors, this is because there is an error when the monitoring information is not configured. After the configuration, if the result of connect_error is NULL, it means normal.
 MySQL [(none)]> select * from mysql_server_connect_log;
 +---------------+------+------------------+-------------------------+---------------+
 | hostname      | port | time_start_us    | connect_success_time_us | connect_error |
@@ -330,7 +330,7 @@ MySQL [(none)]> select * from mysql_server_read_only_log;
 Empty set (0.000 sec)
 
 All 3 nodes are in the group with hostgroup_id=10.
-Now, load the modification of the mysql_replication_hostgroups table just now to RUNTIME to take effect。
+Now, load the modification of the mysql_replication_hostgroups table just now to RUNTIME to take effect.
 MySQL [(none)]> load mysql servers to runtime;
 Query OK, 0 rows affected (0.003 sec)
  
diff --git a/docs/en/docs/admin-manual/http-actions/be/tablet-distribution.md b/docs/en/docs/admin-manual/http-actions/be/tablet-distribution.md
index 90cbe5424446ee..d2f3c4ef89663b 100644
--- a/docs/en/docs/admin-manual/http-actions/be/tablet-distribution.md
+++ b/docs/en/docs/admin-manual/http-actions/be/tablet-distribution.md
@@ -40,7 +40,7 @@ Get the distribution of tablets under each partition between different disks on
     only supports `partition`
 
 * `partition_id`
-    ID of the specified partition，Optional with default all partition。
+    ID of the specified partition，Optional with default all partition.
 
 ## Request body
 
diff --git a/docs/en/docs/admin-manual/http-actions/fe/debug-point-action.md b/docs/en/docs/admin-manual/http-actions/fe/debug-point-action.md
index 84ad9bf324a3f1..4bb59226b5cdc4 100644
--- a/docs/en/docs/admin-manual/http-actions/fe/debug-point-action.md
+++ b/docs/en/docs/admin-manual/http-actions/fe/debug-point-action.md
@@ -239,7 +239,7 @@ None
 ### Examples
 
 
-Disable debug point `foo`。
+Disable debug point `foo`.
 	
 	
 ```
diff --git a/docs/en/docs/admin-manual/http-actions/fe/query-profile-action.md b/docs/en/docs/admin-manual/http-actions/fe/query-profile-action.md
index 095e88e247796c..b670a0a5d0063b 100644
--- a/docs/en/docs/admin-manual/http-actions/fe/query-profile-action.md
+++ b/docs/en/docs/admin-manual/http-actions/fe/query-profile-action.md
@@ -100,7 +100,7 @@ Gets information about select queries for all fe nodes in the cluster.
 
 <version since="1.2">
 
-Admin 和 Root 用户可以查看所有 Query。普通用户仅能查看自己发送的 Query。
+Admin 和 Root 用户可以查看所有 Query.普通用户仅能查看自己发送的 Query.
 
 </version>
 
diff --git a/docs/en/docs/admin-manual/http-actions/fe/statistic-action.md b/docs/en/docs/admin-manual/http-actions/fe/statistic-action.md
index af80ed78005926..3e1d2872f6f91c 100644
--- a/docs/en/docs/admin-manual/http-actions/fe/statistic-action.md
+++ b/docs/en/docs/admin-manual/http-actions/fe/statistic-action.md
@@ -32,7 +32,7 @@ under the License.
 
 ## Description
 
-获取集群统计信息、库表数量等。
+获取集群统计信息、库表数量等.
     
 ## Path parameters
 
diff --git a/docs/en/docs/admin-manual/privilege-ldap/ldap.md b/docs/en/docs/admin-manual/privilege-ldap/ldap.md
index 21c65427fea42e..30cd57744acbbf 100644
--- a/docs/en/docs/admin-manual/privilege-ldap/ldap.md
+++ b/docs/en/docs/admin-manual/privilege-ldap/ldap.md
@@ -90,7 +90,7 @@ You need to configure the LDAP basic information in the fe/conf/ldap.conf file,
   For example, if you use the LDAP user node uid attribute as the username to log into Doris, you can configure it as:    
   ldap_user_filter = (&(uid={login}))；  
   This item can be configured using the LDAP user mailbox prefix as the user name:   
-  ldap_user_filter = (&(mail={login}@baidu.com))。
+  ldap_user_filter = (&(mail={login}@baidu.com)).
 
 * ldap_group_basedn = ou=group,dc=domain,dc=com
   base dn when Doris searches for group information in LDAP. if this item is not configured, LDAP group authorization will not be enabled. Same as ldap_ User_ Similar to basedn, it limits the scope of Doris searching for groups.
diff --git a/docs/en/docs/advanced/cold-hot-separation.md b/docs/en/docs/advanced/cold-hot-separation.md
index fadd0ed8c9c0c0..179f0c43e729f6 100644
--- a/docs/en/docs/advanced/cold-hot-separation.md
+++ b/docs/en/docs/advanced/cold-hot-separation.md
@@ -32,7 +32,7 @@ A big usage scenario in the future is similar to the es log storage. In the log
 1. The price of ordinary cloud disks of cloud manufacturers is higher than that of object storage
 2. In the actual online use of the doris cluster, the utilization rate of ordinary cloud disks cannot reach 100%
 3. Cloud disk is not paid on demand, but object storage can be paid on demand
-4. High availability based on ordinary cloud disks requires multiple replicas, and a replica migration is required for a replica exception. This problem does not exist when data is placed on the object store, because the object store is shared。
+4. High availability based on ordinary cloud disks requires multiple replicas, and a replica migration is required for a replica exception. This problem does not exist when data is placed on the object store, because the object store is shared.
 
 ## Solution
 Set the freeze time on the partition level to indicate how long the partition will be frozen, and define the location of remote storage stored after the freeze. On the be, the daemon thread will periodically determine whether the table needs to be frozen. If it does, it will upload the data to s3.
diff --git a/docs/en/docs/data-operate/import/import-way/binlog-load-manual.md b/docs/en/docs/data-operate/import/import-way/binlog-load-manual.md
index 061bfff950cfc1..717fed6fe34f40 100644
--- a/docs/en/docs/data-operate/import/import-way/binlog-load-manual.md
+++ b/docs/en/docs/data-operate/import/import-way/binlog-load-manual.md
@@ -522,7 +522,7 @@ The following configuration belongs to the system level configuration of SyncJob
 	
 * `max_bytes_sync_commit`
 
-	The maximum size of the data when the transaction is committed. If the data size received by Fe is larger than it, it will immediately commit the transaction and send the accumulated data. The default value is 64MB. If you want to modify this configuration, please ensure that this value is greater than the product of `canal.instance.memory.buffer.size` and `canal.instance.memory.buffer.mmemunit` on the canal side (16MB by default) and `min_bytes_sync_commit`。
+	The maximum size of the data when the transaction is committed. If the data size received by Fe is larger than it, it will immediately commit the transaction and send the accumulated data. The default value is 64MB. If you want to modify this configuration, please ensure that this value is greater than the product of `canal.instance.memory.buffer.size` and `canal.instance.memory.buffer.mmemunit` on the canal side (16MB by default) and `min_bytes_sync_commit`.
 	
 * `max_sync_task_threads_num`
 
diff --git a/docs/en/docs/data-operate/import/import-way/spark-load-manual.md b/docs/en/docs/data-operate/import/import-way/spark-load-manual.md
index c3c9a7dbb0fd44..312e70c686be58 100644
--- a/docs/en/docs/data-operate/import/import-way/spark-load-manual.md
+++ b/docs/en/docs/data-operate/import/import-way/spark-load-manual.md
@@ -281,7 +281,7 @@ PROPERTIES
 
 If Spark load accesses Hadoop cluster resources with Kerberos authentication, we only need to specify the following parameters when creating Spark resources:
 
-- `spark.hadoop.hadoop.security.authentication` Specify the authentication method as Kerberos for Yarn。
+- `spark.hadoop.hadoop.security.authentication` Specify the authentication method as Kerberos for Yarn.
 - `spark.hadoop.yarn.resourcemanager.principal` Specify the principal of kerberos for Yarn.
 - `spark.hadoop.yarn.resourcemanager.keytab` Specify the path to the keytab file of kerberos for Yarn. The file must be an absolute path to a file on the server where the frontend process is located. And can be accessed by the frontend process.
 - `broker.hadoop.security.authentication`: Specify the authentication method as kerberos.
diff --git a/docs/en/docs/ecosystem/flink-doris-connector.md b/docs/en/docs/ecosystem/flink-doris-connector.md
index 672d96d5dea06c..737f303f67e093 100644
--- a/docs/en/docs/ecosystem/flink-doris-connector.md
+++ b/docs/en/docs/ecosystem/flink-doris-connector.md
@@ -761,4 +761,4 @@ This problem is mainly caused by the conditional varchar/string type, which need
 
 15. **Failed to connect to backend: http://host:webserver_port, and Be is still alive**
 
-The issue may have occurred due to configuring the IP address of `be`, which is not reachable by the external Flink cluster.This is mainly because when connecting to `fe`, the address of `be` is resolved through fe. For instance, if you add a be address as '127.0.0.1', the be address obtained by the Flink cluster through fe will be '127.0.0.1:webserver_port', and Flink will connect to that address. When this issue arises, you can resolve it by adding the actual corresponding external IP address of the be to the "with" attribute:`'benodes'="be_ip:webserver_port,be_ip:webserver_port..."`.For the entire database synchronization, the following properties are available`--sink-conf benodes=be_ip:webserver,be_ip:webserver...`。
+The issue may have occurred due to configuring the IP address of `be`, which is not reachable by the external Flink cluster.This is mainly because when connecting to `fe`, the address of `be` is resolved through fe. For instance, if you add a be address as '127.0.0.1', the be address obtained by the Flink cluster through fe will be '127.0.0.1:webserver_port', and Flink will connect to that address. When this issue arises, you can resolve it by adding the actual corresponding external IP address of the be to the "with" attribute:`'benodes'="be_ip:webserver_port,be_ip:webserver_port..."`.For the entire database synchronization, the following properties are available`--sink-conf benodes=be_ip:webserver,be_ip:webserver...`.
diff --git a/docs/en/docs/ecosystem/kyuubi.md b/docs/en/docs/ecosystem/kyuubi.md
index 10a2e102b51a90..76ac1dd191cd8e 100644
--- a/docs/en/docs/ecosystem/kyuubi.md
+++ b/docs/en/docs/ecosystem/kyuubi.md
@@ -44,7 +44,7 @@ unified authentication, engine lifecycle management, etc.
 
 Download Apache Kyuubi from <https://kyuubi.apache.org/zh/releases.html>
 
-Get Apache Kyuubi 1.6.0 or above and extract it to folder。
+Get Apache Kyuubi 1.6.0 or above and extract it to folder.
 
 ### Config Doris as Kyuubi data source
 
diff --git a/docs/en/docs/ecosystem/udaf/remote-user-defined-aggregation-function.md b/docs/en/docs/ecosystem/udaf/remote-user-defined-aggregation-function.md
index 88a0382e48c311..ab38d397cfcdc4 100644
--- a/docs/en/docs/ecosystem/udaf/remote-user-defined-aggregation-function.md
+++ b/docs/en/docs/ecosystem/udaf/remote-user-defined-aggregation-function.md
@@ -81,9 +81,9 @@ PROPERTIES (["key"="value"][,...])
 ```
 Instructions:
 
-1. PROPERTIES中`symbol`Represents the name of the method passed by the RPC call, which must be set。
-2. PROPERTIES中`object_file`Represents the RPC service address. Currently, a single address and a cluster address in BRPC-compatible format are supported. Refer to the cluster connection mode[Format specification](https://github.com/apache/incubator-brpc/blob/master/docs/cn/client.md#%E8%BF%9E%E6%8E%A5%E6%9C%8D%E5%8A%A1%E9%9B%86%E7%BE%A4)。
-3. PROPERTIES中`type`Indicates the UDAF call type, which is Native by default. Rpc is transmitted when Rpc UDAF is used。
+1. PROPERTIES中`symbol`Represents the name of the method passed by the RPC call, which must be set.
+2. PROPERTIES中`object_file`Represents the RPC service address. Currently, a single address and a cluster address in BRPC-compatible format are supported. Refer to the cluster connection mode[Format specification](https://github.com/apache/incubator-brpc/blob/master/docs/cn/client.md#%E8%BF%9E%E6%8E%A5%E6%9C%8D%E5%8A%A1%E9%9B%86%E7%BE%A4).
+3. PROPERTIES中`type`Indicates the UDAF call type, which is Native by default. Rpc is transmitted when Rpc UDAF is used.
 
 Sample:
 ```sql
diff --git a/docs/en/docs/ecosystem/udf/remote-user-defined-function.md b/docs/en/docs/ecosystem/udf/remote-user-defined-function.md
index 5ccd362728e843..f73ef30f52d34c 100644
--- a/docs/en/docs/ecosystem/udf/remote-user-defined-function.md
+++ b/docs/en/docs/ecosystem/udf/remote-user-defined-function.md
@@ -81,10 +81,10 @@ PROPERTIES (["key"="value"][,...])
 ```
 Instructions:
 
-1. PROPERTIES中`symbol`Represents the name of the method passed by the RPC call, which must be set。
-2. PROPERTIES中`object_file`Represents the RPC service address. Currently, a single address and a cluster address in BRPC-compatible format are supported. Refer to the cluster connection mode[Format specification](https://github.com/apache/incubator-brpc/blob/master/docs/cn/client.md#%E8%BF%9E%E6%8E%A5%E6%9C%8D%E5%8A%A1%E9%9B%86%E7%BE%A4)。
-3. PROPERTIES中`type`Indicates the UDF call type, which is Native by default. Rpc is transmitted when Rpc UDF is used。
-4. name: A function belongs to a DB and name is of the form`dbName`.`funcName`. When `dbName` is not explicitly specified, the db of the current session is used`dbName`。
+1. PROPERTIES中`symbol`Represents the name of the method passed by the RPC call, which must be set.
+2. PROPERTIES中`object_file`Represents the RPC service address. Currently, a single address and a cluster address in BRPC-compatible format are supported. Refer to the cluster connection mode[Format specification](https://github.com/apache/incubator-brpc/blob/master/docs/cn/client.md#%E8%BF%9E%E6%8E%A5%E6%9C%8D%E5%8A%A1%E9%9B%86%E7%BE%A4).
+3. PROPERTIES中`type`Indicates the UDF call type, which is Native by default. Rpc is transmitted when Rpc UDF is used.
+4. name: A function belongs to a DB and name is of the form`dbName`.`funcName`. When `dbName` is not explicitly specified, the db of the current session is used`dbName`.
 
 Sample:
 ```sql
diff --git a/docs/en/docs/get-starting/quick-start.md b/docs/en/docs/get-starting/quick-start.md
index db145540cc7106..1a9156f98ce4c7 100644
--- a/docs/en/docs/get-starting/quick-start.md
+++ b/docs/en/docs/get-starting/quick-start.md
@@ -173,14 +173,14 @@ Next, connect to Doris through `mysql` client, mysql supports five SSL modes:
 
 3. `mysql --ssl-mode=REQUIRED -uroot -P9030 -h127.0.0.1`, force the use of SSL encrypted connections.
 
-4.`mysql --ssl-mode=VERIFY_CA --ssl-ca=ca.pem -uroot -P9030 -h127.0.0.1`, force the use of SSL encrypted connection and verify the validity of the server's identity by specifying the CA certificate。
+4.`mysql --ssl-mode=VERIFY_CA --ssl-ca=ca.pem -uroot -P9030 -h127.0.0.1`, force the use of SSL encrypted connection and verify the validity of the server's identity by specifying the CA certificate.
 
-5.`mysql --ssl-mode=VERIFY_CA --ssl-ca=ca.pem --ssl-cert=client-cert.pem --ssl-key=client-key.pem -uroot -P9030 -h127.0.0.1`, force the use of SSL encrypted connection, two-way ssl。
+5.`mysql --ssl-mode=VERIFY_CA --ssl-ca=ca.pem --ssl-cert=client-cert.pem --ssl-key=client-key.pem -uroot -P9030 -h127.0.0.1`, force the use of SSL encrypted connection, two-way ssl.
 
 >Note:
->`--ssl-mode` parameter is introduced by mysql5.7.11 version, please refer to [here](https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-connp-props-security.html) for mysql client version lower than this version。
+>`--ssl-mode` parameter is introduced by mysql5.7.11 version, please refer to [here](https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-connp-props-security.html) for mysql client version lower than this version.
 
-Doris needs a key certificate file to verify the SSL encrypted connection. The default key certificate file is located at `Doris/fe/mysql_ssl_default_certificate/`. For the generation of the key certificate file, please refer to [Key Certificate Configuration](../admin-manual/certificate.md)。
+Doris needs a key certificate file to verify the SSL encrypted connection. The default key certificate file is located at `Doris/fe/mysql_ssl_default_certificate/`. For the generation of the key certificate file, please refer to [Key Certificate Configuration](../admin-manual/certificate.md).
 
 #### Stop FE
 
@@ -347,7 +347,7 @@ Save the above data into `test.csv` file.
 
 4. Import data
 
-Here we import the data saved to the file above into the table we just created via Stream load。
+Here we import the data saved to the file above into the table we just created via Stream load.
 
 ```
 curl  --location-trusted -u root: -T test.csv -H "column_separator:," http://127.0.0.1:8030/api/demo/example_tbl/_stream_load
diff --git a/docs/en/docs/install/source-install/compilation-mac.md b/docs/en/docs/install/source-install/compilation-mac.md
index 527332ccf34b30..159d78684ef151 100644
--- a/docs/en/docs/install/source-install/compilation-mac.md
+++ b/docs/en/docs/install/source-install/compilation-mac.md
@@ -77,7 +77,7 @@ The version of jdk installed using brew is 11, because on macOS, the arm64 versi
 
 ## Start-up
 
-1. Set `file descriptors` （_**NOTICE: If you have closed the current session, you need to set this variable again**_）。
+1. Set `file descriptors` （_**NOTICE: If you have closed the current session, you need to set this variable again**_）.
     ```shell
     ulimit -n 65536
     ```
diff --git a/docs/en/docs/lakehouse/file.md b/docs/en/docs/lakehouse/file.md
index cc78338ee423fc..a5686616794f64 100644
--- a/docs/en/docs/lakehouse/file.md
+++ b/docs/en/docs/lakehouse/file.md
@@ -148,7 +148,7 @@ LIMIT 5;
 
 You can put the Table Value Function anywhere that you used to put Table in the SQL, such as in the WITH or FROM clause in CTE. In this way, you can treat the file as a normal table and conduct analysis conveniently.
 
-你也可以用过 `CREATE VIEW` 语句为 Table Value Function 创建一个逻辑视图。这样，你可以想其他视图一样，对这个 Table Value Function 进行访问、权限管理等操作，也可以让其他用户访问这个 Table Value Function。
+你也可以用过 `CREATE VIEW` 语句为 Table Value Function 创建一个逻辑视图.这样，你可以想其他视图一样，对这个 Table Value Function 进行访问、权限管理等操作，也可以让其他用户访问这个 Table Value Function.
 You can also create a logic view by using `CREATE VIEW` statement for a Table Value Function. So that you can query this view, grant priv on this view or allow other user to access this Table Value Function.
 
 ```
diff --git a/docs/en/docs/query-acceleration/join-optimization/bucket-shuffle-join.md b/docs/en/docs/query-acceleration/join-optimization/bucket-shuffle-join.md
index 4a1775d221b3b6..35870d913517cd 100644
--- a/docs/en/docs/query-acceleration/join-optimization/bucket-shuffle-join.md
+++ b/docs/en/docs/query-acceleration/join-optimization/bucket-shuffle-join.md
@@ -28,7 +28,7 @@ under the License.
 
 Bucket Shuffle Join is a new function officially added in Doris 0.14. The purpose is to provide local optimization for some join queries to reduce the time-consuming of data transmission between nodes and speed up the query.
 
-It's design, implementation can be referred to [ISSUE 4394](https://github.com/apache/incubator-doris/issues/4394)。
+It's design, implementation can be referred to [ISSUE 4394](https://github.com/apache/incubator-doris/issues/4394).
 
 ## Noun Interpretation
 
@@ -91,7 +91,7 @@ You can use the `explain` command to check whether the join is a Bucket Shuffle
 |   |  equal join conjunct: `test`.`k1` = `baseall`.`k1`                                         
 ```
 
-The join type indicates that the join method to be used is：`BUCKET_SHUFFLE`。
+The join type indicates that the join method to be used is：`BUCKET_SHUFFLE`.
 
 ## Planning rules of Bucket Shuffle Join
 
diff --git a/docs/en/docs/query-acceleration/pipeline-execution-engine.md b/docs/en/docs/query-acceleration/pipeline-execution-engine.md
index c862e4a380fbac..33a9f7d85b5c4d 100644
--- a/docs/en/docs/query-acceleration/pipeline-execution-engine.md
+++ b/docs/en/docs/query-acceleration/pipeline-execution-engine.md
@@ -32,7 +32,7 @@ under the License.
 
 Pipeline execution engine is an experimental feature added by Doris in version 2.0. The goal is to replace the current execution engine of Doris's volcano model, fully release the computing power of multi-core CPUs, and limit the number of Doris's query threads to solve the problem of Doris's execution thread bloat.
 
-Its specific design, implementation and effects can be found in [DSIP-027]([DSIP-027: Support Pipeline Exec Engine - DORIS - Apache Software Foundation](https://cwiki.apache.org/confluence/display/DORIS/DSIP-027%3A+Support+Pipeline+Exec+Engine))。
+Its specific design, implementation and effects can be found in [DSIP-027]([DSIP-027: Support Pipeline Exec Engine - DORIS - Apache Software Foundation](https://cwiki.apache.org/confluence/display/DORIS/DSIP-027%3A+Support+Pipeline+Exec+Engine)).
 
 ## Principle
 
diff --git a/docs/en/docs/releasenotes/release-1.2.2.md b/docs/en/docs/releasenotes/release-1.2.2.md
index 53dae650d4d0b6..08fd22571a03f0 100644
--- a/docs/en/docs/releasenotes/release-1.2.2.md
+++ b/docs/en/docs/releasenotes/release-1.2.2.md
@@ -76,7 +76,7 @@ Reference: [https://doris.apache.org/docs/dev/advanced/variables](https://doris.
 
 - Iceberg Catalog Support Hive Metastore and Rest Catalog type.
 
-- ES Catalog support _id column mapping。
+- ES Catalog support _id column mapping.
 
 - Optimize Iceberg V2 read performance with large number of delete rows.
 
diff --git a/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-count.md b/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-count.md
index e40be42011283a..fb90c995890e58 100644
--- a/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-count.md
+++ b/docs/en/docs/sql-manual/sql-functions/bitmap-functions/bitmap-count.md
@@ -30,7 +30,7 @@ under the License.
 
 `BITMAP BITMAP_COUNT(BITMAP lhs)`
 
-Returns the number of input bitmaps。
+Returns the number of input bitmaps.
 
 ### example
 
diff --git a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
index df5030462eed2c..d2d68aa292da96 100644
--- a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
+++ b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Create/CREATE-TABLE.md
@@ -125,7 +125,7 @@ Column definition list:
 
         Default value of the column. If the load data does not specify a value for this column, the system will assign a default value to this column.
         
-        The syntax is: `default default_value`。
+        The syntax is: `default default_value`.
         
         Currently, the default value supports two forms:
 
@@ -260,7 +260,7 @@ Partition information supports three writing methods:
 </version>
 
 
-4. MULTI RANGE：Multi build integer RANGE partitions,Define the left closed and right open interval of the zone, and step size。
+4. MULTI RANGE：Multi build integer RANGE partitions,Define the left closed and right open interval of the zone, and step size.
 
     ```
     PARTITION BY RANGE(int_col)

From 329d57fdd7cc68304a74bb8c3389e693403b458d Mon Sep 17 00:00:00 2001
From: zhengyu <freeman.zhang1992@gmail.com>
Date: Wed, 3 Jan 2024 14:07:51 +0800
Subject: [PATCH 287/299] [regression](move-memtable) test LoadStream
 on_idle_timeout (#29354)

Signed-off-by: freemandealer <freeman.zhang1992@gmail.com>
---
 be/src/common/config.cpp                      |  2 +-
 be/src/runtime/load_stream.cpp                |  2 +-
 be/src/service/internal_service.cpp           |  3 +-
 be/src/vec/sink/load_stream_stub.cpp          |  5 ++
 .../test_load_stream_fault_injection.groovy   | 59 ++++++++++++++++++-
 5 files changed, 66 insertions(+), 5 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ddb235130bc723..f3e8df44e17846 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -777,7 +777,7 @@ DEFINE_Int64(open_load_stream_timeout_ms, "60000"); // 60s
 DEFINE_Int64(close_load_stream_timeout_ms, "600000"); // 10 min
 
 // idle timeout for load stream in ms
-DEFINE_Int64(load_stream_idle_timeout_ms, "600000");
+DEFINE_mInt64(load_stream_idle_timeout_ms, "600000");
 // brpc streaming max_buf_size in bytes
 DEFINE_Int64(load_stream_max_buf_size, "20971520"); // 20MB
 // brpc streaming messages_in_batch
diff --git a/be/src/runtime/load_stream.cpp b/be/src/runtime/load_stream.cpp
index 3aad2566575511..1bc7e7b663729a 100644
--- a/be/src/runtime/load_stream.cpp
+++ b/be/src/runtime/load_stream.cpp
@@ -454,7 +454,7 @@ Status LoadStream::_append_data(const PStreamHeader& header, butil::IOBuf* data)
     IndexStreamSharedPtr index_stream;
 
     int64_t index_id = header.index_id();
-    DBUG_EXECUTE_IF("TabletStream.add_segment.unknown_indexid",
+    DBUG_EXECUTE_IF("TabletStream._append_data.unknown_indexid",
                     { index_id = UNKNOWN_ID_FOR_TEST; });
     auto it = _index_streams_map.find(index_id);
     if (it == _index_streams_map.end()) {
diff --git a/be/src/service/internal_service.cpp b/be/src/service/internal_service.cpp
index a92775a6f6e19e..d27db896ccc9f2 100644
--- a/be/src/service/internal_service.cpp
+++ b/be/src/service/internal_service.cpp
@@ -391,8 +391,7 @@ void PInternalServiceImpl::open_load_stream(google::protobuf::RpcController* con
         }
 
         stream_options.handler = load_stream.get();
-        // TODO : set idle timeout
-        // stream_options.idle_timeout_ms =
+        stream_options.idle_timeout_ms = config::load_stream_idle_timeout_ms;
 
         StreamId streamid;
         if (brpc::StreamAccept(&streamid, *cntl, &stream_options) != 0) {
diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp
index 9ad8d8805ca852..939c88e0b65d1a 100644
--- a/be/src/vec/sink/load_stream_stub.cpp
+++ b/be/src/vec/sink/load_stream_stub.cpp
@@ -21,6 +21,7 @@
 
 #include "olap/rowset/rowset_writer.h"
 #include "util/brpc_client_cache.h"
+#include "util/debug_points.h"
 #include "util/network_util.h"
 #include "util/thrift_util.h"
 #include "util/uid_util.h"
@@ -330,6 +331,10 @@ Status LoadStreamStub::_send_with_retry(butil::IOBuf& buf) {
         int ret;
         {
             SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->orphan_mem_tracker());
+            DBUG_EXECUTE_IF("LoadStreamStub._send_with_retry.delay_before_send", {
+                int64_t delay_ms = dp->param<int64>("delay_ms", 1000);
+                bthread_usleep(delay_ms * 1000);
+            });
             ret = brpc::StreamWrite(_stream_id, buf);
         }
         DBUG_EXECUTE_IF("LoadStreamStub._send_with_retry.stream_write_failed", { ret = EPIPE; });
diff --git a/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
index 09f271fead5331..f58c1226fd8b28 100644
--- a/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_load_stream_fault_injection.groovy
@@ -67,6 +67,52 @@ suite("load_stream_fault_injection", "nonConcurrent") {
         file "baseall.txt"
     }
 
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    def backendId_to_params = [string:[:]]
+
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    def set_be_param = { paramName, paramValue ->
+        // for eache be node, set paramName=paramValue
+        for (String id in backendId_to_backendIP.keySet()) {
+            def beIp = backendId_to_backendIP.get(id)
+            def bePort = backendId_to_backendHttpPort.get(id)
+            def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, paramValue))
+            assertTrue(out.contains("OK"))
+        }
+    }
+
+    def reset_be_param = { paramName ->
+        // for eache be node, reset paramName to default
+        for (String id in backendId_to_backendIP.keySet()) {
+            def beIp = backendId_to_backendIP.get(id)
+            def bePort = backendId_to_backendHttpPort.get(id)
+            def original_value = backendId_to_params.get(id).get(paramName)
+            def (code, out, err) = curl("POST", String.format("http://%s:%s/api/update_config?%s=%s", beIp, bePort, paramName, original_value))
+            assertTrue(out.contains("OK"))
+        }
+    }
+
+    def get_be_param = { paramName ->
+        // for eache be node, get param value by default
+        def paramValue = ""
+        for (String id in backendId_to_backendIP.keySet()) {
+            def beIp = backendId_to_backendIP.get(id)
+            def bePort = backendId_to_backendHttpPort.get(id)
+            // get the config value from be
+            def (code, out, err) = curl("GET", String.format("http://%s:%s/api/show_config?conf_item=%s", beIp, bePort, paramName))
+            assertTrue(code == 0)
+            assertTrue(out.contains(paramName))
+            // parsing
+            def resultList = parseJson(out)[0]
+            assertTrue(resultList.size() == 4)
+            // get original value
+            paramValue = resultList[2]
+            backendId_to_params.get(id, [:]).put(paramName, paramValue)
+        }
+    }
+
     def load_with_injection = { injection, expect_errmsg ->
         try {
             GetDebugPoint().enableDebugPointForAllBEs(injection)
@@ -110,10 +156,21 @@ suite("load_stream_fault_injection", "nonConcurrent") {
     // LoadStream add_segment meet unknown segid in request header
     load_with_injection("TabletStream.add_segment.unknown_segid", "")
     // LoadStream append_data meet unknown index id in request header
-    load_with_injection("abletStream.add_segment.unknown_indexid", "")
+    load_with_injection("TabletStream._append_data.unknown_indexid", "")
     // LoadStream dispatch meet unknown load id
     load_with_injection("LoadStream._dispatch.unknown_loadid", "")
     // LoadStream dispatch meet unknown src id
     load_with_injection("LoadStream._dispatch.unknown_srcid", "")
+
+    // LoadStream meets StreamRPC idle timeout
+    get_be_param("load_stream_idle_timeout_ms")
+    set_be_param("load_stream_idle_timeout_ms", 500)
+    try {
+        load_with_injection("LoadStreamStub._send_with_retry.delay_before_send", "")
+    } catch(Exception e) {
+        logger.info(e.getMessage())
+    } finally {
+        reset_be_param("load_stream_idle_timeout_ms")
+    }
 }
 

From e3c9f535dc8c350f40b755618a019104659dbb05 Mon Sep 17 00:00:00 2001
From: abmdocrt <Yukang.Lian2022@gmail.com>
Date: Wed, 3 Jan 2024 14:45:57 +0800
Subject: [PATCH 288/299] [refactor](wal) refactor some wal code (#29434)

---
 be/src/http/action/http_stream.cpp     | 22 ++++++++++------------
 be/src/http/action/stream_load.cpp     | 24 +++++++++++-------------
 be/src/io/fs/local_file_system.cpp     |  2 +-
 be/src/olap/wal/wal_dirs_info.cpp      | 26 +++++++++++---------------
 be/src/olap/wal/wal_dirs_info.h        |  9 ++++-----
 be/src/olap/wal/wal_manager.cpp        | 22 +++++++++-------------
 be/src/olap/wal/wal_manager.h          | 10 ++++------
 be/src/runtime/group_commit_mgr.cpp    |  6 +++---
 be/test/vec/exec/vwal_scanner_test.cpp |  2 +-
 9 files changed, 54 insertions(+), 69 deletions(-)

diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp
index b97ce2976eb8c8..f1017703570c77 100644
--- a/be/src/http/action/http_stream.cpp
+++ b/be/src/http/action/http_stream.cpp
@@ -334,19 +334,17 @@ Status HttpStreamAction::process_put(HttpRequest* http_req,
     ctx->label = ctx->put_result.params.import_label;
     ctx->put_result.params.__set_wal_id(ctx->wal_id);
     if (http_req != nullptr && http_req->header(HTTP_GROUP_COMMIT) == "async_mode") {
-        if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
-            size_t content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
-            if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
-                ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZ4FRAME ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZOP ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZ4BLOCK ||
-                ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
-                content_length *= 3;
-            }
-            ctx->put_result.params.__set_content_length(content_length);
+        size_t content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
+        if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
+            ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZ4FRAME ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZOP ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZ4BLOCK ||
+            ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
+            content_length *= 3;
         }
+        ctx->put_result.params.__set_content_length(content_length);
     }
 
     return _exec_env->stream_load_executor()->execute_plan_fragment(ctx);
diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index 88e12e19dca449..db8e56ec77b58c 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -633,19 +633,17 @@ Status StreamLoadAction::_process_put(HttpRequest* http_req,
         return plan_status;
     }
     if (http_req->header(HTTP_GROUP_COMMIT) == "async_mode") {
-        if (!http_req->header(HttpHeaders::CONTENT_LENGTH).empty()) {
-            size_t content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
-            if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
-                ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZ4FRAME ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZOP ||
-                ctx->format == TFileFormatType::FORMAT_CSV_LZ4BLOCK ||
-                ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
-                content_length *= 3;
-            }
-            ctx->put_result.params.__set_content_length(content_length);
-        }
+        size_t content_length = std::stol(http_req->header(HttpHeaders::CONTENT_LENGTH));
+        if (ctx->format == TFileFormatType::FORMAT_CSV_GZ ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZO ||
+            ctx->format == TFileFormatType::FORMAT_CSV_BZ2 ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZ4FRAME ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZOP ||
+            ctx->format == TFileFormatType::FORMAT_CSV_LZ4BLOCK ||
+            ctx->format == TFileFormatType::FORMAT_CSV_SNAPPYBLOCK) {
+            content_length *= 3;
+        }
+        ctx->put_result.params.__set_content_length(content_length);
     }
 
     VLOG_NOTICE << "params is " << apache::thrift::ThriftDebugString(ctx->put_result.params);
diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp
index 78ba222d1ffd8c..1828c7c3d2c681 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -186,7 +186,7 @@ Status LocalFileSystem::directory_size(const Path& dir_path, size_t* dir_size) {
                 try {
                     *dir_size += std::filesystem::file_size(entry);
                 } catch (const std::exception& e) {
-                    LOG(INFO) << "{}", e.what();
+                    LOG(INFO) << "failed to get file size, err: {}", e.what();
                 }
             }
         }
diff --git a/be/src/olap/wal/wal_dirs_info.cpp b/be/src/olap/wal/wal_dirs_info.cpp
index 19ad2562778047..79bd024e7d08d9 100644
--- a/be/src/olap/wal/wal_dirs_info.cpp
+++ b/be/src/olap/wal/wal_dirs_info.cpp
@@ -43,13 +43,10 @@ void WalDirInfo::set_used(size_t used) {
     _used = used;
 }
 
-void WalDirInfo::set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated) {
+void WalDirInfo::set_pre_allocated(size_t increase_pre_allocated, size_t decrease_pre_allocated) {
     std::unique_lock wlock(_lock);
-    if (is_add_pre_allocated) {
-        _pre_allocated += pre_allocated;
-    } else {
-        _pre_allocated -= pre_allocated;
-    }
+    _pre_allocated += increase_pre_allocated;
+    _pre_allocated -= decrease_pre_allocated;
 }
 
 size_t WalDirInfo::available() {
@@ -72,9 +69,6 @@ Status WalDirInfo::update_wal_dir_limit(size_t limit) {
         if (wal_disk_limit <= 0) {
             return Status::InternalError("Disk full! Please check your disk usage!");
         }
-        size_t wal_dir_size = 0;
-        RETURN_IF_ERROR(io::global_local_filesystem()->directory_size(_wal_dir, &wal_dir_size));
-        // TODO should be wal_disk_limit + wal_dir_size
         set_limit(wal_disk_limit);
     }
     return Status::OK();
@@ -91,9 +85,9 @@ Status WalDirInfo::update_wal_dir_used(size_t used) {
     return Status::OK();
 }
 
-Status WalDirInfo::update_wal_dir_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated) {
-    set_pre_allocated(pre_allocated, is_add_pre_allocated);
-    return Status::OK();
+void WalDirInfo::update_wal_dir_pre_allocated(size_t increase_pre_allocated,
+                                              size_t decrease_pre_allocated) {
+    set_pre_allocated(increase_pre_allocated, decrease_pre_allocated);
 }
 
 Status WalDirsInfo::add(const std::string& wal_dir, size_t limit, size_t used,
@@ -178,11 +172,13 @@ Status WalDirsInfo::update_all_wal_dir_used() {
     return Status::OK();
 }
 
-Status WalDirsInfo::update_wal_dir_pre_allocated(std::string wal_dir, size_t pre_allocated,
-                                                 bool is_add_pre_allocated) {
+Status WalDirsInfo::update_wal_dir_pre_allocated(std::string wal_dir, size_t increase_pre_allocated,
+                                                 size_t decrease_pre_allocated) {
     for (const auto& wal_dir_info : _wal_dirs_info_vec) {
         if (wal_dir_info->get_wal_dir() == wal_dir) {
-            return wal_dir_info->update_wal_dir_pre_allocated(pre_allocated, is_add_pre_allocated);
+            wal_dir_info->update_wal_dir_pre_allocated(increase_pre_allocated,
+                                                       decrease_pre_allocated);
+            return Status::OK();
         }
     }
     return Status::InternalError("Can not find wal dir in wal disks info.");
diff --git a/be/src/olap/wal/wal_dirs_info.h b/be/src/olap/wal/wal_dirs_info.h
index 91a26af2b5d02a..eda9cc72d30d6f 100644
--- a/be/src/olap/wal/wal_dirs_info.h
+++ b/be/src/olap/wal/wal_dirs_info.h
@@ -42,12 +42,11 @@ class WalDirInfo {
     size_t get_limit();
     void set_limit(size_t limit);
     void set_used(size_t used);
-    // TODO increase_pre_allocated and decrease_pre_allocated
-    void set_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated);
+    void set_pre_allocated(size_t increase_pre_allocated, size_t decrease_pre_allocated);
     size_t available();
     Status update_wal_dir_limit(size_t limit = -1);
     Status update_wal_dir_used(size_t used = -1);
-    Status update_wal_dir_pre_allocated(size_t pre_allocated, bool is_add_pre_allocated = true);
+    void update_wal_dir_pre_allocated(size_t increase_pre_allocated, size_t decrease_pre_allocated);
 
 private:
     std::string _wal_dir;
@@ -70,8 +69,8 @@ class WalDirsInfo {
     Status update_all_wal_dir_limit();
     Status update_wal_dir_used(std::string wal_dir, size_t used = -1);
     Status update_all_wal_dir_used();
-    Status update_wal_dir_pre_allocated(std::string wal_dir, size_t pre_allocated,
-                                        bool is_add_pre_allocated);
+    Status update_wal_dir_pre_allocated(std::string wal_dir, size_t increase_pre_allocated,
+                                        size_t decrease_pre_allocated);
     Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
 
 private:
diff --git a/be/src/olap/wal/wal_manager.cpp b/be/src/olap/wal/wal_manager.cpp
index 9af91cf7b55348..9d92f5c9e6ad5f 100644
--- a/be/src/olap/wal/wal_manager.cpp
+++ b/be/src/olap/wal/wal_manager.cpp
@@ -232,8 +232,8 @@ void WalManager::print_wal_status_queue() {
     LOG(INFO) << ss.str();
 }
 
-Status WalManager::add_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id,
-                                const std::string& label, std::string& base_path) {
+Status WalManager::create_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id,
+                                   const std::string& label, std::string& base_path) {
     base_path = _wal_dirs_info->get_available_random_wal_dir();
     std::stringstream ss;
     ss << base_path << "/" << std::to_string(db_id) << "/" << std::to_string(table_id) << "/"
@@ -286,11 +286,6 @@ Status WalManager::create_wal_writer(int64_t wal_id, std::shared_ptr<WalWriter>&
     LOG(INFO) << "create wal " << wal_path;
     wal_writer = std::make_shared<WalWriter>(wal_path);
     RETURN_IF_ERROR(wal_writer->init());
-    {
-        // TODO no use, should remove it
-        std::lock_guard<std::shared_mutex> wrlock(_wal_lock);
-        _wal_id_to_writer_map.emplace(wal_id, wal_writer);
-    }
     return Status::OK();
 }
 
@@ -431,8 +426,8 @@ Status WalManager::delete_wal(int64_t wal_id, size_t block_queue_pre_allocated)
             _wal_path_map.erase(wal_id);
         }
     }
-    RETURN_IF_ERROR(update_wal_dir_pre_allocated(_get_base_wal_path(wal_path),
-                                                 block_queue_pre_allocated, false));
+    RETURN_IF_ERROR(update_wal_dir_pre_allocated(_get_base_wal_path(wal_path), 0,
+                                                 block_queue_pre_allocated));
     return Status::OK();
 }
 
@@ -481,10 +476,11 @@ Status WalManager::update_wal_dir_used(const std::string& wal_dir, size_t used)
     return _wal_dirs_info->update_wal_dir_used(wal_dir, used);
 }
 
-Status WalManager::update_wal_dir_pre_allocated(const std::string& wal_dir, size_t pre_allocated,
-                                                bool is_add_pre_allocated) {
-    return _wal_dirs_info->update_wal_dir_pre_allocated(wal_dir, pre_allocated,
-                                                        is_add_pre_allocated);
+Status WalManager::update_wal_dir_pre_allocated(const std::string& wal_dir,
+                                                size_t increase_pre_allocated,
+                                                size_t decrease_pre_allocated) {
+    return _wal_dirs_info->update_wal_dir_pre_allocated(wal_dir, increase_pre_allocated,
+                                                        decrease_pre_allocated);
 }
 
 Status WalManager::_update_wal_dir_info_thread() {
diff --git a/be/src/olap/wal/wal_manager.h b/be/src/olap/wal/wal_manager.h
index f4e28445f8f786..f6a3bfef79891b 100644
--- a/be/src/olap/wal/wal_manager.h
+++ b/be/src/olap/wal/wal_manager.h
@@ -63,14 +63,14 @@ class WalManager {
     // wal back pressure
     Status update_wal_dir_limit(const std::string& wal_dir, size_t limit = -1);
     Status update_wal_dir_used(const std::string& wal_dir, size_t used = -1);
-    Status update_wal_dir_pre_allocated(const std::string& wal_dir, size_t pre_allocated,
-                                        bool is_add_pre_allocated);
+    Status update_wal_dir_pre_allocated(const std::string& wal_dir, size_t increase_pre_allocated,
+                                        size_t decrease_pre_allocated);
     Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
     size_t get_max_available_size();
 
     // replay wal
-    Status add_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id, const std::string& label,
-                        std::string& base_path);
+    Status create_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id,
+                           const std::string& label, std::string& base_path);
     Status get_wal_path(int64_t wal_id, std::string& wal_path);
     Status delete_wal(int64_t wal_id, size_t block_queue_pre_allocated = 0);
     Status add_recover_wal(int64_t db_id, int64_t table_id, int64_t wal_id, std::string wal);
@@ -130,8 +130,6 @@ class WalManager {
 
     std::shared_mutex _wal_lock;
     std::unordered_map<int64_t, std::string> _wal_path_map;
-    // TODO no use? need remove it. And the map dose not clear
-    std::unordered_map<int64_t, std::shared_ptr<WalWriter>> _wal_id_to_writer_map;
 
     // TODO Now only used for debug wal status, consider remove it
     std::shared_mutex _wal_status_lock;
diff --git a/be/src/runtime/group_commit_mgr.cpp b/be/src/runtime/group_commit_mgr.cpp
index 2971138d5b6d53..8a81388ecd2948 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -491,8 +491,8 @@ Status GroupCommitMgr::get_load_block_queue(int64_t table_id, const TUniqueId& i
 Status LoadBlockQueue::create_wal(int64_t db_id, int64_t tb_id, int64_t wal_id,
                                   const std::string& import_label, WalManager* wal_manager,
                                   std::vector<TSlotDescriptor>& slot_desc, int be_exe_version) {
-    RETURN_IF_ERROR(ExecEnv::GetInstance()->wal_mgr()->add_wal_path(db_id, tb_id, wal_id,
-                                                                    import_label, _wal_base_path));
+    RETURN_IF_ERROR(ExecEnv::GetInstance()->wal_mgr()->create_wal_path(
+            db_id, tb_id, wal_id, import_label, _wal_base_path));
     _v_wal_writer = std::make_shared<vectorized::VWalWriter>(
             tb_id, wal_id, import_label, wal_manager, slot_desc, be_exe_version);
     return _v_wal_writer->init();
@@ -515,7 +515,7 @@ bool LoadBlockQueue::has_enough_wal_disk_space(size_t pre_allocated) {
         }
     }
     if (pre_allocated < available_bytes) {
-        Status st = wal_mgr->update_wal_dir_pre_allocated(_wal_base_path, pre_allocated, true);
+        Status st = wal_mgr->update_wal_dir_pre_allocated(_wal_base_path, pre_allocated, 0);
         if (!st.ok()) {
             LOG(WARNING) << "update wal dir pre_allocated failed, reason: " << st.to_string();
         }
diff --git a/be/test/vec/exec/vwal_scanner_test.cpp b/be/test/vec/exec/vwal_scanner_test.cpp
index 2d786679d4e6cd..cf9e1733643f12 100644
--- a/be/test/vec/exec/vwal_scanner_test.cpp
+++ b/be/test/vec/exec/vwal_scanner_test.cpp
@@ -218,7 +218,7 @@ void VWalScannerTest::init() {
     _env->_wal_manager = WalManager::create_shared(_env, wal_dir);
     std::string base_path;
     auto st = _env->_wal_manager->_init_wal_dirs_info();
-    st = _env->_wal_manager->add_wal_path(db_id, tb_id, txn_id, label, base_path);
+    st = _env->_wal_manager->create_wal_path(db_id, tb_id, txn_id, label, base_path);
 }
 
 TEST_F(VWalScannerTest, normal) {

From b2a28fcfaac89149431434416cff2670fed71082 Mon Sep 17 00:00:00 2001
From: Dongyang Li <hello_stephen@qq.com>
Date: Wed, 3 Jan 2024 16:02:15 +0800
Subject: [PATCH 289/299] [fix](ci) exclude case load_stream_fault_injection
 (#29465)

---
 regression-test/pipeline/p0/conf/regression-conf.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/regression-test/pipeline/p0/conf/regression-conf.groovy b/regression-test/pipeline/p0/conf/regression-conf.groovy
index 849a1298301e15..e925459e7bf151 100644
--- a/regression-test/pipeline/p0/conf/regression-conf.groovy
+++ b/regression-test/pipeline/p0/conf/regression-conf.groovy
@@ -54,8 +54,8 @@ testDirectories = ""
 // this groups will not be executed
 excludeGroups = ""
 // this suites will not be executed
-
-excludeSuites = "test_profile,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_information_schema_external"
+// load_stream_fault_injection may cause bad disk
+excludeSuites = "load_stream_fault_injection,test_profile,test_spark_load,test_refresh_mtmv,test_bitmap_filter,test_information_schema_external"
 
 // this directories will not be executed
 excludeDirectories = "workload_manager_p1,nereids_rules_p0/subquery"

From 44628d37c8457c37883d5daa38e19fd1d9505700 Mon Sep 17 00:00:00 2001
From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com>
Date: Wed, 3 Jan 2024 18:10:38 +0800
Subject: [PATCH 290/299] Enable minmax push down for unique table while doing
 analyze. (#29462)

---
 .../java/org/apache/doris/statistics/util/StatisticsUtil.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index cee182273679af..d6c171161fc7aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -192,6 +192,7 @@ public static AutoCloseConnectContext buildConnectContext(boolean limitScan) {
         sessionVariable.insertTimeoutS = StatisticsUtil.getAnalyzeTimeout();
         sessionVariable.enableFileCache = false;
         sessionVariable.forbidUnknownColStats = false;
+        sessionVariable.enablePushDownMinMaxOnUnique = true;
         connectContext.setEnv(Env.getCurrentEnv());
         connectContext.setDatabase(FeConstants.INTERNAL_DB_NAME);
         connectContext.setQualifiedUser(UserIdentity.ROOT.getQualifiedUser());

From d19530c4c28baf1fc5498590de6380b542a2fcde Mon Sep 17 00:00:00 2001
From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com>
Date: Wed, 3 Jan 2024 18:13:38 +0800
Subject: [PATCH 291/299] [Fix](Nereids) fix leading hint dealing with big
 brace (#29405)

Co-authored-by: libinfeng <libinfeng@selectdb.com>
---
 .../doris/nereids/hint/LeadingHint.java       | 17 +++++++-
 .../data/nereids_p0/hint/fix_leading.out      | 21 +++++++++
 .../suites/nereids_p0/hint/fix_leading.groovy | 43 +++++++++++++++++++
 3 files changed, 79 insertions(+), 2 deletions(-)
 create mode 100644 regression-test/data/nereids_p0/hint/fix_leading.out
 create mode 100644 regression-test/suites/nereids_p0/hint/fix_leading.groovy

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
index 97af2e262236e6..4d1abbbf951ba1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java
@@ -83,15 +83,28 @@ public LeadingHint(String hintName, List<String> parameters, String originalStri
         super(hintName);
         this.originalString = originalString;
         int level = 0;
+        Stack<Boolean> brace = new Stack<>();
+        String lastParameter = "";
         for (String parameter : parameters) {
             if (parameter.equals("{")) {
-                ++level;
+                if (lastParameter.equals("}")) {
+                    level += 2;
+                    brace.push(true);
+                } else {
+                    ++level;
+                    brace.push(false);
+                }
             } else if (parameter.equals("}")) {
-                level--;
+                if (brace.pop().equals(true)) {
+                    level -= 2;
+                } else {
+                    level--;
+                }
             } else {
                 tablelist.add(parameter);
                 levellist.add(level);
             }
+            lastParameter = parameter;
         }
     }
 
diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out b/regression-test/data/nereids_p0/hint/fix_leading.out
new file mode 100644
index 00000000000000..86abd918f91939
--- /dev/null
+++ b/regression-test/data/nereids_p0/hint/fix_leading.out
@@ -0,0 +1,21 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select1 --
+PhysicalResultSink
+--PhysicalDistribute
+----PhysicalProject
+------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3) and (t1.c1 = t4.c4)) otherCondition=()
+--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=()
+----------PhysicalOlapScan[t1]
+----------PhysicalDistribute
+------------PhysicalOlapScan[t2]
+--------PhysicalDistribute
+----------NestedLoopJoin[CROSS_JOIN]
+------------PhysicalOlapScan[t3]
+------------PhysicalDistribute
+--------------PhysicalOlapScan[t4]
+
+Hint log:
+Used: leading({ t1 t2 } { t3 t4 })
+UnUsed:
+SyntaxError:
+
diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy
new file mode 100644
index 00000000000000..b1da3b44b69203
--- /dev/null
+++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+suite("fix_leading") {
+    // create database and tables
+    sql 'DROP DATABASE IF EXISTS fix_leading'
+    sql 'CREATE DATABASE IF NOT EXISTS fix_leading'
+    sql 'use fix_leading'
+
+    // setting planner to nereids
+    sql 'set enable_nereids_planner=true'
+    sql 'set enable_fallback_to_original_planner=false'
+
+    // create tables
+    sql """drop table if exists t1;"""
+    sql """drop table if exists t2;"""
+    sql """drop table if exists t3;"""
+    sql """drop table if exists t4;"""
+
+    sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');"""
+    sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');"""
+    sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');"""
+    sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');"""
+
+    // bug fix 1: {t1 t2}{t3 t4} miss levels
+    qt_select1 """explain shape plan select /*+ leading({t1 t2}{t3 t4}) */ * from t1 join t2 on c1 = c2 join t3 on c1 = c3 join t4 on c1 = c4;"""
+}

From 2386a1ce5a469272444cac433169b510e02e39c6 Mon Sep 17 00:00:00 2001
From: Vallish Pai <vallishpai@gmail.com>
Date: Wed, 3 Jan 2024 15:45:00 +0530
Subject: [PATCH 292/299] [fix](docs)Fix description for error code 1051 #27751
 (#27977)

---
 docs/en/docs/admin-manual/maint-monitor/doris-error-code.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/en/docs/admin-manual/maint-monitor/doris-error-code.md b/docs/en/docs/admin-manual/maint-monitor/doris-error-code.md
index 9e243222b0e463..70d712b2962734 100644
--- a/docs/en/docs/admin-manual/maint-monitor/doris-error-code.md
+++ b/docs/en/docs/admin-manual/maint-monitor/doris-error-code.md
@@ -38,7 +38,7 @@ under the License.
 | 1047       | The user entered an invalid operation instruction |
 | 1049       | The user specified an invalid database |
 | 1050       | Data table already exists                      |
-| 1051       | Invalid data table                               |
+| 1051       | The user specified an unknown table            |
 | 1052       | The specified column name is ambiguous, and the corresponding column cannot be uniquely determined |
 | 1053       | Illegal data column is specified for Semi-Join/Anti-Join query |
 | 1054       | The specified column does not exist in the table |
@@ -52,6 +52,7 @@ under the License.
 | 1102       | Incorrect database name                        |
 | 1104       | Incorrect data table name                      |
 | 1105       | Other errors                                         |
+| 1109       | The user specified an unknown table in the current database |
 | 1110       | Duplicate columns specified in the subquery |
 | 1111       | Illegal use of aggregate functions in the Where clause |
 | 1113       | The column set of the newly created table cannot be empty |

From 49a3bab399ef743ad38ef9bf2ec5f71e56edea14 Mon Sep 17 00:00:00 2001
From: seawinde <149132972+seawinde@users.noreply.github.com>
Date: Wed, 3 Jan 2024 18:58:18 +0800
Subject: [PATCH 293/299] [fix](nereids) fix aggregate function roll up when
 expression arguments is not equals (#29256)

when aggregate function roll up, we should check the qury and mv function argument is equal
such as mv def and query sql as following, it should not rewrite success, because the  bitmap_union_basic field augument is
not equal to the `count(distinct case when o_shippriority > 10 and o_orderkey IN (1, 3) then o_custkey else null end)`  field in query

mv def:
>      select l_shipdate, o_orderdate, l_partkey, l_suppkey,
>            sum(o_totalprice) as sum_total,
>            max(o_totalprice) as max_total,
>            min(o_totalprice) as min_total,
>            count(*) as count_all,
>            bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic
>           from lineitem
>           left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate
>            group by
>         l_shipdate,
>         o_orderdate,
>          l_partkey,
>         l_suppkey;

query sql:

>             select t1.l_partkey, t1.l_suppkey, o_orderdate,
>           sum(o_totalprice),
>            max(o_totalprice),
>           min(o_totalprice),
>           count(*),
>            count(distinct case when o_shippriority > 10 and o_orderkey IN (1, 3) then o_custkey else null end)
>            from (select * from lineitem where l_shipdate = '2023-12-11') t1
>            left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate
>            group by
>            o_orderdate,
>            l_partkey,
>            l_suppkey;
---
 ...AbstractMaterializedViewAggregateRule.java | 103 ++++++++++++++----
 .../apache/doris/nereids/trees/TreeNode.java  |  28 +++++
 .../doris/nereids/trees/expressions/Any.java  |  10 ++
 .../aggregate_with_roll_up.out                |   6 +
 .../aggregate_with_roll_up.groovy             |  35 ++++++
 5 files changed, 158 insertions(+), 24 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
index 685f8a8c3a9eab..11faaa6a6d3be8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
@@ -26,6 +26,7 @@
 import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping;
 import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping;
 import org.apache.doris.nereids.trees.expressions.Any;
+import org.apache.doris.nereids.trees.expressions.Cast;
 import org.apache.doris.nereids.trees.expressions.ExprId;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -35,11 +36,14 @@
 import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion;
 import org.apache.doris.nereids.trees.expressions.functions.agg.CouldRollUp;
 import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
+import org.apache.doris.nereids.trees.expressions.functions.scalar.ToBitmap;
 import org.apache.doris.nereids.trees.plans.Plan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.types.BigIntType;
 import org.apache.doris.nereids.util.ExpressionUtils;
 
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
@@ -47,10 +51,11 @@
 import org.apache.logging.log4j.Logger;
 
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -60,15 +65,18 @@
  */
 public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule {
 
-    protected static final Map<Expression, Expression>
-            AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = new HashMap<>();
+    // we only support roll up function which has only one argument currently
+    protected static final Multimap<Expression, Expression>
+            AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP = ArrayListMultimap.create();
     protected final String currentClassName = this.getClass().getSimpleName();
 
     private final Logger logger = LogManager.getLogger(this.getClass());
 
     static {
         AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE),
-                new BitmapUnion(Any.INSTANCE));
+                new BitmapUnion(new ToBitmap(new Cast(Any.INSTANCE, BigIntType.INSTANCE))));
+        AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.put(new Count(true, Any.INSTANCE),
+                new BitmapUnion(new ToBitmap(Any.INSTANCE)));
     }
 
     @Override
@@ -249,17 +257,30 @@ protected Plan rewriteQueryByView(MatchMode matchMode,
         return rewrittenAggregate;
     }
 
-    // only support sum roll up, support other agg functions later.
-    private Function rollup(AggregateFunction queryFunction,
-            Expression queryFunctionShuttled,
+    /**
+     * Roll up query aggregate function when query dimension num is less than mv dimension num,
+     *
+     * @param queryAggregateFunction query aggregate function to roll up.
+     * @param queryAggregateFunctionShuttled query aggregate function shuttled by lineage.
+     * @param mvExprToMvScanExprQueryBased mv def sql output expressions to mv result data output mapping.
+     *         <p>
+     *         Such as query is
+     *         select max(a) + 1 from table group by b.
+     *         mv is
+     *         select max(a) from table group by a, b.
+     *         the queryAggregateFunction is max(a), queryAggregateFunctionShuttled is max(a) + 1
+     *         mvExprToMvScanExprQueryBased is { max(a) : MTMVScan(output#0) }
+     */
+    private Function rollup(AggregateFunction queryAggregateFunction,
+            Expression queryAggregateFunctionShuttled,
             Map<Expression, Expression> mvExprToMvScanExprQueryBased) {
-        if (!(queryFunction instanceof CouldRollUp)) {
+        if (!(queryAggregateFunction instanceof CouldRollUp)) {
             return null;
         }
         Expression rollupParam = null;
-        if (mvExprToMvScanExprQueryBased.containsKey(queryFunctionShuttled)) {
+        if (mvExprToMvScanExprQueryBased.containsKey(queryAggregateFunctionShuttled)) {
             // function can rewrite by view
-            rollupParam = mvExprToMvScanExprQueryBased.get(queryFunctionShuttled);
+            rollupParam = mvExprToMvScanExprQueryBased.get(queryAggregateFunctionShuttled);
         } else {
             // function can not rewrite by view, try to use complex roll up param
             // eg: query is count(distinct param), mv sql is bitmap_union(to_bitmap(param))
@@ -267,7 +288,8 @@ private Function rollup(AggregateFunction queryFunction,
                 if (!(mvExprShuttled instanceof Function)) {
                     continue;
                 }
-                if (isAggregateFunctionEquivalent(queryFunction, (Function) mvExprShuttled)) {
+                if (isAggregateFunctionEquivalent(queryAggregateFunction, queryAggregateFunctionShuttled,
+                        (Function) mvExprShuttled)) {
                     rollupParam = mvExprToMvScanExprQueryBased.get(mvExprShuttled);
                 }
             }
@@ -276,7 +298,7 @@ private Function rollup(AggregateFunction queryFunction,
             return null;
         }
         // do roll up
-        return ((CouldRollUp) queryFunction).constructRollUp(rollupParam);
+        return ((CouldRollUp) queryAggregateFunction).constructRollUp(rollupParam);
     }
 
     private Pair<Set<? extends Expression>, Set<? extends Expression>> topPlanSplitToGroupAndFunction(
@@ -347,22 +369,55 @@ protected boolean checkPattern(StructInfo structInfo) {
         return true;
     }
 
-    private boolean isAggregateFunctionEquivalent(Function queryFunction, Function viewFunction) {
+    /**
+     * Check the queryFunction is equivalent to view function when function roll up.
+     * Not only check the function name but also check the argument between query and view aggregate function.
+     * Such as query is
+     * select count(distinct a) + 1 from table group by b.
+     * mv is
+     * select bitmap_union(to_bitmap(a)) from table group by a, b.
+     * the queryAggregateFunction is count(distinct a), queryAggregateFunctionShuttled is count(distinct a) + 1
+     * mvExprToMvScanExprQueryBased is { bitmap_union(to_bitmap(a)) : MTMVScan(output#0) }
+     * This will check the count(distinct a) in query is equivalent to  bitmap_union(to_bitmap(a)) in mv,
+     * and then check their arguments is equivalent.
+     */
+    private boolean isAggregateFunctionEquivalent(Function queryFunction, Expression queryFunctionShuttled,
+            Function viewFunction) {
         if (queryFunction.equals(viewFunction)) {
             return true;
         }
-        // get query equivalent function
-        Expression equivalentFunction = null;
-        for (Map.Entry<Expression, Expression> entry : AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.entrySet()) {
-            if (entry.getKey().equals(queryFunction)) {
-                equivalentFunction = entry.getValue();
+        // check the argument of rollup function is equivalent to view function or not
+        for (Map.Entry<Expression, Collection<Expression>> equivalentFunctionEntry :
+                AGGREGATE_ROLL_UP_EQUIVALENT_FUNCTION_MAP.asMap().entrySet()) {
+            if (equivalentFunctionEntry.getKey().equals(queryFunction)) {
+                // check is have equivalent function or not
+                for (Expression equivalentFunction : equivalentFunctionEntry.getValue()) {
+                    if (!Any.equals(equivalentFunction, viewFunction)) {
+                        continue;
+                    }
+                    // check param in query function is same as the view function
+                    List<Expression> viewFunctionArguments = extractViewArguments(equivalentFunction, viewFunction);
+                    if (queryFunctionShuttled.getArguments().size() != 1 || viewFunctionArguments.size() != 1) {
+                        continue;
+                    }
+                    if (Objects.equals(queryFunctionShuttled.getArguments().get(0), viewFunctionArguments.get(0))) {
+                        return true;
+                    }
+                }
             }
         }
-        // check is have equivalent function or not
-        if (equivalentFunction == null) {
-            return false;
-        }
-        // current compare
-        return equivalentFunction.equals(viewFunction);
+        return false;
+    }
+
+    /**
+     * Extract the view function arguments by equivalentFunction pattern
+     * Such as equivalentFunction def is bitmap_union(to_bitmap(Any.INSTANCE)),
+     * viewFunction is bitmap_union(to_bitmap(case when a = 5 then 1 else 2 end))
+     * after extracting, the return argument is: case when a = 5 then 1 else 2 end
+     */
+    private List<Expression> extractViewArguments(Expression equivalentFunction, Function viewFunction) {
+        Set<Object> exprSetToRemove = equivalentFunction.collectToSet(expr -> !(expr instanceof Any));
+        return viewFunction.collectFirst(expr ->
+                exprSetToRemove.stream().noneMatch(exprToRemove -> exprToRemove.equals(expr)));
     }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
index 557ff43b51d994..00ac71eaf24c33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TreeNode.java
@@ -22,6 +22,7 @@
 import com.google.common.collect.ImmutableSet;
 
 import java.util.ArrayDeque;
+import java.util.ArrayList;
 import java.util.Deque;
 import java.util.List;
 import java.util.Set;
@@ -150,6 +151,19 @@ default NODE_TYPE rewriteUp(Function<NODE_TYPE, NODE_TYPE> rewriteFunction) {
         return rewriteFunction.apply(rewrittenChildren);
     }
 
+    /**
+     * Foreach treeNode. Top-down traverse implicitly, stop traverse if satisfy test.
+     * @param func foreach function
+     */
+    default void foreach(Predicate<TreeNode<NODE_TYPE>> func) {
+        boolean valid = func.test(this);
+        if (!valid) {
+            for (NODE_TYPE child : children()) {
+                child.foreach(func);
+            }
+        }
+    }
+
     /**
      * Foreach treeNode. Top-down traverse implicitly.
      * @param func foreach function
@@ -241,6 +255,20 @@ default <T> Set<T> collectToSet(Predicate<TreeNode<NODE_TYPE>> predicate) {
         return (Set<T>) result.build();
     }
 
+    /**
+     * Collect the nodes that satisfied the predicate firstly.
+     */
+    default <T> List<T> collectFirst(Predicate<TreeNode<NODE_TYPE>> predicate) {
+        List<TreeNode<NODE_TYPE>> result = new ArrayList<>();
+        foreach(node -> {
+            if (result.isEmpty() && predicate.test(node)) {
+                result.add(node);
+            }
+            return !result.isEmpty();
+        });
+        return (List<T>) ImmutableList.copyOf(result);
+    }
+
     /**
      * iterate top down and test predicate if contains any instance of the classes
      * @param types classes array
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Any.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Any.java
index 43d284bf67838c..2e4bc745b2a593 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Any.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Any.java
@@ -24,6 +24,7 @@
 import com.google.common.collect.ImmutableList;
 
 import java.util.List;
+import java.util.Objects;
 
 /**
  * This represents any expression, it means it equals any expression
@@ -55,6 +56,15 @@ public boolean equals(Object o) {
         return true;
     }
 
+    /**
+     * Equals with direction
+     * Since the equals method in Any is always true, that means Any is equals to others, but not equal in reverse.
+     * The expression with Any should always be the first argument.
+     */
+    public static boolean equals(Expression expressionWithAny, Expression target) {
+        return Objects.equals(expressionWithAny, target);
+    }
+
     @Override
     public int hashCode() {
         return 0;
diff --git a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
index fb223bc661b9b1..334980ed00c23b 100644
--- a/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
+++ b/regression-test/data/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.out
@@ -5,6 +5,12 @@
 -- !query13_0_after --
 3	3	2023-12-11	43.20	43.20	43.20	1	0
 
+-- !query13_1_before --
+3	3	2023-12-11	43.20	43.20	43.20	1	0
+
+-- !query13_1_after --
+3	3	2023-12-11	43.20	43.20	43.20	1	0
+
 -- !query14_0_before --
 2	3	2023-12-08	20.00	10.50	9.50	2	0
 2	3	2023-12-12	\N	\N	\N	1	0
diff --git a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
index fd3c02408d954d..e9d1ee76b37293 100644
--- a/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
+++ b/regression-test/suites/nereids_rules_p0/mv/agg_with_roll_up/aggregate_with_roll_up.groovy
@@ -247,6 +247,41 @@ suite("aggregate_with_roll_up") {
     sql """ DROP MATERIALIZED VIEW IF EXISTS mv13_0"""
 
 
+    def mv13_1 = """
+            select l_shipdate, o_orderdate, l_partkey, l_suppkey, 
+            sum(o_totalprice) as sum_total, 
+            max(o_totalprice) as max_total, 
+            min(o_totalprice) as min_total, 
+            count(*) as count_all, 
+            bitmap_union(to_bitmap(case when o_shippriority > 1 and o_orderkey IN (1, 3) then o_custkey else null end)) as bitmap_union_basic 
+            from lineitem 
+            left join orders on lineitem.l_orderkey = orders.o_orderkey and l_shipdate = o_orderdate 
+            group by 
+            l_shipdate, 
+            o_orderdate, 
+            l_partkey, 
+            l_suppkey;
+    """
+    def query13_1 = """
+            select t1.l_partkey, t1.l_suppkey, o_orderdate,
+            sum(o_totalprice),
+            max(o_totalprice),
+            min(o_totalprice),
+            count(*),
+            count(distinct case when o_shippriority > 10 and o_orderkey IN (1, 3) then o_custkey else null end)
+            from (select * from lineitem where l_shipdate = '2023-12-11') t1
+            left join orders on t1.l_orderkey = orders.o_orderkey and t1.l_shipdate = o_orderdate
+            group by
+            o_orderdate, 
+            l_partkey,
+            l_suppkey;
+    """
+    order_qt_query13_1_before "${query13_1}"
+    check_not_match(mv13_1, query13_1, "mv13_1")
+    order_qt_query13_1_after "${query13_1}"
+    sql """ DROP MATERIALIZED VIEW IF EXISTS mv13_1"""
+
+
     // filter inside + right + use roll up dimension
     def mv14_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey, " +
             "sum(o_totalprice) as sum_total, " +

From c84cd30223ca92702f6d61cad22b077686978648 Mon Sep 17 00:00:00 2001
From: Gabriel <gabrielleebuaa@gmail.com>
Date: Wed, 3 Jan 2024 20:29:04 +0800
Subject: [PATCH 294/299] [pipelineX](fix) Fix query cancel timeout (#29460)

There are 2 potential reasons to cancel pipelineX query timeout.

Cancel fragment context first and set ready to execute will set cancel flag to false.
Dead lock.
---
 be/src/runtime/fragment_mgr.cpp          | 38 +++++++++++++-----------
 be/src/runtime/query_context.cpp         |  4 ++-
 be/src/vec/exec/scan/scanner_context.cpp |  1 +
 3 files changed, 24 insertions(+), 19 deletions(-)

diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index d265b5e35b5203..6d8b99c62fd579 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -574,27 +574,29 @@ Status FragmentMgr::start_query_execution(const PExecPlanFragmentStartRequest* r
 
 void FragmentMgr::remove_pipeline_context(
         std::shared_ptr<pipeline::PipelineFragmentContext> f_context) {
-    std::lock_guard<std::mutex> lock(_lock);
-    auto query_id = f_context->get_query_id();
     auto* q_context = f_context->get_query_ctx();
-    std::vector<TUniqueId> ins_ids;
-    f_context->instance_ids(ins_ids);
-    bool all_done = q_context->countdown(ins_ids.size());
-    for (const auto& ins_id : ins_ids) {
-        {
-            std::lock_guard<std::mutex> plock(q_context->pipeline_lock);
-            if (q_context->fragment_id_to_pipeline_ctx.contains(f_context->get_fragment_id())) {
-                q_context->fragment_id_to_pipeline_ctx.erase(f_context->get_fragment_id());
-            }
+    {
+        std::lock_guard<std::mutex> lock(_lock);
+        auto query_id = f_context->get_query_id();
+        std::vector<TUniqueId> ins_ids;
+        f_context->instance_ids(ins_ids);
+        bool all_done = q_context->countdown(ins_ids.size());
+        for (const auto& ins_id : ins_ids) {
+            LOG_INFO("Removing query {} instance {}, all done? {}", print_id(query_id),
+                     print_id(ins_id), all_done);
+            _pipeline_map.erase(ins_id);
+            g_pipeline_fragment_instances_count << -1;
+        }
+        if (all_done) {
+            LOG_INFO("Query {} finished", print_id(query_id));
+            _query_ctx_map.erase(query_id);
         }
-        LOG_INFO("Removing query {} instance {}, all done? {}", print_id(query_id),
-                 print_id(ins_id), all_done);
-        _pipeline_map.erase(ins_id);
-        g_pipeline_fragment_instances_count << -1;
     }
-    if (all_done) {
-        LOG_INFO("Query {} finished", print_id(query_id));
-        _query_ctx_map.erase(query_id);
+    {
+        std::lock_guard<std::mutex> plock(q_context->pipeline_lock);
+        if (q_context->fragment_id_to_pipeline_ctx.contains(f_context->get_fragment_id())) {
+            q_context->fragment_id_to_pipeline_ctx.erase(f_context->get_fragment_id());
+        }
     }
 }
 
diff --git a/be/src/runtime/query_context.cpp b/be/src/runtime/query_context.cpp
index 9a90ea344eb479..e60f29b291551c 100644
--- a/be/src/runtime/query_context.cpp
+++ b/be/src/runtime/query_context.cpp
@@ -79,7 +79,9 @@ void QueryContext::set_ready_to_execute(bool is_cancelled) {
     _execution_dependency->set_ready();
     {
         std::lock_guard<std::mutex> l(_start_lock);
-        _is_cancelled = is_cancelled;
+        if (!_is_cancelled) {
+            _is_cancelled = is_cancelled;
+        }
         _ready_to_execute = true;
     }
     if (query_mem_tracker && is_cancelled) {
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index 9c87967f505568..089f52cf61b6a7 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -428,6 +428,7 @@ bool ScannerContext::set_status_on_error(const Status& status, bool need_lock) {
         _status_error = true;
         _blocks_queue_added_cv.notify_one();
         _should_stop = true;
+        _state->get_query_ctx()->set_exec_status(_process_status);
         _set_scanner_done();
         return true;
     }

From bd8113f4243ed607121a2e76ee69eabd738e32b0 Mon Sep 17 00:00:00 2001
From: yiguolei <676222867@qq.com>
Date: Wed, 3 Jan 2024 20:47:35 +0800
Subject: [PATCH 295/299] [bugfix](scannerscheduler) should minus
 num_of_scanners before check should schedule #28926 (#29331)

---------

Co-authored-by: yiguolei <yiguolei@gmail.com>
---
 be/src/common/config.cpp                   |   3 +
 be/src/common/config.h                     |   2 +
 be/src/common/status.h                     |   8 -
 be/src/exec/exec_node.h                    |   4 +
 be/src/pipeline/exec/scan_operator.cpp     |  64 ++---
 be/src/pipeline/exec/scan_operator.h       |  16 +-
 be/src/pipeline/pipeline_task.h            |  29 ++-
 be/src/pipeline/task_scheduler.cpp         |   2 +
 be/src/runtime/fragment_mgr.cpp            |   2 +-
 be/src/service/backend_service.cpp         |   2 +-
 be/src/vec/exec/scan/pip_scanner_context.h |  21 +-
 be/src/vec/exec/scan/scanner_context.cpp   | 280 ++++++++-------------
 be/src/vec/exec/scan/scanner_context.h     |  48 ++--
 be/src/vec/exec/scan/scanner_scheduler.cpp |  61 +++--
 be/src/vec/exec/scan/scanner_scheduler.h   |   4 +-
 be/src/vec/exec/scan/vscan_node.cpp        |  36 ++-
 be/src/vec/exec/scan/vscan_node.h          |  28 ++-
 be/src/vec/exec/scan/vscanner.h            |  13 -
 regression-test/pipeline/p0/conf/be.conf   |   2 +
 19 files changed, 272 insertions(+), 353 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index f3e8df44e17846..040fc5ae21705b 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1158,6 +1158,9 @@ DEFINE_mBool(enable_column_type_check, "true");
 // 128 MB
 DEFINE_mInt64(local_exchange_buffer_mem_limit, "134217728");
 
+// Default 300s, if its value <= 0, then log is disabled
+DEFINE_mInt64(enable_debug_log_timeout_secs, "0");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index cfb8ed02f328b3..9a287a0c246717 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1232,6 +1232,8 @@ DECLARE_mInt32(variant_max_merged_tablet_schema_size);
 
 DECLARE_mInt64(local_exchange_buffer_mem_limit);
 
+DECLARE_mInt64(enable_debug_log_timeout_secs);
+
 DECLARE_mBool(enable_column_type_check);
 
 #ifdef BE_TEST
diff --git a/be/src/common/status.h b/be/src/common/status.h
index c0d92152877574..d7f4481c1dc3aa 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -434,14 +434,6 @@ class [[nodiscard]] Status {
 
     bool ok() const { return _code == ErrorCode::OK; }
 
-    // Convert into TStatus. Call this if 'status_container' contains an optional
-    // TStatus field named 'status'. This also sets __isset.status.
-    template <typename T>
-    void set_t_status(T* status_container) const {
-        to_thrift(&status_container->status);
-        status_container->__isset.status = true;
-    }
-
     // Convert into TStatus.
     void to_thrift(TStatus* status) const;
     TStatus to_thrift() const;
diff --git a/be/src/exec/exec_node.h b/be/src/exec/exec_node.h
index 69f7771767fb1b..aaa2f6ee07f20d 100644
--- a/be/src/exec/exec_node.h
+++ b/be/src/exec/exec_node.h
@@ -239,6 +239,10 @@ class ExecNode {
 
     size_t children_count() const { return _children.size(); }
 
+    // when the fragment is normal finished, call this method to do some finish work
+    // such as send the last buffer to remote.
+    virtual Status try_close(RuntimeState* state) { return Status::OK(); }
+
 protected:
     friend class DataSink;
 
diff --git a/be/src/pipeline/exec/scan_operator.cpp b/be/src/pipeline/exec/scan_operator.cpp
index de3214b0465dc7..37be7a01818dac 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -53,6 +53,7 @@ bool ScanOperator::can_read() {
             return false;
         }
     } else {
+        // If scanner meet any error, done == true
         if (_node->_eos || _node->_scanner_ctx->done()) {
             // _eos: need eos
             // _scanner_ctx->done(): need finish
@@ -64,14 +65,6 @@ bool ScanOperator::can_read() {
     }
 }
 
-bool ScanOperator::is_pending_finish() const {
-    return _node->_scanner_ctx && !_node->_scanner_ctx->no_schedule();
-}
-
-Status ScanOperator::try_close(RuntimeState* state) {
-    return _node->try_close(state);
-}
-
 bool ScanOperator::runtime_filters_are_ready_or_timeout() {
     return _node->runtime_filters_are_ready_or_timeout();
 }
@@ -81,9 +74,8 @@ std::string ScanOperator::debug_string() const {
     fmt::format_to(debug_string_buffer, "{}, scanner_ctx is null: {} ",
                    SourceOperator::debug_string(), _node->_scanner_ctx == nullptr);
     if (_node->_scanner_ctx) {
-        fmt::format_to(debug_string_buffer, ", num_running_scanners = {}, num_scheduling_ctx = {} ",
-                       _node->_scanner_ctx->get_num_running_scanners(),
-                       _node->_scanner_ctx->get_num_scheduling_ctx());
+        fmt::format_to(debug_string_buffer, ", num_running_scanners = {}",
+                       _node->_scanner_ctx->debug_string());
     }
     return fmt::to_string(debug_string_buffer);
 }
@@ -101,9 +93,6 @@ std::string ScanOperator::debug_string() const {
 template <typename Derived>
 ScanLocalState<Derived>::ScanLocalState(RuntimeState* state, OperatorXBase* parent)
         : ScanLocalStateBase(state, parent) {
-    _finish_dependency = std::make_shared<FinishDependency>(
-            parent->operator_id(), parent->node_id(), parent->get_name() + "_FINISH_DEPENDENCY",
-            state->get_query_ctx());
     _filter_dependency = std::make_shared<RuntimeFilterDependency>(
             parent->operator_id(), parent->node_id(), parent->get_name() + "_FILTER_DEPENDENCY",
             state->get_query_ctx());
@@ -175,7 +164,6 @@ Status ScanLocalState<Derived>::open(RuntimeState* state) {
 
     auto status = _eos ? Status::OK() : _prepare_scanners();
     if (_scanner_ctx) {
-        _finish_dependency->block();
         DCHECK(!_eos && _num_scanners->value() > 0);
         RETURN_IF_ERROR(_scanner_ctx->init());
         RETURN_IF_ERROR(state->exec_env()->scanner_scheduler()->submit(_scanner_ctx));
@@ -569,15 +557,14 @@ std::string ScanLocalState<Derived>::debug_string(int indentation_level) const {
                    _eos.load());
     if (_scanner_ctx) {
         fmt::format_to(debug_string_buffer, "");
-        fmt::format_to(
-                debug_string_buffer,
-                ", Scanner Context: (_is_finished = {}, _should_stop = {}, "
-                "_num_running_scanners={}, "
-                "_num_scheduling_ctx = {}, _num_unfinished_scanners = {}, status = {}, error = {})",
-                _scanner_ctx->is_finished(), _scanner_ctx->should_stop(),
-                _scanner_ctx->get_num_running_scanners(), _scanner_ctx->get_num_scheduling_ctx(),
-                _scanner_ctx->get_num_unfinished_scanners(), _scanner_ctx->status().to_string(),
-                _scanner_ctx->status_error());
+        fmt::format_to(debug_string_buffer,
+                       ", Scanner Context: (_is_finished = {}, _should_stop = {}, "
+                       "_num_running_scanners={}, "
+                       " _num_unfinished_scanners = {}, status = {})",
+                       _scanner_ctx->is_finished(), _scanner_ctx->should_stop(),
+                       _scanner_ctx->get_num_running_scanners(),
+                       _scanner_ctx->get_num_unfinished_scanners(),
+                       _scanner_ctx->status().to_string());
     }
 
     return fmt::to_string(debug_string_buffer);
@@ -1225,23 +1212,27 @@ template <typename Derived>
 Status ScanLocalState<Derived>::_prepare_scanners() {
     std::list<vectorized::VScannerSPtr> scanners;
     RETURN_IF_ERROR(_init_scanners(&scanners));
+    // Init scanner wrapper
+    for (auto it = scanners.begin(); it != scanners.end(); ++it) {
+        _scanners.emplace_back(std::make_shared<vectorized::ScannerDelegate>(*it));
+    }
     if (scanners.empty()) {
         _eos = true;
         _scan_dependency->set_ready();
     } else {
         COUNTER_SET(_num_scanners, static_cast<int64_t>(scanners.size()));
-        RETURN_IF_ERROR(_start_scanners(scanners));
+        RETURN_IF_ERROR(_start_scanners(_scanners));
     }
     return Status::OK();
 }
 
 template <typename Derived>
 Status ScanLocalState<Derived>::_start_scanners(
-        const std::list<vectorized::VScannerSPtr>& scanners) {
+        const std::list<std::shared_ptr<vectorized::ScannerDelegate>>& scanners) {
     auto& p = _parent->cast<typename Derived::Parent>();
     _scanner_ctx = PipXScannerContext::create_shared(
             state(), this, p._output_tuple_desc, p.output_row_descriptor(), scanners, p.limit(),
-            state()->scan_queue_mem_limit(), _scan_dependency, _finish_dependency);
+            state()->scan_queue_mem_limit(), _scan_dependency);
     return Status::OK();
 }
 
@@ -1317,9 +1308,6 @@ Status ScanLocalState<Derived>::_init_profile() {
 
     _max_scanner_thread_num = ADD_COUNTER(_runtime_profile, "MaxScannerThreadNum", TUnit::UNIT);
 
-    _wait_for_finish_dependency_timer =
-            ADD_TIMER(_runtime_profile, "WaitForPendingFinishDependency");
-
     return Status::OK();
 }
 
@@ -1427,17 +1415,6 @@ Status ScanOperatorX<LocalStateType>::open(RuntimeState* state) {
     return Status::OK();
 }
 
-template <typename LocalStateType>
-Status ScanOperatorX<LocalStateType>::try_close(RuntimeState* state) {
-    auto& local_state = get_local_state(state);
-    if (local_state._scanner_ctx) {
-        // mark this scanner ctx as should_stop to make sure scanners will not be scheduled anymore
-        // TODO: there is a lock in `set_should_stop` may cause some slight impact
-        local_state._scanner_ctx->set_should_stop();
-    }
-    return Status::OK();
-}
-
 template <typename Derived>
 Status ScanLocalState<Derived>::close(RuntimeState* state) {
     if (_closed) {
@@ -1449,10 +1426,9 @@ Status ScanLocalState<Derived>::close(RuntimeState* state) {
 
     SCOPED_TIMER(exec_time_counter());
     if (_scanner_ctx) {
-        _scanner_ctx->clear_and_join(reinterpret_cast<ScanLocalStateBase*>(this), state);
+        _scanner_ctx->stop_scanners(state);
     }
     COUNTER_SET(_wait_for_dependency_timer, _scan_dependency->watcher_elapse_time());
-    COUNTER_SET(_wait_for_finish_dependency_timer, _finish_dependency->watcher_elapse_time());
     COUNTER_SET(_wait_for_rf_timer, _filter_dependency->watcher_elapse_time());
 
     return PipelineXLocalState<ScanDependency>::close(state);
@@ -1509,7 +1485,7 @@ Status ScanOperatorX<LocalStateType>::get_block(RuntimeState* state, vectorized:
     if (eos) {
         source_state = SourceState::FINISHED;
         // reach limit, stop the scanners.
-        local_state._scanner_ctx->set_should_stop();
+        local_state._scanner_ctx->stop_scanners(state);
     }
 
     return Status::OK();
diff --git a/be/src/pipeline/exec/scan_operator.h b/be/src/pipeline/exec/scan_operator.h
index 3ad1ac1536a89d..fa7e44a8fb7895 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -32,6 +32,9 @@
 namespace doris {
 class ExecNode;
 } // namespace doris
+namespace doris::vectorized {
+class ScannerDelegate;
+}
 
 namespace doris::pipeline {
 class PipScannerContext;
@@ -49,13 +52,9 @@ class ScanOperator : public SourceOperator<vectorized::VScanNode> {
 
     bool can_read() override; // for source
 
-    bool is_pending_finish() const override;
-
     bool runtime_filters_are_ready_or_timeout() override;
 
     std::string debug_string() const override;
-
-    Status try_close(RuntimeState* state) override;
 };
 
 class ScanDependency final : public Dependency {
@@ -174,7 +173,6 @@ class ScanLocalStateBase : public PipelineXLocalState<ScanDependency>,
     RuntimeProfile::Counter* _wait_for_scanner_done_timer = nullptr;
     // time of prefilter input block from scanner
     RuntimeProfile::Counter* _wait_for_eos_timer = nullptr;
-    RuntimeProfile::Counter* _wait_for_finish_dependency_timer = nullptr;
     RuntimeProfile::Counter* _wait_for_rf_timer = nullptr;
 };
 
@@ -215,7 +213,6 @@ class ScanLocalState : public ScanLocalStateBase {
     int64_t get_push_down_count() override;
 
     RuntimeFilterDependency* filterdependency() override { return _filter_dependency.get(); };
-    Dependency* finishdependency() override { return _finish_dependency.get(); }
 
 protected:
     template <typename LocalStateType>
@@ -351,7 +348,7 @@ class ScanLocalState : public ScanLocalStateBase {
     Status _prepare_scanners();
 
     // Submit the scanner to the thread pool and start execution
-    Status _start_scanners(const std::list<vectorized::VScannerSPtr>& scanners);
+    Status _start_scanners(const std::list<std::shared_ptr<vectorized::ScannerDelegate>>& scanners);
 
     // For some conjunct there is chance to elimate cast operator
     // Eg. Variant's sub column could eliminate cast in storage layer if
@@ -414,14 +411,13 @@ class ScanLocalState : public ScanLocalStateBase {
 
     std::shared_ptr<RuntimeFilterDependency> _filter_dependency;
 
-    std::shared_ptr<Dependency> _finish_dependency;
+    // ScanLocalState owns the ownership of scanner, scanner context only has its weakptr
+    std::list<std::shared_ptr<vectorized::ScannerDelegate>> _scanners;
 };
 
 template <typename LocalStateType>
 class ScanOperatorX : public OperatorX<LocalStateType> {
 public:
-    Status try_close(RuntimeState* state) override;
-
     Status init(const TPlanNode& tnode, RuntimeState* state) override;
     Status prepare(RuntimeState* state) override { return OperatorXBase::prepare(state); }
     Status open(RuntimeState* state) override;
diff --git a/be/src/pipeline/pipeline_task.h b/be/src/pipeline/pipeline_task.h
index 700741f3f548e5..30022792e8ce76 100644
--- a/be/src/pipeline/pipeline_task.h
+++ b/be/src/pipeline/pipeline_task.h
@@ -23,6 +23,7 @@
 #include <string>
 #include <vector>
 
+#include "common/config.h"
 #include "common/status.h"
 #include "exec/operator.h"
 #include "pipeline.h"
@@ -264,6 +265,32 @@ class PipelineTask {
     bool is_running() { return _running.load(); }
     void set_running(bool running) { _running = running; }
 
+    bool is_exceed_debug_timeout() {
+        if (_has_exceed_timeout) {
+            return true;
+        }
+        // If enable_debug_log_timeout_secs <= 0, then disable the log
+        if (_pipeline_task_watcher.elapsed_time() >
+            config::enable_debug_log_timeout_secs * 1000l * 1000l * 1000l) {
+            _has_exceed_timeout = true;
+            return true;
+        }
+        return false;
+    }
+
+    void log_detail_if_need() {
+        if (config::enable_debug_log_timeout_secs < 1) {
+            return;
+        }
+        if (is_exceed_debug_timeout()) {
+            LOG(INFO) << "query id|instanceid " << print_id(_state->query_id()) << "|"
+                      << print_id(_state->fragment_instance_id())
+                      << " current pipeline exceed run time "
+                      << config::enable_debug_log_timeout_secs << " seconds. Task state "
+                      << get_state_name(get_state()) << debug_string();
+        }
+    }
+
 protected:
     void _finish_p_dependency() {
         for (const auto& p : _pipeline->_parents) {
@@ -278,7 +305,7 @@ class PipelineTask {
     uint32_t _index;
     PipelinePtr _pipeline;
     bool _dependency_finish = false;
-
+    bool _has_exceed_timeout = false;
     bool _prepared;
     bool _opened;
     RuntimeState* _state = nullptr;
diff --git a/be/src/pipeline/task_scheduler.cpp b/be/src/pipeline/task_scheduler.cpp
index 91980aeec7a8d8..d253c6a589ae6b 100644
--- a/be/src/pipeline/task_scheduler.cpp
+++ b/be/src/pipeline/task_scheduler.cpp
@@ -119,6 +119,7 @@ void BlockedTaskScheduler::_schedule() {
         while (iter != local_blocked_tasks.end()) {
             auto* task = *iter;
             auto state = task->get_state();
+            task->log_detail_if_need();
             if (state == PipelineTaskState::PENDING_FINISH) {
                 // should cancel or should finish
                 if (task->is_pending_finish()) {
@@ -236,6 +237,7 @@ void TaskScheduler::_do_work(size_t index) {
             static_cast<void>(_task_queue->push_back(task, index));
             continue;
         }
+        task->log_detail_if_need();
         task->set_running(true);
         task->set_task_queue(_task_queue.get());
         auto* fragment_ctx = task->fragment_context();
diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp
index 6d8b99c62fd579..0d32c16186a60e 100644
--- a/be/src/runtime/fragment_mgr.cpp
+++ b/be/src/runtime/fragment_mgr.cpp
@@ -213,7 +213,7 @@ void FragmentMgr::coordinator_callback(const ReportStatusRequest& req) {
     params.__set_backend_num(req.backend_num);
     params.__set_fragment_instance_id(req.fragment_instance_id);
     params.__set_fragment_id(req.fragment_id);
-    exec_status.set_t_status(&params);
+    params.__set_status(exec_status.to_thrift());
     params.__set_done(req.done);
     params.__set_query_type(req.runtime_state->query_type());
     params.__set_finished_scan_ranges(req.runtime_state->num_finished_range());
diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp
index 432f38697c0544..fb98d60dce4493 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -405,7 +405,7 @@ void BackendService::exec_plan_fragment(TExecPlanFragmentResult& return_val,
                                         const TExecPlanFragmentParams& params) {
     LOG(INFO) << "exec_plan_fragment() instance_id=" << print_id(params.params.fragment_instance_id)
               << " coord=" << params.coord << " backend#=" << params.backend_num;
-    start_plan_fragment_execution(params).set_t_status(&return_val);
+    return_val.__set_status(start_plan_fragment_execution(params).to_thrift());
 }
 
 Status BackendService::start_plan_fragment_execution(const TExecPlanFragmentParams& exec_params) {
diff --git a/be/src/vec/exec/scan/pip_scanner_context.h b/be/src/vec/exec/scan/pip_scanner_context.h
index 9a717ec08b26d8..71db343840be48 100644
--- a/be/src/vec/exec/scan/pip_scanner_context.h
+++ b/be/src/vec/exec/scan/pip_scanner_context.h
@@ -30,9 +30,9 @@ class PipScannerContext final : public vectorized::ScannerContext {
     PipScannerContext(RuntimeState* state, vectorized::VScanNode* parent,
                       const TupleDescriptor* output_tuple_desc,
                       const RowDescriptor* output_row_descriptor,
-                      const std::list<vectorized::VScannerSPtr>& scanners, int64_t limit_,
-                      int64_t max_bytes_in_blocks_queue, const std::vector<int>& col_distribute_ids,
-                      const int num_parallel_instances)
+                      const std::list<std::shared_ptr<vectorized::ScannerDelegate>>& scanners,
+                      int64_t limit_, int64_t max_bytes_in_blocks_queue,
+                      const std::vector<int>& col_distribute_ids, const int num_parallel_instances)
             : vectorized::ScannerContext(state, parent, output_tuple_desc, output_row_descriptor,
                                          scanners, limit_, max_bytes_in_blocks_queue,
                                          num_parallel_instances),
@@ -55,6 +55,9 @@ class PipScannerContext final : public vectorized::ScannerContext {
         std::vector<vectorized::BlockUPtr> merge_blocks;
         {
             std::unique_lock<std::mutex> l(*_queue_mutexs[id]);
+            // The pipeline maybe wake up by scanner.done. If there are still any data
+            // in the queue, should read the data first and then check if the scanner.done
+            // if done, then eos is returned to indicate that the scan operator finished.
             if (_blocks_queues[id].empty()) {
                 *eos = done();
                 return Status::OK();
@@ -97,9 +100,6 @@ class PipScannerContext final : public vectorized::ScannerContext {
         return Status::OK();
     }
 
-    // We should make those method lock free.
-    bool done() override { return _is_finished || _should_stop; }
-
     void append_blocks_to_queue(std::vector<vectorized::BlockUPtr>& blocks) override {
         const int queue_size = _blocks_queues.size();
         const int block_size = blocks.size();
@@ -277,13 +277,12 @@ class PipXScannerContext final : public vectorized::ScannerContext {
     PipXScannerContext(RuntimeState* state, ScanLocalStateBase* local_state,
                        const TupleDescriptor* output_tuple_desc,
                        const RowDescriptor* output_row_descriptor,
-                       const std::list<vectorized::VScannerSPtr>& scanners, int64_t limit_,
-                       int64_t max_bytes_in_blocks_queue,
-                       std::shared_ptr<pipeline::ScanDependency> dependency,
-                       std::shared_ptr<pipeline::Dependency> finish_dependency)
+                       const std::list<std::shared_ptr<vectorized::ScannerDelegate>>& scanners,
+                       int64_t limit_, int64_t max_bytes_in_blocks_queue,
+                       std::shared_ptr<pipeline::ScanDependency> dependency)
             : vectorized::ScannerContext(state, output_tuple_desc, output_row_descriptor, scanners,
                                          limit_, max_bytes_in_blocks_queue, 1, local_state,
-                                         dependency, finish_dependency) {}
+                                         dependency) {}
     Status get_block_from_queue(RuntimeState* state, vectorized::BlockUPtr* block, bool* eos,
                                 int id, bool wait = false) override {
         std::unique_lock l(_transfer_lock);
diff --git a/be/src/vec/exec/scan/scanner_context.cpp b/be/src/vec/exec/scan/scanner_context.cpp
index 089f52cf61b6a7..c1d064c601d301 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -47,11 +47,11 @@ using namespace std::chrono_literals;
 
 ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* output_tuple_desc,
                                const RowDescriptor* output_row_descriptor,
-                               const std::list<VScannerSPtr>& scanners, int64_t limit_,
-                               int64_t max_bytes_in_blocks_queue, const int num_parallel_instances,
+                               const std::list<std::shared_ptr<ScannerDelegate>>& scanners,
+                               int64_t limit_, int64_t max_bytes_in_blocks_queue,
+                               const int num_parallel_instances,
                                pipeline::ScanLocalStateBase* local_state,
-                               std::shared_ptr<pipeline::ScanDependency> dependency,
-                               std::shared_ptr<pipeline::Dependency> finish_dependency)
+                               std::shared_ptr<pipeline::ScanDependency> dependency)
         : HasTaskExecutionCtx(state),
           _state(state),
           _parent(nullptr),
@@ -66,11 +66,10 @@ ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* outpu
           _max_bytes_in_queue(std::max(max_bytes_in_blocks_queue, (int64_t)1024) *
                               num_parallel_instances),
           _scanner_scheduler(state->exec_env()->scanner_scheduler()),
-          _scanners(scanners),
-          _scanners_ref(scanners.begin(), scanners.end()),
+          _scanners(scanners.begin(), scanners.end()),
+          _all_scanners(scanners.begin(), scanners.end()),
           _num_parallel_instances(num_parallel_instances),
-          _dependency(dependency),
-          _finish_dependency(finish_dependency) {
+          _dependency(dependency) {
     DCHECK(_output_row_descriptor == nullptr ||
            _output_row_descriptor->tuple_descriptors().size() == 1);
     _query_id = _state->get_query_ctx()->query_id();
@@ -98,8 +97,9 @@ ScannerContext::ScannerContext(RuntimeState* state, const TupleDescriptor* outpu
 ScannerContext::ScannerContext(doris::RuntimeState* state, doris::vectorized::VScanNode* parent,
                                const doris::TupleDescriptor* output_tuple_desc,
                                const RowDescriptor* output_row_descriptor,
-                               const std::list<VScannerSPtr>& scanners, int64_t limit_,
-                               int64_t max_bytes_in_blocks_queue, const int num_parallel_instances,
+                               const std::list<std::shared_ptr<ScannerDelegate>>& scanners,
+                               int64_t limit_, int64_t max_bytes_in_blocks_queue,
+                               const int num_parallel_instances,
                                pipeline::ScanLocalStateBase* local_state)
         : HasTaskExecutionCtx(state),
           _state(state),
@@ -115,8 +115,8 @@ ScannerContext::ScannerContext(doris::RuntimeState* state, doris::vectorized::VS
           _max_bytes_in_queue(std::max(max_bytes_in_blocks_queue, (int64_t)1024) *
                               num_parallel_instances),
           _scanner_scheduler(state->exec_env()->scanner_scheduler()),
-          _scanners(scanners),
-          _scanners_ref(scanners.begin(), scanners.end()),
+          _scanners(scanners.begin(), scanners.end()),
+          _all_scanners(scanners.begin(), scanners.end()),
           _num_parallel_instances(num_parallel_instances) {
     DCHECK(_output_row_descriptor == nullptr ||
            _output_row_descriptor->tuple_descriptors().size() == 1);
@@ -192,10 +192,6 @@ Status ScannerContext::init() {
     }
 #endif
 
-    // 4. This ctx will be submitted to the scanner scheduler right after init.
-    // So set _num_scheduling_ctx to 1 here.
-    _num_scheduling_ctx = 1;
-
     _num_unfinished_scanners = _scanners.size();
 
     if (_parent) {
@@ -288,11 +284,9 @@ Status ScannerContext::get_block_from_queue(RuntimeState* state, vectorized::Blo
         bool is_scheduled = false;
         if (!done() && to_be_schedule && _num_running_scanners == 0) {
             is_scheduled = true;
-            auto state = _scanner_scheduler->submit(shared_from_this());
-            if (state.ok()) {
-                _num_scheduling_ctx++;
-            } else {
-                set_status_on_error(state, false);
+            auto submit_status = _scanner_scheduler->submit(shared_from_this());
+            if (!submit_status.ok()) {
+                set_status_on_error(submit_status, false);
             }
         }
 
@@ -384,60 +378,36 @@ Status ScannerContext::validate_block_schema(Block* block) {
     return Status::OK();
 }
 
-void ScannerContext::set_should_stop() {
-    std::lock_guard l(_transfer_lock);
-    _should_stop = true;
-    _set_scanner_done();
-    for (const VScannerWPtr& scanner : _scanners_ref) {
-        if (VScannerSPtr sc = scanner.lock()) {
-            sc->try_stop();
-        }
-    }
-    _blocks_queue_added_cv.notify_one();
-    set_ready_to_finish();
-}
-
 void ScannerContext::inc_num_running_scanners(int32_t inc) {
     std::lock_guard l(_transfer_lock);
     _num_running_scanners += inc;
 }
 
-void ScannerContext::dec_num_scheduling_ctx() {
-    std::lock_guard l(_transfer_lock);
-    _num_scheduling_ctx--;
-    set_ready_to_finish();
-    if (_num_running_scanners == 0 && _num_scheduling_ctx == 0) {
-        _ctx_finish_cv.notify_one();
-    }
-}
-
-void ScannerContext::set_ready_to_finish() {
-    // `_should_stop == true` means this task has already ended and wait for pending finish now.
-    if (_finish_dependency && done() && _num_running_scanners == 0 && _num_scheduling_ctx == 0) {
-        _finish_dependency->set_ready();
-    }
-}
-
-bool ScannerContext::set_status_on_error(const Status& status, bool need_lock) {
+void ScannerContext::set_status_on_error(const Status& status, bool need_lock) {
     std::unique_lock l(_transfer_lock, std::defer_lock);
     if (need_lock) {
         l.lock();
     }
     if (this->status().ok()) {
         _process_status = status;
-        _status_error = true;
         _blocks_queue_added_cv.notify_one();
         _should_stop = true;
         _state->get_query_ctx()->set_exec_status(_process_status);
         _set_scanner_done();
-        return true;
     }
-    return false;
 }
 
-template <typename Parent>
-Status ScannerContext::_close_and_clear_scanners(Parent* parent, RuntimeState* state) {
-    std::unique_lock l(_scanners_lock);
+void ScannerContext::stop_scanners(RuntimeState* state) {
+    std::unique_lock l(_transfer_lock);
+    _should_stop = true;
+    _set_scanner_done();
+    for (const std::weak_ptr<ScannerDelegate>& scanner : _all_scanners) {
+        if (std::shared_ptr<ScannerDelegate> sc = scanner.lock()) {
+            sc->_scanner->try_stop();
+        }
+    }
+    _blocks_queue.clear();
+    // TODO yiguolei, call mark close to scanners
     if (state->enable_profile()) {
         std::stringstream scanner_statistics;
         std::stringstream scanner_rows_read;
@@ -445,76 +415,38 @@ Status ScannerContext::_close_and_clear_scanners(Parent* parent, RuntimeState* s
         scanner_statistics << "[";
         scanner_rows_read << "[";
         scanner_wait_worker_time << "[";
-        for (auto finished_scanner_time : _finished_scanner_runtime) {
-            scanner_statistics << PrettyPrinter::print(finished_scanner_time, TUnit::TIME_NS)
-                               << ", ";
-        }
-        for (auto finished_scanner_rows : _finished_scanner_rows_read) {
-            scanner_rows_read << PrettyPrinter::print(finished_scanner_rows, TUnit::UNIT) << ", ";
-        }
-        for (auto finished_scanner_wait_time : _finished_scanner_wait_worker_time) {
-            scanner_wait_worker_time
-                    << PrettyPrinter::print(finished_scanner_wait_time, TUnit::TIME_NS) << ", ";
-        }
-        // Only unfinished scanners here
-        for (auto& scanner : _scanners) {
-            // Scanners are in ObjPool in ScanNode,
-            // so no need to delete them here.
+        // Scanners can in 3 state
+        //  state 1: in scanner context, not scheduled
+        //  state 2: in scanner worker pool's queue, scheduled but not running
+        //  state 3: scanner is running.
+        for (auto& scanner_ref : _all_scanners) {
+            auto scanner = scanner_ref.lock();
+            if (scanner == nullptr) {
+                continue;
+            }
             // Add per scanner running time before close them
-            scanner_statistics << PrettyPrinter::print(scanner->get_time_cost_ns(), TUnit::TIME_NS)
+            scanner_statistics << PrettyPrinter::print(scanner->_scanner->get_time_cost_ns(),
+                                                       TUnit::TIME_NS)
                                << ", ";
-            scanner_rows_read << PrettyPrinter::print(scanner->get_rows_read(), TUnit::UNIT)
+            scanner_rows_read << PrettyPrinter::print(scanner->_scanner->get_rows_read(),
+                                                      TUnit::UNIT)
                               << ", ";
             scanner_wait_worker_time
-                    << PrettyPrinter::print(scanner->get_scanner_wait_worker_timer(),
+                    << PrettyPrinter::print(scanner->_scanner->get_scanner_wait_worker_timer(),
                                             TUnit::TIME_NS)
                     << ", ";
+            // since there are all scanners, some scanners is running, so that could not call scanner
+            // close here.
         }
         scanner_statistics << "]";
         scanner_rows_read << "]";
         scanner_wait_worker_time << "]";
-        parent->scanner_profile()->add_info_string("PerScannerRunningTime",
-                                                   scanner_statistics.str());
-        parent->scanner_profile()->add_info_string("PerScannerRowsRead", scanner_rows_read.str());
-        parent->scanner_profile()->add_info_string("PerScannerWaitTime",
-                                                   scanner_wait_worker_time.str());
-    }
-    // Only unfinished scanners here
-    for (auto& scanner : _scanners) {
-        static_cast<void>(scanner->close(state));
-        // Scanners are in ObjPool in ScanNode,
-        // so no need to delete them here.
+        _scanner_profile->add_info_string("PerScannerRunningTime", scanner_statistics.str());
+        _scanner_profile->add_info_string("PerScannerRowsRead", scanner_rows_read.str());
+        _scanner_profile->add_info_string("PerScannerWaitTime", scanner_wait_worker_time.str());
     }
-    _scanners.clear();
-    return Status::OK();
-}
 
-template <typename Parent>
-void ScannerContext::clear_and_join(Parent* parent, RuntimeState* state) {
-    std::unique_lock l(_transfer_lock);
-    do {
-        if (_num_running_scanners == 0 && _num_scheduling_ctx == 0) {
-            break;
-        } else {
-            DCHECK(!state->enable_pipeline_exec())
-                    << " _num_running_scanners: " << _num_running_scanners
-                    << " _num_scheduling_ctx: " << _num_scheduling_ctx;
-            while (!(_num_running_scanners == 0 && _num_scheduling_ctx == 0)) {
-                _ctx_finish_cv.wait(l);
-            }
-            break;
-        }
-    } while (false);
-    // Must wait all running scanners stop running.
-    // So that we can make sure to close all scanners.
-    static_cast<void>(_close_and_clear_scanners(parent, state));
-
-    _blocks_queue.clear();
-}
-
-bool ScannerContext::no_schedule() {
-    std::unique_lock l(_transfer_lock);
-    return _num_running_scanners == 0 && _num_scheduling_ctx == 0;
+    _blocks_queue_added_cv.notify_one();
 }
 
 void ScannerContext::_set_scanner_done() {
@@ -527,12 +459,13 @@ std::string ScannerContext::debug_string() {
     return fmt::format(
             "id: {}, sacnners: {}, blocks in queue: {},"
             " status: {}, _should_stop: {}, _is_finished: {}, free blocks: {},"
-            " limit: {}, _num_running_scanners: {}, _num_scheduling_ctx: {}, _max_thread_num: {},"
-            " _block_per_scanner: {}, _cur_bytes_in_queue: {}, MAX_BYTE_OF_QUEUE: {}",
+            " limit: {}, _num_running_scanners: {}, _max_thread_num: {},"
+            " _block_per_scanner: {}, _cur_bytes_in_queue: {}, MAX_BYTE_OF_QUEUE: {}, "
+            "num_ctx_scheduled: {}, query_id: {}",
             ctx_id, _scanners.size(), _blocks_queue.size(), status().ok(), _should_stop,
-            _is_finished, _free_blocks.size_approx(), limit, _num_running_scanners,
-            _num_scheduling_ctx, _max_thread_num, _block_per_scanner, _cur_bytes_in_queue,
-            _max_bytes_in_queue);
+            _is_finished, _free_blocks.size_approx(), limit, _num_running_scanners, _max_thread_num,
+            _block_per_scanner, _cur_bytes_in_queue, _max_bytes_in_queue, num_ctx_scheduled(),
+            print_id(_query_id));
 }
 
 void ScannerContext::reschedule_scanner_ctx() {
@@ -540,84 +473,71 @@ void ScannerContext::reschedule_scanner_ctx() {
     if (done()) {
         return;
     }
-    auto state = _scanner_scheduler->submit(shared_from_this());
+    auto submit_status = _scanner_scheduler->submit(shared_from_this());
     //todo(wb) rethinking is it better to mark current scan_context failed when submit failed many times?
-    if (state.ok()) {
-        _num_scheduling_ctx++;
-    } else {
-        set_status_on_error(state, false);
+    if (!submit_status.ok()) {
+        set_status_on_error(submit_status, false);
     }
 }
 
-void ScannerContext::push_back_scanner_and_reschedule(VScannerSPtr scanner) {
-    {
-        std::unique_lock l(_scanners_lock);
-        _scanners.push_front(scanner);
-    }
+void ScannerContext::push_back_scanner_and_reschedule(std::shared_ptr<ScannerDelegate> scanner) {
     std::lock_guard l(_transfer_lock);
-
-    // In pipeline engine, doris will close scanners when `no_schedule`.
-    // We have to decrease _num_running_scanners before schedule, otherwise
-    // schedule does not woring due to _num_running_scanners.
-    _num_running_scanners--;
-    set_ready_to_finish();
-
-    if (!done() && should_be_scheduled()) {
-        auto state = _scanner_scheduler->submit(shared_from_this());
-        if (state.ok()) {
-            _num_scheduling_ctx++;
+    // Use a transfer lock to avoid the scanner be scheduled concurrently. For example, that after
+    // calling "_scanners.push_front(scanner)", there may be other ctx in scheduler
+    // to schedule that scanner right away, and in that schedule run, the scanner may be marked as closed
+    // before we call the following if() block.
+    //LOG(INFO) << "yyyy one scanner finished " << debug_string();
+    {
+        --_num_running_scanners;
+        if (scanner->_scanner->need_to_close()) {
+            --_num_unfinished_scanners;
+            if (_num_unfinished_scanners == 0) {
+                _dispose_coloate_blocks_not_in_queue();
+                _is_finished = true;
+                _set_scanner_done();
+                _blocks_queue_added_cv.notify_one();
+                return;
+            }
         } else {
-            set_status_on_error(state, false);
+            _scanners.push_front(scanner);
         }
     }
 
-    // Notice that after calling "_scanners.push_front(scanner)", there may be other ctx in scheduler
-    // to schedule that scanner right away, and in that schedule run, the scanner may be marked as closed
-    // before we call the following if() block.
-    // So we need "scanner->set_counted_down()" to avoid "_num_unfinished_scanners" being decreased twice by
-    // same scanner.
-    if (scanner->need_to_close() && scanner->set_counted_down() &&
-        (--_num_unfinished_scanners) == 0) {
-        _dispose_coloate_blocks_not_in_queue();
-        _is_finished = true;
-        _set_scanner_done();
-        _blocks_queue_added_cv.notify_one();
+    if (should_be_scheduled()) {
+        auto submit_status = _scanner_scheduler->submit(shared_from_this());
+        if (!submit_status.ok()) {
+            set_status_on_error(submit_status, false);
+        }
     }
-    _ctx_finish_cv.notify_one();
 }
 
-void ScannerContext::get_next_batch_of_scanners(std::list<VScannerSPtr>* current_run) {
+// This method is called in scanner scheduler, and task context is hold
+void ScannerContext::get_next_batch_of_scanners(
+        std::list<std::weak_ptr<ScannerDelegate>>* current_run) {
+    std::lock_guard l(_transfer_lock);
+    // Update the sched counter for profile
+    Defer defer {[&]() { _scanner_sched_counter->update(current_run->size()); }};
     // 1. Calculate how many scanners should be scheduled at this run.
-    int thread_slot_num = 0;
-    {
-        // If there are enough space in blocks queue,
-        // the scanner number depends on the _free_blocks numbers
-        thread_slot_num = get_available_thread_slot_num();
-    }
+    // If there are enough space in blocks queue,
+    // the scanner number depends on the _free_blocks numbers
+    int thread_slot_num = get_available_thread_slot_num();
 
     // 2. get #thread_slot_num scanners from ctx->scanners
     // and put them into "this_run".
-    {
-        std::unique_lock l(_scanners_lock);
-        for (int i = 0; i < thread_slot_num && !_scanners.empty();) {
-            VScannerSPtr scanner = _scanners.front();
-            _scanners.pop_front();
-            if (scanner->need_to_close()) {
-                _finished_scanner_runtime.push_back(scanner->get_time_cost_ns());
-                _finished_scanner_rows_read.push_back(scanner->get_rows_read());
-                _finished_scanner_wait_worker_time.push_back(
-                        scanner->get_scanner_wait_worker_timer());
-                static_cast<void>(scanner->close(_state));
-            } else {
-                current_run->push_back(scanner);
-                i++;
-            }
+    for (int i = 0; i < thread_slot_num && !_scanners.empty();) {
+        std::weak_ptr<ScannerDelegate> scanner_ref = _scanners.front();
+        std::shared_ptr<ScannerDelegate> scanner = scanner_ref.lock();
+        _scanners.pop_front();
+        if (scanner == nullptr) {
+            continue;
+        }
+        if (scanner->_scanner->need_to_close()) {
+            static_cast<void>(scanner->_scanner->close(_state));
+        } else {
+            current_run->push_back(scanner_ref);
+            i++;
         }
     }
 }
 
-template void ScannerContext::clear_and_join(pipeline::ScanLocalStateBase* parent,
-                                             RuntimeState* state);
-template void ScannerContext::clear_and_join(VScanNode* parent, RuntimeState* state);
-
 } // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/scanner_context.h b/be/src/vec/exec/scan/scanner_context.h
index 8e840a47465882..c4df896bacb664 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -53,6 +53,7 @@ class TaskGroup;
 namespace vectorized {
 
 class VScanner;
+class ScannerDelegate;
 class VScanNode;
 class ScannerScheduler;
 class SimplifiedScanScheduler;
@@ -72,7 +73,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
 public:
     ScannerContext(RuntimeState* state, VScanNode* parent, const TupleDescriptor* output_tuple_desc,
                    const RowDescriptor* output_row_descriptor,
-                   const std::list<VScannerSPtr>& scanners, int64_t limit_,
+                   const std::list<std::shared_ptr<ScannerDelegate>>& scanners, int64_t limit_,
                    int64_t max_bytes_in_blocks_queue, const int num_parallel_instances = 1,
                    pipeline::ScanLocalStateBase* local_state = nullptr);
 
@@ -94,9 +95,9 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
 
     // When a scanner complete a scan, this method will be called
     // to return the scanner to the list for next scheduling.
-    void push_back_scanner_and_reschedule(VScannerSPtr scanner);
+    void push_back_scanner_and_reschedule(std::shared_ptr<ScannerDelegate> scanner);
 
-    bool set_status_on_error(const Status& status, bool need_lock = true);
+    void set_status_on_error(const Status& status, bool need_lock = true);
 
     Status status() {
         if (_process_status.is<ErrorCode::END_OF_FILE>()) {
@@ -105,42 +106,27 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
         return _process_status;
     }
 
-    // Called by ScanNode.
-    // Used to notify the scheduler that this ScannerContext can stop working.
-    void set_should_stop();
-
     // Return true if this ScannerContext need no more process
-    virtual bool done() { return _is_finished || _should_stop; }
+    bool done() const { return _is_finished || _should_stop; }
     bool is_finished() { return _is_finished.load(); }
     bool should_stop() { return _should_stop.load(); }
-    bool status_error() { return _status_error.load(); }
 
     void inc_num_running_scanners(int32_t scanner_inc);
 
-    void set_ready_to_finish();
-
     int get_num_running_scanners() const { return _num_running_scanners; }
 
     int get_num_unfinished_scanners() const { return _num_unfinished_scanners; }
 
-    void dec_num_scheduling_ctx();
-
-    int get_num_scheduling_ctx() const { return _num_scheduling_ctx; }
-
-    void get_next_batch_of_scanners(std::list<VScannerSPtr>* current_run);
-
-    template <typename Parent>
-    void clear_and_join(Parent* parent, RuntimeState* state);
-
-    bool no_schedule();
+    void get_next_batch_of_scanners(std::list<std::weak_ptr<ScannerDelegate>>* current_run);
 
     virtual std::string debug_string();
 
     RuntimeState* state() { return _state; }
 
     void incr_num_ctx_scheduling(int64_t num) { _scanner_ctx_sched_counter->update(num); }
+
+    int64_t num_ctx_scheduled() { return _scanner_ctx_sched_counter->value(); }
     void incr_ctx_scheduling_time(int64_t num) { _scanner_ctx_sched_time->update(num); }
-    void incr_num_scanner_scheduling(int64_t num) { _scanner_sched_counter->update(num); }
 
     std::string parent_name();
 
@@ -172,6 +158,7 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
     SimplifiedScanScheduler* get_simple_scan_scheduler() { return _simple_scan_scheduler; }
 
     virtual void reschedule_scanner_ctx();
+    void stop_scanners(RuntimeState* state);
 
     // the unique id of this context
     std::string ctx_id;
@@ -181,18 +168,13 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
 
     bool _should_reset_thread_name = true;
 
-private:
-    template <typename Parent>
-    Status _close_and_clear_scanners(Parent* parent, RuntimeState* state);
-
 protected:
     ScannerContext(RuntimeState* state_, const TupleDescriptor* output_tuple_desc,
                    const RowDescriptor* output_row_descriptor,
-                   const std::list<VScannerSPtr>& scanners_, int64_t limit_,
+                   const std::list<std::shared_ptr<ScannerDelegate>>& scanners_, int64_t limit_,
                    int64_t max_bytes_in_blocks_queue_, const int num_parallel_instances,
                    pipeline::ScanLocalStateBase* local_state,
-                   std::shared_ptr<pipeline::ScanDependency> dependency,
-                   std::shared_ptr<pipeline::Dependency> finish_dependency);
+                   std::shared_ptr<pipeline::ScanDependency> dependency);
     virtual void _dispose_coloate_blocks_not_in_queue() {}
 
     void _set_scanner_done();
@@ -232,7 +214,6 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
     //      Always be set by ScannerScheduler.
     //      True means all scanners are finished to scan.
     Status _process_status;
-    std::atomic_bool _status_error = false;
     std::atomic_bool _should_stop = false;
     std::atomic_bool _is_finished = false;
 
@@ -276,9 +257,11 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
     // and then if the scanner is not finished, will be pushed back to this list.
     // Not need to protect by lock, because only one scheduler thread will access to it.
     std::mutex _scanners_lock;
-    std::list<VScannerSPtr> _scanners;
+    // Scanner's ownership belong to vscannode or scanoperator, scanner context does not own it.
+    // ScannerContext has to check if scanner is deconstructed before use it.
+    std::list<std::weak_ptr<ScannerDelegate>> _scanners;
     // weak pointer for _scanners, used in stop function
-    std::vector<VScannerWPtr> _scanners_ref;
+    std::vector<std::weak_ptr<ScannerDelegate>> _all_scanners;
     std::vector<int64_t> _finished_scanner_runtime;
     std::vector<int64_t> _finished_scanner_rows_read;
     std::vector<int64_t> _finished_scanner_wait_worker_time;
@@ -295,7 +278,6 @@ class ScannerContext : public std::enable_shared_from_this<ScannerContext>,
     RuntimeProfile::Counter* _scanner_wait_batch_timer = nullptr;
 
     std::shared_ptr<pipeline::ScanDependency> _dependency = nullptr;
-    std::shared_ptr<pipeline::Dependency> _finish_dependency = nullptr;
 };
 } // namespace vectorized
 } // namespace doris
diff --git a/be/src/vec/exec/scan/scanner_scheduler.cpp b/be/src/vec/exec/scan/scanner_scheduler.cpp
index 29b53b39353bba..4223bd7abb0c68 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.cpp
+++ b/be/src/vec/exec/scan/scanner_scheduler.cpp
@@ -143,6 +143,8 @@ Status ScannerScheduler::submit(std::shared_ptr<ScannerContext> ctx) {
     if (ctx->done()) {
         return Status::EndOfFile("ScannerContext is done");
     }
+    //LOG(WARNING) << "yyyy " << Status::InternalError("Too many scheduled");
+    //LOG(WARNING) << "yyyy " << ctx->debug_string();
     ctx->queue_idx = (_queue_idx++ % QUEUE_NUM);
     if (!_pending_queues[ctx->queue_idx]->blocking_put(ctx)) {
         return Status::InternalError("failed to submit scanner context to scheduler");
@@ -174,28 +176,23 @@ void ScannerScheduler::_schedule_thread(int queue_id) {
 void ScannerScheduler::_schedule_scanners(std::shared_ptr<ScannerContext> ctx) {
     auto task_lock = ctx->task_exec_ctx();
     if (task_lock == nullptr) {
-        // LOG(WARNING) << "could not lock task execution context, query " << print_id(_query_id)
-        //             << " maybe finished";
+        LOG(WARNING) << "could not lock task execution context, query " << ctx->debug_string()
+                     << " maybe finished";
         return;
     }
+    //LOG(INFO) << "yyyy scheduled, query " << ctx->debug_string() << " maybe finished";
     MonotonicStopWatch watch;
     watch.reset();
     watch.start();
     ctx->incr_num_ctx_scheduling(1);
-    size_t size = 0;
-    Defer defer {[&]() {
-        ctx->incr_num_scanner_scheduling(size);
-        ctx->dec_num_scheduling_ctx();
-    }};
 
     if (ctx->done()) {
         return;
     }
 
-    std::list<VScannerSPtr> this_run;
+    std::list<std::weak_ptr<ScannerDelegate>> this_run;
     ctx->get_next_batch_of_scanners(&this_run);
-    size = this_run.size();
-    if (!size) {
+    if (this_run.empty()) {
         // There will be 2 cases when this_run is empty:
         // 1. The blocks queue reaches limit.
         //      The consumer will continue scheduling the ctx.
@@ -214,9 +211,14 @@ void ScannerScheduler::_schedule_scanners(std::shared_ptr<ScannerContext> ctx) {
     if (ctx->thread_token != nullptr) {
         // TODO llj tg how to treat this?
         while (iter != this_run.end()) {
-            (*iter)->start_wait_worker_timer();
-            auto s = ctx->thread_token->submit_func(
-                    [this, scanner = *iter, ctx] { this->_scanner_scan(this, ctx, scanner); });
+            std::shared_ptr<ScannerDelegate> scanner_delegate = (*iter).lock();
+            if (scanner_delegate == nullptr) {
+                continue;
+            }
+            scanner_delegate->_scanner->start_wait_worker_timer();
+            auto s = ctx->thread_token->submit_func([this, scanner_ref = *iter, ctx]() {
+                this->_scanner_scan(this, ctx, scanner_ref);
+            });
             if (s.ok()) {
                 this_run.erase(iter++);
             } else {
@@ -226,28 +228,32 @@ void ScannerScheduler::_schedule_scanners(std::shared_ptr<ScannerContext> ctx) {
         }
     } else {
         while (iter != this_run.end()) {
-            (*iter)->start_wait_worker_timer();
-            TabletStorageType type = (*iter)->get_storage_type();
+            std::shared_ptr<ScannerDelegate> scanner_delegate = (*iter).lock();
+            if (scanner_delegate == nullptr) {
+                continue;
+            }
+            scanner_delegate->_scanner->start_wait_worker_timer();
+            TabletStorageType type = scanner_delegate->_scanner->get_storage_type();
             bool ret = false;
             if (type == TabletStorageType::STORAGE_TYPE_LOCAL) {
                 if (auto* scan_sche = ctx->get_simple_scan_scheduler()) {
-                    auto work_func = [this, scanner = *iter, ctx] {
-                        this->_scanner_scan(this, ctx, scanner);
+                    auto work_func = [this, scanner_ref = *iter, ctx]() {
+                        this->_scanner_scan(this, ctx, scanner_ref);
                     };
                     SimplifiedScanTask simple_scan_task = {work_func, ctx};
                     ret = scan_sche->get_scan_queue()->try_put(simple_scan_task);
                 } else {
                     PriorityThreadPool::Task task;
-                    task.work_function = [this, scanner = *iter, ctx] {
-                        this->_scanner_scan(this, ctx, scanner);
+                    task.work_function = [this, scanner_ref = *iter, ctx]() {
+                        this->_scanner_scan(this, ctx, scanner_ref);
                     };
                     task.priority = nice;
                     ret = _local_scan_thread_pool->offer(task);
                 }
             } else {
                 PriorityThreadPool::Task task;
-                task.work_function = [this, scanner = *iter, ctx] {
-                    this->_scanner_scan(this, ctx, scanner);
+                task.work_function = [this, scanner_ref = *iter, ctx]() {
+                    this->_scanner_scan(this, ctx, scanner_ref);
                 };
                 task.priority = nice;
                 ret = _remote_scan_thread_pool->offer(task);
@@ -265,13 +271,22 @@ void ScannerScheduler::_schedule_scanners(std::shared_ptr<ScannerContext> ctx) {
 }
 
 void ScannerScheduler::_scanner_scan(ScannerScheduler* scheduler,
-                                     std::shared_ptr<ScannerContext> ctx, VScannerSPtr scanner) {
+                                     std::shared_ptr<ScannerContext> ctx,
+                                     std::weak_ptr<ScannerDelegate> scanner_ref) {
     auto task_lock = ctx->task_exec_ctx();
     if (task_lock == nullptr) {
         // LOG(WARNING) << "could not lock task execution context, query " << print_id(_query_id)
         //             << " maybe finished";
         return;
     }
+    //LOG_EVERY_N(INFO, 100) << "start running scanner from ctx " << ctx->debug_string();
+    // will release scanner if it is the last one, task lock is hold here, to ensure
+    // that scanner could call scannode's method during deconstructor
+    std::shared_ptr<ScannerDelegate> scanner_delegate = scanner_ref.lock();
+    auto& scanner = scanner_delegate->_scanner;
+    if (scanner_delegate == nullptr) {
+        return;
+    }
     SCOPED_ATTACH_TASK(scanner->runtime_state());
     // for cpu hard limit, thread name should not be reset
     if (ctx->_should_reset_thread_name) {
@@ -404,7 +419,7 @@ void ScannerScheduler::_scanner_scan(ScannerScheduler* scheduler,
     if (eos || should_stop) {
         scanner->mark_to_need_to_close();
     }
-    ctx->push_back_scanner_and_reschedule(scanner);
+    ctx->push_back_scanner_and_reschedule(scanner_delegate);
 }
 
 void ScannerScheduler::_register_metrics() {
diff --git a/be/src/vec/exec/scan/scanner_scheduler.h b/be/src/vec/exec/scan/scanner_scheduler.h
index eb4d1380e3947c..9fedd27dbd8bb3 100644
--- a/be/src/vec/exec/scan/scanner_scheduler.h
+++ b/be/src/vec/exec/scan/scanner_scheduler.h
@@ -36,7 +36,7 @@ class BlockingQueue;
 } // namespace doris
 
 namespace doris::vectorized {
-
+class ScannerDelegate;
 class ScannerContext;
 
 // Responsible for the scheduling and execution of all Scanners of a BE node.
@@ -79,7 +79,7 @@ class ScannerScheduler {
     void _schedule_scanners(std::shared_ptr<ScannerContext> ctx);
     // execution thread function
     void _scanner_scan(ScannerScheduler* scheduler, std::shared_ptr<ScannerContext> ctx,
-                       VScannerSPtr scanner);
+                       std::weak_ptr<ScannerDelegate> scanner);
 
     void _register_metrics();
 
diff --git a/be/src/vec/exec/scan/vscan_node.cpp b/be/src/vec/exec/scan/vscan_node.cpp
index c45468e0514d30..88f44edfdb8c08 100644
--- a/be/src/vec/exec/scan/vscan_node.cpp
+++ b/be/src/vec/exec/scan/vscan_node.cpp
@@ -197,9 +197,13 @@ Status VScanNode::alloc_resource(RuntimeState* state) {
             if (_scanner_ctx) {
                 DCHECK(!_eos && _num_scanners->value() > 0);
                 RETURN_IF_ERROR(_scanner_ctx->init());
+                //LOG(INFO) << "yyyy instance " << print_id(state->fragment_instance_id())
+                //          << " submit scanner ctx " << _scanner_ctx->debug_string();
                 RETURN_IF_ERROR(_state->exec_env()->scanner_scheduler()->submit(_scanner_ctx));
             }
             if (_shared_scan_opt) {
+                LOG(INFO) << "instance shared scan enabled"
+                          << print_id(state->fragment_instance_id());
                 _shared_scanner_controller->set_scanner_context(id(),
                                                                 _eos ? nullptr : _scanner_ctx);
             }
@@ -273,7 +277,7 @@ Status VScanNode::get_next(RuntimeState* state, vectorized::Block* block, bool*
     reached_limit(block, eos);
     if (*eos) {
         // reach limit, stop the scanners.
-        _scanner_ctx->set_should_stop();
+        _scanner_ctx->stop_scanners(state);
     }
 
     return Status::OK();
@@ -318,8 +322,8 @@ Status VScanNode::_init_profile() {
     return Status::OK();
 }
 
-Status VScanNode::_start_scanners(const std::list<VScannerSPtr>& scanners,
-                                  const int query_parallel_instance_num) {
+void VScanNode::_start_scanners(const std::list<std::shared_ptr<ScannerDelegate>>& scanners,
+                                const int query_parallel_instance_num) {
     if (_is_pipeline_scan) {
         int max_queue_size = _shared_scan_opt ? std::max(query_parallel_instance_num, 1) : 1;
         _scanner_ctx = pipeline::PipScannerContext::create_shared(
@@ -330,41 +334,29 @@ Status VScanNode::_start_scanners(const std::list<VScannerSPtr>& scanners,
                                                      _output_row_descriptor.get(), scanners,
                                                      limit(), _state->scan_queue_mem_limit());
     }
-    return Status::OK();
 }
 
 Status VScanNode::close(RuntimeState* state) {
     if (is_closed()) {
         return Status::OK();
     }
+
     RETURN_IF_ERROR(ExecNode::close(state));
     return Status::OK();
 }
 
 void VScanNode::release_resource(RuntimeState* state) {
     if (_scanner_ctx) {
-        if (!state->enable_pipeline_exec()) {
+        if (!state->enable_pipeline_exec() || _should_create_scanner) {
             // stop and wait the scanner scheduler to be done
             // _scanner_ctx may not be created for some short circuit case.
-            _scanner_ctx->set_should_stop();
-            _scanner_ctx->clear_and_join(this, state);
-        } else if (_should_create_scanner) {
-            _scanner_ctx->clear_and_join(this, state);
+            _scanner_ctx->stop_scanners(state);
         }
     }
-
+    _scanners.clear();
     ExecNode::release_resource(state);
 }
 
-Status VScanNode::try_close(RuntimeState* state) {
-    if (_scanner_ctx) {
-        // mark this scanner ctx as should_stop to make sure scanners will not be scheduled anymore
-        // TODO: there is a lock in `set_should_stop` may cause some slight impact
-        _scanner_ctx->set_should_stop();
-    }
-    return Status::OK();
-}
-
 Status VScanNode::_normalize_conjuncts() {
     // The conjuncts is always on output tuple, so use _output_tuple_desc;
     std::vector<SlotDescriptor*> slots = _output_tuple_desc->slots();
@@ -1330,11 +1322,15 @@ VScanNode::PushDownType VScanNode::_should_push_down_in_predicate(VInPredicate*
 Status VScanNode::_prepare_scanners(const int query_parallel_instance_num) {
     std::list<VScannerSPtr> scanners;
     RETURN_IF_ERROR(_init_scanners(&scanners));
+    // Init scanner wrapper
+    for (auto it = scanners.begin(); it != scanners.end(); ++it) {
+        _scanners.emplace_back(std::make_shared<ScannerDelegate>(*it));
+    }
     if (scanners.empty()) {
         _eos = true;
     } else {
         COUNTER_SET(_num_scanners, static_cast<int64_t>(scanners.size()));
-        RETURN_IF_ERROR(_start_scanners(scanners, query_parallel_instance_num));
+        _start_scanners(_scanners, query_parallel_instance_num);
     }
     return Status::OK();
 }
diff --git a/be/src/vec/exec/scan/vscan_node.h b/be/src/vec/exec/scan/vscan_node.h
index 7cd8b15991ef2d..1d54df4f4b4a59 100644
--- a/be/src/vec/exec/scan/vscan_node.h
+++ b/be/src/vec/exec/scan/vscan_node.h
@@ -87,6 +87,21 @@ struct FilterPredicates {
     std::vector<std::pair<std::string, std::shared_ptr<HybridSetBase>>> in_filters;
 };
 
+// We want to close scanner automatically, so using a delegate class
+// and call close method in the delegate class's dctor.
+class ScannerDelegate {
+public:
+    VScannerSPtr _scanner;
+    ScannerDelegate(VScannerSPtr& scanner_ptr) : _scanner(scanner_ptr) {}
+    ~ScannerDelegate() {
+        Status st = _scanner->close(_scanner->runtime_state());
+        if (!st.ok()) {
+            LOG(WARNING) << "close scanner failed, st = " << st;
+        }
+    }
+    ScannerDelegate(ScannerDelegate&&) = delete;
+};
+
 class VScanNode : public ExecNode, public RuntimeFilterConsumer {
 public:
     VScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
@@ -156,8 +171,6 @@ class VScanNode : public ExecNode, public RuntimeFilterConsumer {
     Status alloc_resource(RuntimeState* state) override;
     void release_resource(RuntimeState* state) override;
 
-    Status try_close(RuntimeState* state);
-
     bool should_run_serial() const {
         return _should_run_serial || _state->enable_scan_node_run_serial();
     }
@@ -276,8 +289,11 @@ class VScanNode : public ExecNode, public RuntimeFilterConsumer {
     int _max_scan_key_num;
     int _max_pushdown_conditions_per_column;
 
-    // Each scan node will generates a ScannerContext to manage all Scanners.
-    // See comments of ScannerContext for more details
+    // ScanNode owns the ownership of scanner, scanner context only has its weakptr
+    std::list<std::shared_ptr<ScannerDelegate>> _scanners;
+
+    // Each scan node will generates a ScannerContext to do schedule work
+    // ScannerContext will be added to scanner scheduler
     std::shared_ptr<ScannerContext> _scanner_ctx = nullptr;
 
     // indicate this scan node has no more data to return
@@ -451,8 +467,8 @@ class VScanNode : public ExecNode, public RuntimeFilterConsumer {
                                       const std::string& fn_name, int slot_ref_child = -1);
 
     // Submit the scanner to the thread pool and start execution
-    Status _start_scanners(const std::list<VScannerSPtr>& scanners,
-                           const int query_parallel_instance_num);
+    void _start_scanners(const std::list<std::shared_ptr<ScannerDelegate>>& scanners,
+                         const int query_parallel_instance_num);
 };
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/exec/scan/vscanner.h b/be/src/vec/exec/scan/vscanner.h
index b00c3f348f41d3..9fdaddcea21653 100644
--- a/be/src/vec/exec/scan/vscanner.h
+++ b/be/src/vec/exec/scan/vscanner.h
@@ -148,16 +148,6 @@ class VScanner {
 
     void set_status_on_failure(const Status& st) { _status = st; }
 
-    // return false if _is_counted_down is already true,
-    // otherwise, set _is_counted_down to true and return true.
-    bool set_counted_down() {
-        if (_is_counted_down) {
-            return false;
-        }
-        _is_counted_down = true;
-        return true;
-    }
-
 protected:
     void _discard_conjuncts() {
         for (auto& conjunct : _conjuncts) {
@@ -221,8 +211,6 @@ class VScanner {
     int64_t _scan_cpu_timer = 0;
 
     bool _is_load = false;
-    // set to true after decrease the "_num_unfinished_scanners" in scanner context
-    bool _is_counted_down = false;
 
     bool _is_init = true;
 
@@ -233,6 +221,5 @@ class VScanner {
 };
 
 using VScannerSPtr = std::shared_ptr<VScanner>;
-using VScannerWPtr = std::weak_ptr<VScanner>;
 
 } // namespace doris::vectorized
diff --git a/regression-test/pipeline/p0/conf/be.conf b/regression-test/pipeline/p0/conf/be.conf
index 2a25830a083f8d..0aca39d4f13933 100644
--- a/regression-test/pipeline/p0/conf/be.conf
+++ b/regression-test/pipeline/p0/conf/be.conf
@@ -77,3 +77,5 @@ enable_feature_binlog=true
 max_sys_mem_available_low_water_mark_bytes=69206016
 user_files_secure_path=/
 enable_debug_points=true
+# debug scanner context dead loop
+enable_debug_log_timeout_secs=300

From d93812d23f99108bca6aaa5bb45f2ceeac92fb66 Mon Sep 17 00:00:00 2001
From: xueweizhang <zxw520blue1@163.com>
Date: Wed, 3 Jan 2024 20:47:56 +0800
Subject: [PATCH 296/299] [fix](auditloader) support audit table millisecond
 and fix stmt truncated by '\r' (#29479)

Signed-off-by: nextdreamblue <zxw520blue1@163.com>
---
 .../org/apache/doris/plugin/audit/AuditLoaderPlugin.java  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
index 3cfb0eeeaee3a5..4c5586ed881331 100755
--- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
+++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
@@ -57,7 +57,7 @@
 public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
     private final static Logger LOG = LogManager.getLogger(AuditLoaderPlugin.class);
 
-    private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
+    private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")
             .withZone(ZoneId.systemDefault());
 
     private StringBuilder auditLogBuffer = new StringBuilder();
@@ -180,7 +180,9 @@ private void fillLogBuffer(AuditEvent event, StringBuilder logBuffer) {
         logBuffer.append(event.peakMemoryBytes).append("\t");
         // trim the query to avoid too long
         // use `getBytes().length` to get real byte length
-        String stmt = truncateByBytes(event.stmt).replace("\n", " ").replace("\t", " ");
+        String stmt = truncateByBytes(event.stmt).replace("\n", " ")
+                                                    .replace("\t", " ")
+                                                    .replace("\r", " ");
         LOG.debug("receive audit event with stmt: {}", stmt);
         logBuffer.append(stmt).append("\n");
     }
@@ -355,7 +357,7 @@ public void run() {
 
     public static String longToTimeString(long timeStamp) {
         if (timeStamp <= 0L) {
-            return "1900-01-01 00:00:00";
+            return "1900-01-01 00:00:00.000";
         }
         return DATETIME_FORMAT.format(LocalDateTime.ofInstant(Instant.ofEpochMilli(timeStamp), ZoneId.systemDefault()));
     }

From d6cb2d6d5c0ccbc454a801d7ef68c7241bb3cd6c Mon Sep 17 00:00:00 2001
From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com>
Date: Wed, 3 Jan 2024 20:48:11 +0800
Subject: [PATCH 297/299] [improvement](compaction) start 1 cumu compaction
 thread each disk by default (#29430)

---
 be/src/agent/task_worker_pool.cpp             |  3 +-
 be/src/common/config.cpp                      |  4 +-
 be/src/olap/olap_server.cpp                   | 90 ++++++++++++-------
 docs/en/docs/admin-manual/config/be-config.md | 12 +--
 .../docs/admin-manual/config/be-config.md     | 10 +--
 5 files changed, 75 insertions(+), 44 deletions(-)

diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp
index f9d43ba23180fd..a845b6253f90f6 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1528,7 +1528,8 @@ void PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
                     if (!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
                         tablet->published_count.fetch_add(1);
                         int64_t published_count = tablet->published_count.load();
-                        if (published_count % 10 == 0) {
+                        if (tablet->exceed_version_limit(config::max_tablet_version_num * 2 / 3) &&
+                            published_count % 20 == 0) {
                             auto st = _engine.submit_compaction_task(
                                     tablet, CompactionType::CUMULATIVE_COMPACTION, true);
                             if (!st.ok()) [[unlikely]] {
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 040fc5ae21705b..a53885bd1361f8 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -365,8 +365,8 @@ DEFINE_mInt32(ordered_data_compaction_min_segment_size, "10485760");
 
 // This config can be set to limit thread number in compaction thread pool.
 DEFINE_mInt32(max_base_compaction_threads, "4");
-DEFINE_mInt32(max_cumu_compaction_threads, "10");
-DEFINE_mInt32(max_single_replica_compaction_threads, "10");
+DEFINE_mInt32(max_cumu_compaction_threads, "-1");
+DEFINE_mInt32(max_single_replica_compaction_threads, "-1");
 
 DEFINE_Bool(enable_base_compaction_idle_sched, "true");
 DEFINE_mInt64(base_compaction_min_rowset_num, "5");
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 9ee24dee19ab62..60c309584a22ea 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -93,6 +93,33 @@ volatile uint32_t g_schema_change_active_threads = 0;
 static const uint64_t DEFAULT_SEED = 104729;
 static const uint64_t MOD_PRIME = 7652413;
 
+static int32_t get_cumu_compaction_threads_num(size_t data_dirs_num) {
+    int32_t threads_num = config::max_cumu_compaction_threads;
+    if (threads_num == -1) {
+        threads_num = data_dirs_num;
+    }
+    threads_num = threads_num <= 0 ? 1 : threads_num;
+    return threads_num;
+}
+
+static int32_t get_base_compaction_threads_num(size_t data_dirs_num) {
+    int32_t threads_num = config::max_base_compaction_threads;
+    if (threads_num == -1) {
+        threads_num = data_dirs_num;
+    }
+    threads_num = threads_num <= 0 ? 1 : threads_num;
+    return threads_num;
+}
+
+static int32_t get_single_replica_compaction_threads_num(size_t data_dirs_num) {
+    int32_t threads_num = config::max_single_replica_compaction_threads;
+    if (threads_num == -1) {
+        threads_num = data_dirs_num;
+    }
+    threads_num = threads_num <= 0 ? 1 : threads_num;
+    return threads_num;
+}
+
 Status StorageEngine::start_bg_threads() {
     RETURN_IF_ERROR(Thread::create(
             "StorageEngine", "unused_rowset_monitor_thread",
@@ -118,17 +145,22 @@ Status StorageEngine::start_bg_threads() {
         data_dirs.push_back(tmp_store.second);
     }
 
+    auto base_compaction_threads = get_base_compaction_threads_num(data_dirs.size());
+    auto cumu_compaction_threads = get_cumu_compaction_threads_num(data_dirs.size());
+    auto single_replica_compaction_threads =
+            get_single_replica_compaction_threads_num(data_dirs.size());
+
     RETURN_IF_ERROR(ThreadPoolBuilder("BaseCompactionTaskThreadPool")
-                            .set_min_threads(config::max_base_compaction_threads)
-                            .set_max_threads(config::max_base_compaction_threads)
+                            .set_min_threads(base_compaction_threads)
+                            .set_max_threads(base_compaction_threads)
                             .build(&_base_compaction_thread_pool));
     RETURN_IF_ERROR(ThreadPoolBuilder("CumuCompactionTaskThreadPool")
-                            .set_min_threads(config::max_cumu_compaction_threads)
-                            .set_max_threads(config::max_cumu_compaction_threads)
+                            .set_min_threads(cumu_compaction_threads)
+                            .set_max_threads(cumu_compaction_threads)
                             .build(&_cumu_compaction_thread_pool));
     RETURN_IF_ERROR(ThreadPoolBuilder("SingleReplicaCompactionTaskThreadPool")
-                            .set_min_threads(config::max_single_replica_compaction_threads)
-                            .set_max_threads(config::max_single_replica_compaction_threads)
+                            .set_min_threads(single_replica_compaction_threads)
+                            .set_max_threads(single_replica_compaction_threads)
                             .build(&_single_replica_compaction_thread_pool));
 
     if (config::enable_segcompaction) {
@@ -458,64 +490,62 @@ void StorageEngine::_tablet_path_check_callback() {
 }
 
 void StorageEngine::_adjust_compaction_thread_num() {
-    if (_base_compaction_thread_pool->max_threads() != config::max_base_compaction_threads) {
+    auto base_compaction_threads_num = get_base_compaction_threads_num(_store_map.size());
+    if (_base_compaction_thread_pool->max_threads() != base_compaction_threads_num) {
         int old_max_threads = _base_compaction_thread_pool->max_threads();
-        Status status =
-                _base_compaction_thread_pool->set_max_threads(config::max_base_compaction_threads);
+        Status status = _base_compaction_thread_pool->set_max_threads(base_compaction_threads_num);
         if (status.ok()) {
             VLOG_NOTICE << "update base compaction thread pool max_threads from " << old_max_threads
-                        << " to " << config::max_base_compaction_threads;
+                        << " to " << base_compaction_threads_num;
         }
     }
-    if (_base_compaction_thread_pool->min_threads() != config::max_base_compaction_threads) {
+    if (_base_compaction_thread_pool->min_threads() != base_compaction_threads_num) {
         int old_min_threads = _base_compaction_thread_pool->min_threads();
-        Status status =
-                _base_compaction_thread_pool->set_min_threads(config::max_base_compaction_threads);
+        Status status = _base_compaction_thread_pool->set_min_threads(base_compaction_threads_num);
         if (status.ok()) {
             VLOG_NOTICE << "update base compaction thread pool min_threads from " << old_min_threads
-                        << " to " << config::max_base_compaction_threads;
+                        << " to " << base_compaction_threads_num;
         }
     }
 
-    if (_cumu_compaction_thread_pool->max_threads() != config::max_cumu_compaction_threads) {
+    auto cumu_compaction_threads_num = get_cumu_compaction_threads_num(_store_map.size());
+    if (_cumu_compaction_thread_pool->max_threads() != cumu_compaction_threads_num) {
         int old_max_threads = _cumu_compaction_thread_pool->max_threads();
-        Status status =
-                _cumu_compaction_thread_pool->set_max_threads(config::max_cumu_compaction_threads);
+        Status status = _cumu_compaction_thread_pool->set_max_threads(cumu_compaction_threads_num);
         if (status.ok()) {
             VLOG_NOTICE << "update cumu compaction thread pool max_threads from " << old_max_threads
-                        << " to " << config::max_cumu_compaction_threads;
+                        << " to " << cumu_compaction_threads_num;
         }
     }
-    if (_cumu_compaction_thread_pool->min_threads() != config::max_cumu_compaction_threads) {
+    if (_cumu_compaction_thread_pool->min_threads() != cumu_compaction_threads_num) {
         int old_min_threads = _cumu_compaction_thread_pool->min_threads();
-        Status status =
-                _cumu_compaction_thread_pool->set_min_threads(config::max_cumu_compaction_threads);
+        Status status = _cumu_compaction_thread_pool->set_min_threads(cumu_compaction_threads_num);
         if (status.ok()) {
             VLOG_NOTICE << "update cumu compaction thread pool min_threads from " << old_min_threads
-                        << " to " << config::max_cumu_compaction_threads;
+                        << " to " << cumu_compaction_threads_num;
         }
     }
 
+    auto single_replica_compaction_threads_num =
+            get_single_replica_compaction_threads_num(_store_map.size());
     if (_single_replica_compaction_thread_pool->max_threads() !=
-        config::max_single_replica_compaction_threads) {
+        single_replica_compaction_threads_num) {
         int old_max_threads = _single_replica_compaction_thread_pool->max_threads();
         Status status = _single_replica_compaction_thread_pool->set_max_threads(
-                config::max_single_replica_compaction_threads);
+                single_replica_compaction_threads_num);
         if (status.ok()) {
             VLOG_NOTICE << "update single replica compaction thread pool max_threads from "
-                        << old_max_threads << " to "
-                        << config::max_single_replica_compaction_threads;
+                        << old_max_threads << " to " << single_replica_compaction_threads_num;
         }
     }
     if (_single_replica_compaction_thread_pool->min_threads() !=
-        config::max_single_replica_compaction_threads) {
+        single_replica_compaction_threads_num) {
         int old_min_threads = _single_replica_compaction_thread_pool->min_threads();
         Status status = _single_replica_compaction_thread_pool->set_min_threads(
-                config::max_single_replica_compaction_threads);
+                single_replica_compaction_threads_num);
         if (status.ok()) {
             VLOG_NOTICE << "update single replica compaction thread pool min_threads from "
-                        << old_min_threads << " to "
-                        << config::max_single_replica_compaction_threads;
+                        << old_min_threads << " to " << single_replica_compaction_threads_num;
         }
     }
 }
diff --git a/docs/en/docs/admin-manual/config/be-config.md b/docs/en/docs/admin-manual/config/be-config.md
index 1b8ee9423dc76e..ec53e9a81107eb 100644
--- a/docs/en/docs/admin-manual/config/be-config.md
+++ b/docs/en/docs/admin-manual/config/be-config.md
@@ -482,9 +482,9 @@ There are two ways to configure BE configuration items:
 * Default value: 10485760
 
 #### `max_base_compaction_threads`
-
+git 
 * Type: int32
-* Description: The maximum of thread number in base compaction thread pool.
+* Description: The maximum of thread number in base compaction thread pool, -1 means one thread per disk.
 * Default value: 4
 
 #### `generate_compaction_tasks_interval_ms`
@@ -625,8 +625,8 @@ BaseCompaction:546859:
 #### `max_cumu_compaction_threads`
 
 * Type: int32
-* Description: The maximum of thread number in cumulative compaction thread pool.
-* Default value: 10
+* Description: The maximum of thread number in cumulative compaction thread pool, -1 means one thread per disk.
+* Default value: -1
 
 #### `enable_segcompaction`
 
@@ -686,8 +686,8 @@ BaseCompaction:546859:
 #### `max_single_replica_compaction_threads`
 
 * Type: int32
-* Description: The maximum of thread number in single replica compaction thread pool.
-* Default value: 10
+* Description: The maximum of thread number in single replica compaction thread pool. -1 means one thread per disk.
+* Default value: -1
 
 #### `update_replica_infos_interval_seconds`
 
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md b/docs/zh-CN/docs/admin-manual/config/be-config.md
index 7b99c1f2cb9068..db4e6764a290cf 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -498,7 +498,7 @@ BE 重启后该配置将失效。如果想持久化修改结果，使用如下
 #### `max_base_compaction_threads`
 
 * 类型：int32
-* 描述：Base Compaction线程池中线程数量的最大值。
+* 描述：Base Compaction线程池中线程数量的最大值, -1 表示每个磁盘一个线程。
 * 默认值：4
 
 #### `generate_compaction_tasks_interval_ms`
@@ -639,8 +639,8 @@ BaseCompaction:546859:
 #### `max_cumu_compaction_threads`
 
 * 类型：int32
-* 描述：Cumulative Compaction线程池中线程数量的最大值。
-* 默认值：10
+* 描述：Cumulative Compaction线程池中线程数量的最大值, -1 表示每个磁盘一个线程。
+* 默认值：-1
 
 #### `enable_segcompaction`
 
@@ -712,8 +712,8 @@ BaseCompaction:546859:
 #### `max_single_replica_compaction_threads`
 
 * 类型：int32
-* 描述：Single Replica Compaction 线程池中线程数量的最大值。
-* 默认值：10
+* 描述：Single Replica Compaction 线程池中线程数量的最大值, -1 表示每个磁盘一个线程。
+* 默认值：-1
 
 #### `update_replica_infos_interval_seconds`
 

From afaefa3a9e779ba4adc587b1ff4c0f85fea18937 Mon Sep 17 00:00:00 2001
From: TengJianPing <18241664+jacktengg@users.noreply.github.com>
Date: Wed, 3 Jan 2024 21:02:10 +0800
Subject: [PATCH 298/299] [regression](decimalv2) add schema change test case
 for decimalv2 (#29474)

---
 .../test_agg_keys_schema_change_decimalv2.out |  37 +++
 ...st_agg_keys_schema_change_decimalv2.groovy | 294 ++++++++++++++++++
 2 files changed, 331 insertions(+)
 create mode 100644 regression-test/data/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.out
 create mode 100644 regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy

diff --git a/regression-test/data/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.out b/regression-test/data/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.out
new file mode 100644
index 00000000000000..e9a914ce486ffd
--- /dev/null
+++ b/regression-test/data/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.out
@@ -0,0 +1,37 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql1 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.999	999999999999999999.999999994	999999999999999999.995
+
+-- !sql2 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.999	\N	999999999999999999.999999994	999999999999999999.995
+
+-- !sql3 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.999	\N	999999999999999999.999999994	999999999999999999.995
+
+-- !sql4 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.999	999999999999999999.999999994	999999999999999999.995
+
+-- !sql5 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.999	999999999999999999.999999994	999999999999999999.995
+
+-- !sql5_2 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.9990000000	999999999999999999.999999994	999999999999999999.995
+
+-- !sql6 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.9990000000	999999999999999999.99999999	999999999999999999.995
+
+-- !sql7 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.9990000000	999999999999999999.99999999	999999999999999999.995
+
+-- !sql8 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.9990000000	999999999999999999.99999999	999999999999999999.995
+
+-- !sql9 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.9990000000	999999999999999999.99999999	999999999999999999.995
+
+-- !sql9_2 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.9990000000	1000000000000000000.0000000	999999999999999999.995
+
+-- !sql9_3 --
+111111111111111111.111111111	999999999999999999.994	999999999999999999.9990000000	999999999999999999.99999999	999999999999999999.995
+
diff --git a/regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy b/regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy
new file mode 100644
index 00000000000000..88798db3c6c552
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/decimalv2/test_agg_keys_schema_change_decimalv2.groovy
@@ -0,0 +1,294 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_agg_keys_schema_change_decimalv2", "nonConcurrent") {
+    def config_row = sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'disable_decimalv2'; """
+    String old_value1 = config_row[0][1]
+
+    config_row = sql """ ADMIN SHOW FRONTEND CONFIG LIKE 'enable_decimal_conversion'; """
+    String old_value2 = config_row[0][1]
+
+    sql """
+        admin set frontend config("enable_decimal_conversion" = "false");
+    """
+    sql """
+        admin set frontend config("disable_decimalv2" = "false");
+    """
+
+    def tbName = "test_agg_keys_schema_change_decimalv2"
+    def getJobState = { tableName ->
+         def jobStateResult = sql """  SHOW ALTER TABLE COLUMN WHERE IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+         logger.info(jobStateResult.toString());
+         return jobStateResult[0][9]
+    }
+
+    String backend_id;
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+
+    backend_id = backendId_to_backendIP.keySet()[0]
+    def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id))
+    logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err)
+    assertEquals(code, 0)
+    def configList = parseJson(out.trim())
+    assert configList instanceof List
+
+    def do_compact = { tableName ->
+        String[][] tablets = sql """ show tablets from ${tableName}; """
+        for (String[] tablet in tablets) {
+            String tablet_id = tablet[0]
+            backend_id = tablet[2]
+            logger.info("run compaction:" + tablet_id)
+            (code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id )
+            logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
+        }
+
+        // wait for all compactions done
+        for (String[] tablet in tablets) {
+            boolean running = true
+            do {
+                Thread.sleep(100)
+                String tablet_id = tablet[0]
+                backend_id = tablet[2]
+                (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
+                logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
+                assertEquals(code, 0)
+                def compactionStatus = parseJson(out.trim())
+                assertEquals("success", compactionStatus.status.toLowerCase())
+                running = compactionStatus.run_status
+            } while (running)
+        }
+    }
+
+    sql """ DROP TABLE IF EXISTS ${tbName} FORCE"""
+    // Create table and disable light weight schema change
+    sql """
+           CREATE TABLE IF NOT EXISTS ${tbName}
+           (
+               `decimalv2k1` DECIMALV2(27,9),
+               `decimalv2k2` DECIMALV2(21,3),
+               `decimalv2k3` DECIMALV2(21,3),
+               `decimalv2v1` DECIMALV2(27,9) SUM,
+               `decimalv2v2` DECIMALV2(21,3) SUM
+           )
+           AGGREGATE  KEY(`decimalv2k1`,`decimalv2k2`, decimalv2k3)
+           DISTRIBUTED BY HASH(`decimalv2k1`) BUCKETS 1
+           PROPERTIES("replication_num" = "1", "light_schema_change" = "false");
+        """
+    sql """ insert into ${tbName} values
+        (111111111111111111.111111111,999999999999999999.994,999999999999999999.999,999999999999999999.999999994,999999999999999999.995);
+    """
+    qt_sql1 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    sql """ alter table ${tbName} add column `decimalv2v3` decimalv2(27,9) """
+    int max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+    sql """sync"""
+    qt_sql2 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+    do_compact(tbName)
+    sql """sync"""
+    qt_sql3 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    sql """ alter table ${tbName} drop column `decimalv2v3` """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql4 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // DECIMALV2(21,3) -> decimalv3 OK
+    sql """ alter table ${tbName} modify column decimalv2k2 DECIMALV3(21,3) key """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql5 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // DECIMALV2(21,3) -> decimalv3 OK
+    sql """ alter table ${tbName} modify column decimalv2k3 DECIMALV3(38,10) key """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql5_2 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // DECIMALV2(27,9) -> decimalv3, round scale part, not overflow
+    sql """ alter table ${tbName} modify column decimalv2v1 DECIMALV3(26,8) sum """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql6 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // DECIMALV2(21,3) -> decimalv3,  round scale part, overflow
+    sql """ alter table ${tbName} modify column decimalv2v2 DECIMALV3(20,2) sum """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "CANCELLED") {
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql7 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // DECIMALV2(21,3) -> decimalv3,  narrow integral, overflow
+    sql """ alter table ${tbName} modify column decimalv2v2 DECIMALV3(20,3) sum """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "CANCELLED") {
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql8 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+
+    // DECIMALV3(21,3) -> decimalv2 OK
+    sql """ alter table ${tbName} modify column decimalv2k2 DECIMALV2(21,3) key """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql9 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // DECIMALV3(26,8) -> decimalv2
+    sql """ alter table ${tbName} modify column decimalv2v1 DECIMALV2(25,7) sum """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql9_2 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // DECIMALV3(26,8) -> decimalv2, narrow integer
+    sql """ alter table ${tbName} modify column decimalv2v1 DECIMALV2(25,8) sum """
+    max_try_time = 1000
+    while (max_try_time--){
+        String result = getJobState(tbName)
+        if (result == "FINISHED") {
+            sleep(3000)
+            break
+        } else {
+            sleep(1000)
+            if (max_try_time < 1){
+                assertEquals(1,2)
+            }
+        }
+    }
+
+    sql """sync"""
+    qt_sql9_3 """select * from ${tbName} ORDER BY 1,2,3,4;"""
+
+    // restore disable_decimalv2 to old_value
+    sql """ ADMIN SET FRONTEND CONFIG ("disable_decimalv2" = "${old_value1}"); """
+
+    // restore enable_decimal_conversion to old_value
+    sql """ ADMIN SET FRONTEND CONFIG ("enable_decimal_conversion" = "${old_value2}"); """
+}

From 0d0b9d64dd7b57f9879e7536f5b61aadd08cfdb0 Mon Sep 17 00:00:00 2001
From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com>
Date: Wed, 3 Jan 2024 21:27:54 +0800
Subject: [PATCH 299/299] [improve](move-memtable) add move memtable too many
 segments fault injection (#29342)

---
 .../test_too_many_segments_fault_injection.out               | 2 ++
 .../test_too_many_segments_fault_injection.groovy            | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/regression-test/data/fault_injection_p0/test_too_many_segments_fault_injection.out b/regression-test/data/fault_injection_p0/test_too_many_segments_fault_injection.out
index afeab4c41d0905..863b85b658e2f1 100644
--- a/regression-test/data/fault_injection_p0/test_too_many_segments_fault_injection.out
+++ b/regression-test/data/fault_injection_p0/test_too_many_segments_fault_injection.out
@@ -1,3 +1,5 @@
 -- This file is automatically generated. You should know what you did if you want to edit this
 -- !select_default --
 
+-- !select_default --
+
diff --git a/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy
index de7bcac5f10348..e1ea554b976fda 100644
--- a/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy
+++ b/regression-test/suites/fault_injection_p0/test_too_many_segments_fault_injection.groovy
@@ -110,6 +110,7 @@ suite("test_too_many_segments", "nonConcurrent") { // the epic -238 case
                     assertTrue(1 == 2, "load Timeout: $uuid")
                 }
             }
+            logger.info(result[0][7].toString())
             assertTrue(result[0][7].contains("-238")) // EPIC!
 
             result = sql """ show load where label="$uuid" order by createtime desc limit 1; """
@@ -120,5 +121,9 @@ suite("test_too_many_segments", "nonConcurrent") { // the epic -238 case
             GetDebugPoint().disableDebugPointForAllBEs("BetaRowsetWriter._check_segment_number_limit_too_many_segments")
         }
     }
+    
+    sql """ set enable_memtable_on_sink_node=true """
+    runLoadWithTooManySegments()
+    sql """ set enable_memtable_on_sink_node=false """
     runLoadWithTooManySegments()
 }