From cde3b71619e3f1403b4e9437f860cbd3737e3b4d Mon Sep 17 00:00:00 2001 From: Vikram Ahuja Date: Thu, 20 Nov 2025 15:46:56 +0530 Subject: [PATCH] HIVE-29331 Remove code where MAPRED_MIN_SPLIT_SIZE value is deciding split size in case of Tez --- .../hive/ql/exec/tez/HiveSplitGenerator.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java index 5222e3b868d3..9a9102e7d595 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java @@ -303,17 +303,14 @@ public List initialize() throws Exception { int availableSlots = getAvailableSlotsCalculator().getAvailableSlots(); - if (HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPRED_MIN_SPLIT_SIZE, 1) <= 1) { - // broken configuration from mapred-default.xml - final long blockSize = conf.getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, - DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); - final long minGrouping = conf.getLong( - TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE, - TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT); - final long preferredSplitSize = Math.min(blockSize / 2, minGrouping); - HiveConf.setLongVar(jobConf, HiveConf.ConfVars.MAPRED_MIN_SPLIT_SIZE, preferredSplitSize); - LOG.info("The preferred split size is " + preferredSplitSize); - } + final long blockSize = conf.getLongBytes(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); + final long minGrouping = conf.getLong( + TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE, + TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT); + final long preferredSplitSize = Math.min(blockSize / 2, minGrouping); + HiveConf.setLongVar(jobConf, HiveConf.ConfVars.MAPRED_MIN_SPLIT_SIZE, preferredSplitSize); + LOG.info("The preferred split size is " + preferredSplitSize); float waves; // Create the un-grouped splits