Skip to content
This repository was archived by the owner on May 6, 2024. It is now read-only.

Commit b6e3d02

Browse files
jikunshangjikunshang
committed
update patch
1 parent e717793 commit b6e3d02

File tree

3 files changed

+176
-3
lines changed

3 files changed

+176
-3
lines changed

‎ci/scripts/prepare_source_code_cicd.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,9 @@
2121

2222
PRESTO_LOCAL_PATH=/workspace/github-workspace/presto
2323
rm -rf ${PRESTO_LOCAL_PATH}
24-
PATCH_NAME=presto-bdtk-67b3bf.patch
24+
PATCH_NAME=presto-bdtk-4df708.patch
2525
PATCH_PATH=$(pwd)/ci/scripts/${PATCH_NAME}
26-
PRESTO_BDTK_COMMIT_ID=67b3bf5251f81131328dbd183685fb50e5a7ac2c
26+
PRESTO_BDTK_COMMIT_ID=4df708c463d79fc4c7b97e21d7136af54251bf50
2727

2828
git clone https://github.com/prestodb/presto.git ${PRESTO_LOCAL_PATH}
2929
pushd ${PRESTO_LOCAL_PATH}

‎ci/scripts/presto-bdtk-4df708.patch‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
diff --git a/presto-native-execution/Makefile b/presto-native-execution/Makefile
2+
index f77cff05fc..04b24adb70 100644
3+
--- a/presto-native-execution/Makefile
4+
+++ b/presto-native-execution/Makefile
5+
@@ -61,7 +61,7 @@ velox-submodule: #: Check out code for velox submodule
6+
7+
submodules: velox-submodule
8+
9+
-cmake: submodules #: Use CMake to create a Makefile build system
10+
+cmake: #: Use CMake to create a Makefile build system
11+
cmake -B "$(BUILD_BASE_DIR)/$(BUILD_DIR)" $(FORCE_COLOR) $(CMAKE_FLAGS) $(EXTRA_CMAKE_FLAGS)
12+
13+
build: #: Build the software based in BUILD_DIR and BUILD_TYPE variables
14+
diff --git a/presto-native-execution/presto_cpp/main/CMakeLists.txt b/presto-native-execution/presto_cpp/main/CMakeLists.txt
15+
index 6591a99374..245c2aef1b 100644
16+
--- a/presto-native-execution/presto_cpp/main/CMakeLists.txt
17+
+++ b/presto-native-execution/presto_cpp/main/CMakeLists.txt
18+
@@ -16,6 +16,8 @@ add_subdirectory(common)
19+
add_subdirectory(thrift)
20+
add_subdirectory(connectors)
21+
22+
+link_dirctory(./lib)
23+
+
24+
add_library(
25+
presto_server_lib
26+
Announcer.cpp
27+
@@ -29,6 +31,19 @@ add_library(
28+
TaskManager.cpp
29+
TaskResource.cpp)
30+
31+
+set(
32+
+ CIDER_VELOX_LIB
33+
+ cider_velox_static
34+
+ velox_substrait_plan_converter
35+
+ velox_duckdb_parser
36+
+ velox_exec_test_lib
37+
+ velox_arrow_bridge
38+
+ cider_static
39+
+ cider_function
40+
+ cider_runtime_function
41+
+ LLVM
42+
+)
43+
+
44+
add_dependencies(presto_server_lib presto_operators presto_protocol
45+
presto_types presto_thrift-cpp2 presto_thrift_extra)
46+
47+
@@ -61,7 +76,9 @@ target_link_libraries(
48+
${ANTLR4_RUNTIME}
49+
${GLOG}
50+
${GFLAGS_LIBRARIES}
51+
- pthread)
52+
+ pthread
53+
+ ${CIDER_VELOX_LIB})
54+
+
55+
56+
if(PRESTO_ENABLE_PARQUET)
57+
target_link_libraries(presto_server_lib velox_dwio_parquet_reader)
58+
diff --git a/presto-native-execution/presto_cpp/main/PrestoServer.cpp b/presto-native-execution/presto_cpp/main/PrestoServer.cpp
59+
index 3631b7d4c0..38e4c5e8af 100644
60+
--- a/presto-native-execution/presto_cpp/main/PrestoServer.cpp
61+
+++ b/presto-native-execution/presto_cpp/main/PrestoServer.cpp
62+
@@ -44,6 +44,7 @@
63+
#include "velox/functions/prestosql/registration/RegistrationFunctions.h"
64+
#include "velox/functions/prestosql/window/WindowFunctionsRegistration.h"
65+
#include "velox/serializers/PrestoSerializer.h"
66+
+#include "BDTK/cpp/src/cider-velox/src/CiderVeloxPluginCtx.h"
67+
68+
#ifdef PRESTO_ENABLE_PARQUET
69+
#include "velox/dwio/parquet/RegisterParquetReader.h" // @manual
70+
@@ -247,6 +248,10 @@ void PrestoServer::run() {
71+
velox::parquet::ParquetReaderType::NATIVE);
72+
#endif
73+
74+
+ if (FLAGS_enable_velox_plugin_BDTK) {
75+
+ facebook::velox::plugin::CiderVeloxPluginCtx::init(SystemConfig::instance()->ciderConfPath());
76+
+ }
77+
+
78+
taskManager_ = std::make_unique<TaskManager>(
79+
systemConfig->values(), nodeConfig->values());
80+
taskManager_->setBaseUri(fmt::format(kBaseUriFormat, address_, servicePort));
81+
diff --git a/presto-native-execution/presto_cpp/main/TaskResource.cpp b/presto-native-execution/presto_cpp/main/TaskResource.cpp
82+
index 545fbd119c..4aada17404 100644
83+
--- a/presto-native-execution/presto_cpp/main/TaskResource.cpp
84+
+++ b/presto-native-execution/presto_cpp/main/TaskResource.cpp
85+
@@ -22,6 +22,9 @@
86+
#include "presto_cpp/presto_protocol/presto_protocol.h"
87+
#include "velox/common/time/Timer.h"
88+
#include "velox/type/tz/TimeZoneMap.h"
89+
+#include "BDTK/cpp/src/cider-velox/src/CiderVeloxPluginCtx.h"
90+
+
91+
+DEFINE_bool(enable_velox_plugin_BDTK, true, "switch to turn on velox plugin using BDTK");
92+
93+
namespace facebook::presto {
94+
95+
@@ -320,6 +323,11 @@ proxygen::RequestHandler* TaskResource::createOrUpdateBatchTask(
96+
shuffleName, std::move(serializedShuffleWriteInfo), pool_.get());
97+
planFragment = converter.toVeloxQueryPlan(
98+
prestoPlan, taskUpdateRequest.tableWriteInfo, taskId);
99+
+ auto rootNode = planFragment.planNode;
100+
+ LOG(INFO) << "Root node is " << rootNode->name();
101+
+ if (FLAGS_enable_velox_plugin_BDTK) {
102+
+ planFragment.planNode = facebook::velox::plugin::CiderVeloxPluginCtx::transformVeloxPlan(rootNode);
103+
+ }
104+
});
105+
}
106+
107+
diff --git a/presto-native-execution/presto_cpp/main/TaskResource.h b/presto-native-execution/presto_cpp/main/TaskResource.h
108+
index 38642cb406..4e5fce17d9 100644
109+
--- a/presto-native-execution/presto_cpp/main/TaskResource.h
110+
+++ b/presto-native-execution/presto_cpp/main/TaskResource.h
111+
@@ -17,6 +17,8 @@
112+
#include "presto_cpp/main/http/HttpServer.h"
113+
#include "velox/common/memory/Memory.h"
114+
115+
+DECLARE_bool(enable_velox_plugin_BDTK);
116+
+
117+
namespace facebook::presto {
118+
119+
class TaskResource {
120+
diff --git a/presto-native-execution/presto_cpp/main/common/Configs.cpp b/presto-native-execution/presto_cpp/main/common/Configs.cpp
121+
index 7ad0f3c7e4..d4ada34861 100644
122+
--- a/presto-native-execution/presto_cpp/main/common/Configs.cpp
123+
+++ b/presto-native-execution/presto_cpp/main/common/Configs.cpp
124+
@@ -53,6 +53,10 @@ std::string SystemConfig::prestoVersion() const {
125+
return requiredProperty(std::string(kPrestoVersion));
126+
}
127+
128+
+std::string SystemConfig::ciderConfPath() const {
129+
+ return requiredProperty(std::string(kCiderConfPath));
130+
+}
131+
+
132+
std::string SystemConfig::discoveryUri() const {
133+
return requiredProperty(std::string(kDiscoveryUri));
134+
}
135+
diff --git a/presto-native-execution/presto_cpp/main/common/Configs.h b/presto-native-execution/presto_cpp/main/common/Configs.h
136+
index e7049ff16d..595d62e24f 100644
137+
--- a/presto-native-execution/presto_cpp/main/common/Configs.h
138+
+++ b/presto-native-execution/presto_cpp/main/common/Configs.h
139+
@@ -76,6 +76,7 @@ class SystemConfig : public ConfigBase {
140+
static constexpr std::string_view kPrestoVersion{"presto.version"};
141+
static constexpr std::string_view kHttpServerHttpPort{
142+
"http-server.http.port"};
143+
+ static constexpr std::string_view kCiderConfPath{"cider.conf_path"};
144+
// This option allows a port closed in TIME_WAIT state to be reused
145+
// immediately upon worker startup. This property is mainly used by batch
146+
// processing. For interactive query, the worker uses a dynamic port upon
147+
@@ -142,6 +143,8 @@ class SystemConfig : public ConfigBase {
148+
149+
std::string discoveryUri() const;
150+
151+
+ std::string ciderConfPath() const;
152+
+
153+
int32_t maxDriversPerTask() const;
154+
155+
int32_t concurrentLifespansPerTask() const;
156+
diff --git a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp
157+
index f9260e0573..a9eb72da0c 100644
158+
--- a/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp
159+
+++ b/presto-native-execution/presto_cpp/main/types/PrestoToVeloxQueryPlan.cpp
160+
@@ -656,9 +656,9 @@ std::shared_ptr<connector::ConnectorTableHandle> toConnectorTableHandle(
161+
if (auto hiveLayout =
162+
std::dynamic_pointer_cast<const protocol::HiveTableLayoutHandle>(
163+
tableHandle.connectorTableLayout)) {
164+
- VELOX_CHECK(
165+
- hiveLayout->pushdownFilterEnabled,
166+
- "Table scan with filter pushdown disabled is not supported");
167+
+ // VELOX_CHECK(
168+
+ // hiveLayout->pushdownFilterEnabled,
169+
+ // "Table scan with filter pushdown disabled is not supported");
170+
171+
for (const auto& entry : hiveLayout->partitionColumns) {
172+
partitionColumns.emplace(entry.name, toColumnHandle(&entry));

‎cpp/src/cider/tests/nextgen/functional/HashJoinOpNextgenTest.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,8 @@ INNER_HASH_JOIN_TEST_UNIT_ARROW_FORMAT(CiderArrowOneToManyRandomNullableJoinTest
252252
// returned null values are incorrect.
253253
// LEFT_HASH_JOIN_TEST_UNIT_ARROW_FORMAT(CiderArrowOneToOneSeqNotNullJoinTest,
254254
// LeftJoinArrowOneToOneSeqNoNullTest, *, int, =) // NOLINT
255-
// LEFT_HASH_JOIN_TEST_UNIT_ARROW_FORMAT(CiderArrowOneToOneSeqNotNullJoinTest, ArrowOneToOneSeqNoNullLeftJoinTest2, *, bigint, =) // NOLINT
255+
// LEFT_HASH_JOIN_TEST_UNIT_ARROW_FORMAT(CiderArrowOneToOneSeqNotNullJoinTest,
256+
// ArrowOneToOneSeqNoNullLeftJoinTest2, *, bigint, =) // NOLINT
256257
// LEFT_HASH_JOIN_TEST_UNIT_ARROW_FORMAT(CiderArrowOneToOneSeqNullableJoinTest,
257258
// LeftJoinArrowOneToOneSeqNoNullableTest, *, int, =) // NOLINT
258259
// LEFT_HASH_JOIN_TEST_UNIT_ARROW_FORMAT(CiderArrowOneToOneSeqNullableJoinTest,

0 commit comments

Comments
 (0)