Skip to content

Commit 94c39e8

Browse files
hnwyllmmhnwyllmm
authored
Thread pool (oceanbase#332)
### What problem were solved in this pull request? Issue Number: close oceanbase#322 oceanbase#301 oceanbase#143 oceanbase#47 ref oceanbase#138 Problem: 当前使用的seda处理框架和libevent消息处理模式不适用于SQL处理模型。 SQL 请求处理模式是来一个请求应答后才能接收新的请求。但是当前收到某个连接的消息后在返回应答之前，可以接收新的请求，导致可以同时处理同一个连接的多个请求，出现并发错误。 另外，同一个SQL请求，需要有一个统一的处理上下文，不能在线程间切来切去。而seda将处理请求分为多个stage，会在不同的线程间切换。 ### What is changed and how it works? 新增线程模型概念，用来处理连接上的请求。设计文档参考 miniob-thread-model.md。 对于新连接监听，放在主线程来做。接收到新连接时，交给线程模型来处理，可以是一个连接一个线程，也可以是一个线程池统一管理。 不管使用哪种模型，在处理一个SQL结束之前，不会去监听该连接上是否有新的消息到达，因此也不会出现一个连接上多个请求同时处理的情况。 ### Other information 本设计参考了Java的线程池和Percona/MariaDB的线程池（虽然没有实现）设计
1 parent 75f9cf8 commit 94c39e8

File tree

80 files changed

+2591
-4980
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

80 files changed

+2591
-4980
lines changed

‎.github/workflows/test.yml

Lines changed: 52 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -27,34 +27,55 @@ jobs:
2727
2828
# sysbench cannot work property on this platform.
2929
# I found that sysbench would send more request before receiving last response
30-
# sysbench-test:
31-
# runs-on: ubuntu-latest
32-
33-
34-
# steps:
35-
# - name: Checkout repository and submodules
36-
# uses: actions/checkout@v2
37-
38-
# - name: install sysbench and mariadb-client
39-
# shell: bash
40-
# run: |
41-
# curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh -o script.deb.sh
42-
# sudo bash script.deb.sh
43-
# sudo apt -y install sysbench mariadb-client
44-
45-
# - name: start server
46-
# shell: bash
47-
# run: |
48-
# sudo bash build.sh init
49-
# bash build.sh -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF
50-
# nohup ./build_debug/bin/observer -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc &
51-
# sleep 10 && echo "wake up"
52-
# mysql --version
53-
# mysql -S /tmp/miniob.sock -e "show tables"
54-
55-
# - name: sysbench test
56-
# shell: bash
57-
# run: |
58-
# cd test/sysbench
59-
# sysbench --mysql-socket=/tmp/miniob.sock --threads=10 miniob_insert prepare
60-
# sysbench --mysql-socket=/tmp/miniob.sock --threads=10 miniob_insert run
30+
sysbench-test:
31+
strategy:
32+
matrix:
33+
thread_model: ['one-thread-per-connection', 'java-thread-pool']
34+
test_case: ['miniob_insert', 'miniob_delete', 'miniob_select']
35+
36+
runs-on: ubuntu-latest
37+
steps:
38+
- name: Checkout repository and submodules
39+
uses: actions/checkout@v2
40+
41+
- name: install sysbench and mariadb-client
42+
shell: bash
43+
run: |
44+
curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh -o script.deb.sh
45+
sudo bash script.deb.sh
46+
sudo apt -y install sysbench mariadb-client
47+
48+
- name: start server
49+
shell: bash
50+
run: |
51+
sudo bash build.sh init
52+
bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF
53+
nohup ./build_release/bin/observer -T ${{ matrix.thread_model }} -s /tmp/miniob.sock -f etc/observer.ini -P mysql -t mvcc &
54+
sleep 10 && echo "wake up"
55+
mysql --version
56+
mysql -S /tmp/miniob.sock -e "show tables"
57+
58+
- name: sysbench test
59+
shell: bash
60+
run: |
61+
cd test/sysbench
62+
sysbench --mysql-socket=/tmp/miniob.sock --threads=10 ${{ matrix.test_case }} prepare
63+
sysbench --mysql-socket=/tmp/miniob.sock --threads=10 ${{ matrix.test_case }} run
64+
65+
benchmark-test:
66+
runs-on: ubuntu-latest
67+
steps:
68+
- name: Checkout repository and submodules
69+
uses: actions/checkout@v2
70+
71+
- name: build observer and benchmark
72+
shell: bash
73+
run: |
74+
sudo bash build.sh init
75+
bash build.sh release -DCONCURRENCY=ON -DWITH_UNIT_TESTS=OFF -DWITH_BENCHMARK=ON
76+
77+
- name: testing
78+
shell: bash
79+
run: |
80+
cd build_release/bin/
81+
for file in `find ./ -name "*_concurrency_test" -executable`; do $file; if [ $? -ne 0 ]; then exit 1; fi; done

‎CMakeLists.txt

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ OPTION(ENABLE_ASAN "Enable build with address sanitizer" ON)
1414
OPTION(ENABLE_TSAN "Build with thread sanitizer" OFF)
1515
OPTION(ENABLE_UBSAN "Build with undefined behavior sanitizer" OFF)
1616
OPTION(WITH_UNIT_TESTS "Compile miniob with unit tests" ON)
17+
OPTION(WITH_BENCHMARK "Compile benchmark" OFF)
18+
OPTION(ENABLE_COVERAGE "Enable unittest coverage" OFF)
1719
OPTION(CONCURRENCY "Support concurrency operations" OFF)
1820
OPTION(STATIC_STDLIB "Link std library static or dynamic, such as libgcc, libstdc++, libasan" OFF)
1921

@@ -63,6 +65,7 @@ IF ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND ${STATIC_STDLIB})
6365
ENDIF()
6466

6567
IF (ENABLE_ASAN)
68+
MESSAGE(STATUS "Instrumenting with Address Sanitizer")
6669
SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -fno-omit-frame-pointer -fsanitize=address -fsanitize-address-use-after-scope")
6770
IF ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND ${STATIC_STDLIB})
6871
ADD_LINK_OPTIONS(-static-libasan)
@@ -87,18 +90,18 @@ IF (ENABLE_TSAN)
8790
ENDIF (ENABLE_TSAN)
8891

8992
IF (ENABLE_UBSAN)
90-
# Only success on Mac Clang
91-
MESSAGE(STATUS "Instrumenting with Undefined Behavior Sanitizer")
92-
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-omit-frame-pointer")
93-
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=undefined")
94-
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=implicit-conversion")
95-
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=implicit-integer-truncation")
96-
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=integer")
97-
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=nullability")
98-
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=vptr")
99-
SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} ${UBSAN_FLAGS}")
100-
SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${UBSAN_FLAGS}")
101-
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${UBSAN_FLAGS}")
93+
# Only success on Mac Clang
94+
MESSAGE(STATUS "Instrumenting with Undefined Behavior Sanitizer")
95+
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fno-omit-frame-pointer")
96+
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=undefined")
97+
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=implicit-conversion")
98+
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=implicit-integer-truncation")
99+
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=integer")
100+
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=nullability")
101+
SET(UBSAN_FLAGS "${UBSAN_FLAGS} -fsanitize=vptr")
102+
SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} ${UBSAN_FLAGS}")
103+
SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${UBSAN_FLAGS}")
104+
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${UBSAN_FLAGS}")
102105
ENDIF (ENABLE_UBSAN)
103106

104107
IF (CMAKE_INSTALL_PREFIX)
@@ -127,7 +130,9 @@ ENDIF ()
127130
INCLUDE_DIRECTORIES(. ${PROJECT_SOURCE_DIR}/deps /usr/local/include)
128131

129132
IF(WITH_UNIT_TESTS)
130-
SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -fprofile-arcs -ftest-coverage")
133+
IF (ENABLE_COVERAGE)
134+
SET(CMAKE_COMMON_FLAGS "${CMAKE_COMMON_FLAGS} -fprofile-arcs -ftest-coverage")
135+
ENDIF (ENABLE_COVERAGE)
131136
enable_testing()
132137
ENDIF(WITH_UNIT_TESTS)
133138

@@ -140,9 +145,12 @@ ADD_SUBDIRECTORY(deps)
140145
ADD_SUBDIRECTORY(src/obclient)
141146
ADD_SUBDIRECTORY(src/observer)
142147
ADD_SUBDIRECTORY(test/perf)
143-
ADD_SUBDIRECTORY(benchmark)
144148
ADD_SUBDIRECTORY(tools)
145149

150+
IF (WITH_BENCHMARK)
151+
ADD_SUBDIRECTORY(benchmark)
152+
ENDIF(WITH_BENCHMARK)
153+
146154
IF(WITH_UNIT_TESTS)
147155
ADD_SUBDIRECTORY(unittest)
148156
ENDIF(WITH_UNIT_TESTS)

‎benchmark/bplus_tree_concurrency_test.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ See the Mulan PSL v2 for more details. */
1616
#include <stdexcept>
1717

1818
#include "common/log/log.h"
19-
#include "integer_generator.h"
19+
#include "common/math/integer_generator.h"
2020
#include "storage/buffer/disk_buffer_pool.h"
2121
#include "storage/index/bplus_tree.h"
2222

‎benchmark/record_manager_concurrency_test.cpp

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ See the Mulan PSL v2 for more details. */
1818
#include <stdexcept>
1919

2020
#include "common/log/log.h"
21-
#include "integer_generator.h"
21+
#include "common/math/integer_generator.h"
2222
#include "storage/buffer/disk_buffer_pool.h"
2323
#include "storage/common/condition_filter.h"
2424
#include "storage/record/record_manager.h"
@@ -87,7 +87,7 @@ class BenchmarkBase : public Fixture
8787

8888
string log_name = this->Name() + ".log";
8989
string record_filename = this->record_filename();
90-
LoggerFactory::init_default(log_name.c_str(), LOG_LEVEL_TRACE);
90+
LoggerFactory::init_default(log_name.c_str(), LOG_LEVEL_INFO);
9191

9292
std::call_once(init_bpm_flag, []() { BufferPoolManager::set_instance(&bpm); });
9393

@@ -110,8 +110,8 @@ class BenchmarkBase : public Fixture
110110
LOG_WARN("failed to init record file handler. rc=%s", strrc(rc));
111111
throw runtime_error("failed to init record file handler");
112112
}
113-
LOG_INFO(
114-
"test %s setup done. threads=%d, thread index=%d", this->Name().c_str(), state.threads(), state.thread_index());
113+
LOG_INFO("test %s setup done. threads=%d, thread index=%d",
114+
this->Name().c_str(), state.threads(), state.thread_index());
115115
}
116116

117117
virtual void TearDown(const State &state)
@@ -156,9 +156,9 @@ class BenchmarkBase : public Fixture
156156

157157
uint32_t GetRangeMax(const State &state) const
158158
{
159-
uint32_t max = static_cast<uint32_t>(state.range(0) * 3);
159+
int32_t max = static_cast<int32_t>(state.range(0) * 3);
160160
if (max <= 0) {
161-
max = (1 << 31);
161+
max = INT32_MAX - 1;
162162
}
163163
return max;
164164
}
@@ -261,24 +261,33 @@ class DeletionBenchmark : public BenchmarkBase
261261

262262
void SetUp(const State &state) override
263263
{
264+
BenchmarkBase::SetUp(state);
265+
264266
if (0 != state.thread_index()) {
267+
while (!setup_done_) {
268+
this_thread::sleep_for(chrono::milliseconds(100));
269+
}
265270
return;
266271
}
267272

268-
BenchmarkBase::SetUp(state);
269-
270273
uint32_t max = GetRangeMax(state);
271274
ASSERT(max > 0, "invalid argument count. %ld", state.range(0));
272275
FillUp(0, max, rids_);
276+
ASSERT(rids_.size() > 0, "invalid argument count. %ld", rids_.size());
277+
setup_done_ = true;
273278
}
274279

275280
protected:
276-
vector<RID> rids_;
281+
// 从实际测试情况看，每个线程都会执行setup，但是它们操作的对象都是同一个
282+
// 但是每个线程set up结束后，就会执行测试了。如果不等待的话，就会导致有些
283+
// 线程访问的数据不是想要的结果
284+
volatile bool setup_done_ = false;
285+
vector<RID> rids_;
277286
};
278287

279288
BENCHMARK_DEFINE_F(DeletionBenchmark, Deletion)(State &state)
280289
{
281-
IntegerGenerator generator(0, static_cast<int>(rids_.size()));
290+
IntegerGenerator generator(0, static_cast<int>(rids_.size() - 1));
282291
Stat stat;
283292

284293
for (auto _ : state) {

‎build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ function do_init
9696
cd ${TOPDIR}/deps/3rd/benchmark && \
9797
mkdir -p build && \
9898
cd build && \
99-
${CMAKE_COMMAND} .. -DBENCHMARK_ENABLE_TESTING=OFF -DBENCHMARK_INSTALL_DOCS=OFF -DBENCHMARK_ENABLE_GTEST_TESTS=OFF -DBENCHMARK_USE_BUNDLED_GTEST=OFF -DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF && \
99+
${CMAKE_COMMAND} .. -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBENCHMARK_ENABLE_TESTING=OFF -DBENCHMARK_INSTALL_DOCS=OFF -DBENCHMARK_ENABLE_GTEST_TESTS=OFF -DBENCHMARK_USE_BUNDLED_GTEST=OFF -DBENCHMARK_ENABLE_ASSEMBLY_TESTS=OFF && \
100100
${MAKE_COMMAND} -j4 && \
101101
${MAKE_COMMAND} install
102102

‎deps/common/log/log.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -333,6 +333,15 @@ int Log::out(const LOG_LEVEL console_level, const LOG_LEVEL log_level, T &msg)
333333

334334
#endif // ASSERT
335335

336+
#ifndef TRACE
337+
#ifdef DEBUG
338+
#define TRACE(format, ...) LOG_TRACE(format, ##__VA_ARGS__)
339+
#else // DEBUG
340+
#define TRACE(...)
341+
#endif // DEBUG
342+
343+
#endif // TRACE
344+
336345
#define SYS_OUTPUT_FILE_POS ", File:" << __FILE__ << ", line:" << __LINE__ << ",function:" << __FUNCTION__
337346
#define SYS_OUTPUT_ERROR ",error:" << errno << ":" << strerror(errno)
338347

‎benchmark/integer_generator.h renamed to ‎deps/common/math/integer_generator.h

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,29 @@ See the Mulan PSL v2 for more details. */
1111
//
1212
// Created by Wangyunlai on 2023/05/04
1313
//
14+
15+
#pragma once
16+
1417
#include <random>
1518

19+
namespace common {
20+
1621
class IntegerGenerator
1722
{
1823
public:
1924
IntegerGenerator(int min, int max) : distrib_(min, max) {}
2025

26+
IntegerGenerator(const IntegerGenerator &other) = delete;
27+
IntegerGenerator(IntegerGenerator &&) = delete;
28+
IntegerGenerator &operator=(const IntegerGenerator &) = delete;
29+
2130
int next() { return distrib_(rd_); }
31+
int min() const { return distrib_.min(); }
32+
int max() const { return distrib_.max(); }
2233

2334
private:
2435
std::random_device rd_;
2536
std::uniform_int_distribution<> distrib_;
26-
};
37+
};
38+
39+
} // namespace common

‎deps/common/os/process_param.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,15 @@ class ProcessParam
7272
}
7373
const std::string &trx_kit_name() const { return trx_kit_name_; }
7474

75+
void set_thread_handling_name(const char *thread_handling_name)
76+
{
77+
if (thread_handling_name) {
78+
thread_handling_name_ = thread_handling_name;
79+
}
80+
}
81+
82+
const std::string &thread_handling_name() const { return thread_handling_name_; }
83+
7584
void set_buffer_pool_memory_size(int bytes) { buffer_pool_memory_size_ = bytes; }
7685

7786
int buffer_pool_memory_size() const { return buffer_pool_memory_size_; }
@@ -87,6 +96,7 @@ class ProcessParam
8796
std::string unix_socket_path_;
8897
std::string protocol_;
8998
std::string trx_kit_name_;
99+
std::string thread_handling_name_;
90100
int buffer_pool_memory_size_ = -1;
91101
};
92102

‎deps/common/os/signal.cpp

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ See the Mulan PSL v2 for more details. */
1717
#include "pthread.h"
1818
namespace common {
1919

20-
void setSignalHandler(int sig, sighandler_t func)
20+
void set_signal_handler(int sig, sighandler_t func)
2121
{
2222
struct sigaction newsa, oldsa;
2323
sigemptyset(&newsa.sa_mask);
@@ -33,16 +33,16 @@ void setSignalHandler(int sig, sighandler_t func)
3333
/*
3434
** Set Singal handling Fucntion
3535
*/
36-
void setSignalHandler(sighandler_t func)
36+
void set_signal_handler(sighandler_t func)
3737
{
38-
setSignalHandler(SIGQUIT, func);
39-
setSignalHandler(SIGINT, func);
40-
setSignalHandler(SIGHUP, func);
41-
setSignalHandler(SIGTERM, func);
38+
set_signal_handler(SIGQUIT, func);
39+
set_signal_handler(SIGINT, func);
40+
set_signal_handler(SIGHUP, func);
41+
set_signal_handler(SIGTERM, func);
4242
signal(SIGPIPE, SIG_IGN);
4343
}
4444

45-
void blockDefaultSignals(sigset_t *signal_set, sigset_t *old_set)
45+
void block_default_signals(sigset_t *signal_set, sigset_t *old_set)
4646
{
4747
sigemptyset(signal_set);
4848
#ifndef DEBUG
@@ -54,7 +54,7 @@ void blockDefaultSignals(sigset_t *signal_set, sigset_t *old_set)
5454
pthread_sigmask(SIG_BLOCK, signal_set, old_set);
5555
}
5656

57-
void unBlockDefaultSignals(sigset_t *signal_set, sigset_t *old_set)
57+
void unblock_default_signals(sigset_t *signal_set, sigset_t *old_set)
5858
{
5959
sigemptyset(signal_set);
6060
#ifndef DEBUG
@@ -65,7 +65,7 @@ void unBlockDefaultSignals(sigset_t *signal_set, sigset_t *old_set)
6565
pthread_sigmask(SIG_UNBLOCK, signal_set, old_set);
6666
}
6767

68-
void *waitForSignals(void *args)
68+
void *wait_for_signals(void *args)
6969
{
7070
LOG_INFO("Start thread to wait signals.");
7171
sigset_t *signal_set = (sigset_t *)args;
@@ -81,7 +81,7 @@ void *waitForSignals(void *args)
8181
return NULL;
8282
}
8383

84-
void startWaitForSignals(sigset_t *signal_set)
84+
void start_wait_for_signals(sigset_t *signal_set)
8585
{
8686
pthread_t pThread;
8787
pthread_attr_t pThreadAttrs;
@@ -90,6 +90,6 @@ void startWaitForSignals(sigset_t *signal_set)
9090
pthread_attr_init(&pThreadAttrs);
9191
pthread_attr_setdetachstate(&pThreadAttrs, PTHREAD_CREATE_DETACHED);
9292

93-
pthread_create(&pThread, &pThreadAttrs, waitForSignals, (void *)signal_set);
93+
pthread_create(&pThread, &pThreadAttrs, wait_for_signals, (void *)signal_set);
9494
}
9595
} // namespace common

0 commit comments

Comments
 (0)