From 45d90cf98bae182a2779800056f6f5a797a5b4cc Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Mon, 26 Jul 2021 04:45:21 -0400 Subject: [PATCH 01/16] Refactor support for SHOW commands in DESCRIBE. --- src/include/traffic_cop/traffic_cop.h | 14 +++---- .../postgres/postgres_network_commands.cpp | 40 ++++++++++++++++++- src/network/postgres/postgres_packet_util.cpp | 2 +- src/traffic_cop/traffic_cop.cpp | 5 +-- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/src/include/traffic_cop/traffic_cop.h b/src/include/traffic_cop/traffic_cop.h index 7b730dd6f4..54fea5b7ab 100644 --- a/src/include/traffic_cop/traffic_cop.h +++ b/src/include/traffic_cop/traffic_cop.h @@ -190,15 +190,15 @@ class TrafficCop { common::ManagedPointer statement) const; /** - * Contains the logic to handle SHOW statements. Currently a hack to only support SHOW TRANSACTION ISOLATION LEVEL - * @param connection_ctx The context to be used to access the internal txn. - * @param statement The show statement to be executed. - * @param out Packet writer for writing results. - * @return The result of the operation. + * Contains the logic to handle SHOW statements. + * @param connection_ctx The context to be used to access the internal txn. + * @param out The packet writer to return results. + * @param statement The statement to be executed. + * @return The result of the operation. */ TrafficCopResult ExecuteShowStatement(common::ManagedPointer connection_ctx, - common::ManagedPointer statement, - common::ManagedPointer out) const; + common::ManagedPointer out, + common::ManagedPointer statement) const; /** * Contains the logic to reason about CREATE execution. diff --git a/src/network/postgres/postgres_network_commands.cpp b/src/network/postgres/postgres_network_commands.cpp index 3d7442d94f..ca03e15564 100644 --- a/src/network/postgres/postgres_network_commands.cpp +++ b/src/network/postgres/postgres_network_commands.cpp @@ -10,8 +10,34 @@ #include "network/postgres/postgres_packet_util.h" #include "network/postgres/postgres_protocol_interpreter.h" #include "network/postgres/statement.h" +#include "parser/variable_show_statement.h" #include "traffic_cop/traffic_cop.h" +namespace noisepage::network { +/** + * Write the row description for a SHOW statement. + * + * @param out Packet writer for writing results. + * @param statement The SHOW statement to be described. + */ +static void WriteShowRowDescription(const common::ManagedPointer out, + const common::ManagedPointer statement) { + // TODO(WAN): This code exists because the SHOW statement does not go through the optimizer and therefore does not + // have a corresponding OutputSchema to go through the usual (SELECT) code path in DescribeCommand. + const auto &show_stmt UNUSED_ATTRIBUTE = + statement->RootStatement().CastManagedPointerTo(); + + const std::string ¶m_name = show_stmt->GetName(); + auto expr = std::make_unique(execution::sql::SqlTypeId::Varchar); + expr->SetAlias(parser::AliasType(param_name)); + std::vector cols; + cols.emplace_back(param_name, execution::sql::SqlTypeId::Varchar, std::move(expr)); + + out->WriteRowDescription(cols, {network::FieldFormat::text}); +} + +} // namespace noisepage::network + namespace noisepage::network { static Transition FinishSimpleQueryCommand(const common::ManagedPointer out, @@ -64,6 +90,8 @@ static void ExecutePortal(const common::ManagedPointerExecuteDropStatement(connection_ctx, physical_plan, query_type); } else if (query_type == network::QueryType::QUERY_EXPLAIN) { result = t_cop->ExecuteExplainStatement(connection_ctx, out, portal); + } else if (query_type == network::QueryType::QUERY_SHOW) { + result = t_cop->ExecuteShowStatement(connection_ctx, out, portal->GetStatement()); } if (result.type_ == trafficcop::ResultType::COMPLETE) { @@ -148,7 +176,7 @@ Transition SimpleQueryCommand::Exec(const common::ManagedPointerExecuteShowStatement(connection, common::ManagedPointer(statement), out); + auto show_result = t_cop->ExecuteShowStatement(connection, out, common::ManagedPointer(statement)); NOISEPAGE_ASSERT(show_result.type_ == trafficcop::ResultType::COMPLETE, "TODO this should be fixed to handle failure."); out->WriteCommandComplete(network::QueryType::QUERY_SHOW, 0); @@ -195,6 +223,8 @@ Transition SimpleQueryCommand::Exec(const common::ManagedPointerResultFormats()); } else if (query_type == network::QueryType::QUERY_EXPLAIN) { out->WriteExplainRowDescription(); + } else if (query_type == network::QueryType::QUERY_SHOW) { + WriteShowRowDescription(out, portal->GetStatement()); } ExecutePortal(connection, common::ManagedPointer(portal), out, t_cop, @@ -468,6 +498,8 @@ Transition DescribeCommand::Exec(const common::ManagedPointerResultFormats()); } else if (portal->GetStatement()->GetQueryType() == network::QueryType::QUERY_EXPLAIN) { out->WriteExplainRowDescription(); + } else if (portal->GetStatement()->GetQueryType() == network::QueryType::QUERY_SHOW) { + WriteShowRowDescription(out, portal->GetStatement()); } else { out->WriteNoData(); } @@ -547,7 +579,11 @@ Transition ExecuteCommand::Exec(const common::ManagedPointerExecuteShowStatement(connection, out, portal->GetStatement()); + NOISEPAGE_ASSERT(show_result.type_ == trafficcop::ResultType::COMPLETE, + "TODO(WAN) this should be fixed to handle failure."); + out->WriteCommandComplete(query_type, std::get(show_result.extra_)); + return Transition::PROCEED; } // This logic relies on ordering of values in the enum's definition and is documented there as well. diff --git a/src/network/postgres/postgres_packet_util.cpp b/src/network/postgres/postgres_packet_util.cpp index 4b95790da7..71950e1e83 100644 --- a/src/network/postgres/postgres_packet_util.cpp +++ b/src/network/postgres/postgres_packet_util.cpp @@ -99,7 +99,7 @@ parser::ConstantValueExpression PostgresPacketUtil::TextValueToInternalValue( } default: // TODO(Matt): Note that not all types are handled yet. Add them as we support them. - UNREACHABLE("Unsupported type for parameter."); + UNREACHABLE(fmt::format("Unsupported type for parameter {}.", type).c_str()); } } diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index 4dbf599f82..e446ab87a8 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -224,8 +224,8 @@ TrafficCopResult TrafficCop::ExecuteSetStatement(common::ManagedPointer connection_ctx, - common::ManagedPointer statement, - const common::ManagedPointer out) const { + common::ManagedPointer out, + common::ManagedPointer statement) const { NOISEPAGE_ASSERT(connection_ctx->TransactionState() == network::NetworkTransactionStateType::IDLE, "This is a non-transactional operation and we should not be in a transaction."); NOISEPAGE_ASSERT(statement->GetQueryType() == network::QueryType::QUERY_SHOW, @@ -245,7 +245,6 @@ TrafficCopResult TrafficCop::ExecuteShowStatement( cols.emplace_back(param_name, execution::sql::SqlTypeId::Varchar, std::move(expr)); execution::sql::StringVal result{param_val.c_str()}; - out->WriteRowDescription(cols, {network::FieldFormat::text}); out->WriteDataRow(reinterpret_cast(&result), cols, {network::FieldFormat::text}); return {ResultType::COMPLETE, 0u}; } From 4ce441189302a175f3daf2d77ee14c2559b9e63a Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Mon, 26 Jul 2021 05:56:55 -0400 Subject: [PATCH 02/16] Adapt CI framework to use oltpbench_tim. --- script/testing/oltpbench/constants.py | 23 +++++++++------------- script/testing/oltpbench/test_case_oltp.py | 16 +++++++-------- script/testing/oltpbench/test_oltpbench.py | 10 ++++++++-- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/script/testing/oltpbench/constants.py b/script/testing/oltpbench/constants.py index 3d6fa97849..f0203e9f99 100755 --- a/script/testing/oltpbench/constants.py +++ b/script/testing/oltpbench/constants.py @@ -3,11 +3,15 @@ from ..util.constants import DIR_TMP # git settings for OLTPBench. +OLTPBENCH_VERSION = "oltpbench2-20.1.3-SNAPSHOT" OLTPBENCH_GIT_URL = "https://github.com/oltpbenchmark/oltpbench.git" OLTPBENCH_GIT_LOCAL_PATH = os.path.join(DIR_TMP, "oltpbench") +OLTPBENCH_GIT_TARGET_PATH = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "target") +OLTPBENCH_GIT_FINAL_PATH = os.path.join(OLTPBENCH_GIT_TARGET_PATH, OLTPBENCH_VERSION) OLTPBENCH_GIT_CLEAN_COMMAND = "rm -rf {}".format(OLTPBENCH_GIT_LOCAL_PATH) -OLTPBENCH_GIT_CLONE_COMMAND = "git clone --depth 1 {} {}".format(OLTPBENCH_GIT_URL, - OLTPBENCH_GIT_LOCAL_PATH) +OLTPBENCH_GIT_CLONE_COMMAND = "git clone --single-branch --branch oltpbench_tim --depth 1 {} {}".format( + OLTPBENCH_GIT_URL, + OLTPBENCH_GIT_LOCAL_PATH) # OLTPBench default settings. OLTPBENCH_DEFAULT_TIME = 30 @@ -21,7 +25,7 @@ OLTPBENCH_DEFAULT_DBTYPE = "noisepage" OLTPBENCH_DEFAULT_DRIVER = "org.postgresql.Driver" OLTPBENCH_DEFAULT_RATE = "unlimited" -OLTPBENCH_DEFAULT_BIN = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "oltpbenchmark") +OLTPBENCH_DEFAULT_BIN = "java -jar oltpbench2.jar " OLTPBENCH_DEFAULT_DATABASE_RESTART = True OLTPBENCH_DEFAULT_DATABASE_CREATE = True OLTPBENCH_DEFAULT_DATABASE_LOAD = True @@ -30,17 +34,8 @@ OLTPBENCH_DEFAULT_WAL_ENABLE = True OLTPBENCH_DEFAULT_CONTINUE_ON_ERROR = False -OLTPBENCH_DIR_CONFIG = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "config") -OLTPBENCH_DIR_TEST_RESULT = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "results") - -# ant commands for invoking OLTPBench. -OLTPBENCH_ANT_BUILD_FILE = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "build.xml") -OLTPBENCH_ANT_COMMANDS = [ - "ant bootstrap -buildfile {}".format(OLTPBENCH_ANT_BUILD_FILE), - "ant resolve -buildfile {}".format(OLTPBENCH_ANT_BUILD_FILE), - "ant clean -buildfile {}".format(OLTPBENCH_ANT_BUILD_FILE), - "ant build -buildfile {}".format(OLTPBENCH_ANT_BUILD_FILE), -] +OLTPBENCH_DIR_CONFIG = os.path.join(OLTPBENCH_GIT_FINAL_PATH, "config", "noisepage") +OLTPBENCH_DIR_TEST_RESULT = os.path.join(OLTPBENCH_GIT_FINAL_PATH, "results") # API endpoints for Performance Storage Service # Each pair represents different environment. One could choose where the benchmark testing result will be uploaded to diff --git a/script/testing/oltpbench/test_case_oltp.py b/script/testing/oltpbench/test_case_oltp.py index 74bcba480c..30ec0632fa 100644 --- a/script/testing/oltpbench/test_case_oltp.py +++ b/script/testing/oltpbench/test_case_oltp.py @@ -79,15 +79,15 @@ def _init_test_case(self): self.test_output_file = os.path.join(self.test_result_dir, "oltpbench.log") - # oltpbench historgrams results - json format + # oltpbench histograms results - json format self.test_histograms_json_file = self.args.get("test_json_histograms") if not self.test_histograms_json_file: self.test_histograms_json_file = "oltp_histograms_" + self.filename_suffix + ".json" self.test_histogram_path = os.path.join( - constants.OLTPBENCH_GIT_LOCAL_PATH, self.test_histograms_json_file) + constants.OLTPBENCH_GIT_FINAL_PATH, self.test_histograms_json_file) # oltpbench initiate database and load data - self.oltp_flag = "--histograms --execute={EXECUTE} -s {BUCKETS}".format( + self.oltp_flag = "--execute={EXECUTE} -s {BUCKETS}".format( EXECUTE=self.db_execute, BUCKETS=self.buckets) # oltpbench test command @@ -98,7 +98,7 @@ def _init_test_case(self): XML=self.xml_config, FLAGS=self.oltp_flag, HISTOGRAMS=self.test_histogram_path) - self.test_command_cwd = constants.OLTPBENCH_GIT_LOCAL_PATH + self.test_command_cwd = constants.OLTPBENCH_GIT_FINAL_PATH def run_pre_test(self): self._config_xml_file() @@ -149,9 +149,9 @@ def _get_db_url(self): def _config_xml_file(self): xml = ElementTree.parse(self.xml_template) root = xml.getroot() - root.find("dbtype").text = constants.OLTPBENCH_DEFAULT_DBTYPE + root.find("type").text = constants.OLTPBENCH_DEFAULT_DBTYPE root.find("driver").text = constants.OLTPBENCH_DEFAULT_DRIVER - root.find("DBUrl").text = self._get_db_url() + root.find("url").text = self._get_db_url() root.find("username").text = constants.OLTPBENCH_DEFAULT_USERNAME root.find("password").text = constants.OLTPBENCH_DEFAULT_PASSWORD root.find("isolation").text = str(self.transaction_isolation) @@ -199,9 +199,7 @@ def _validate_result(self): with open(self.test_histogram_path) as oltp_result_file: test_result = json.load(oltp_result_file) unexpected_result = test_result.get("unexpected", {}).get("HISTOGRAM") - if unexpected_result and unexpected_result.keys(): + if unexpected_result: for test in unexpected_result.keys(): if unexpected_result[test] != 0: raise RuntimeError(str(unexpected_result)) - else: - raise RuntimeError(str(unexpected_result)) diff --git a/script/testing/oltpbench/test_oltpbench.py b/script/testing/oltpbench/test_oltpbench.py index 5217ed38ea..d9430f2865 100644 --- a/script/testing/oltpbench/test_oltpbench.py +++ b/script/testing/oltpbench/test_oltpbench.py @@ -2,6 +2,8 @@ from ..util.test_server import TestServer from . import constants +import os + class TestOLTPBench(TestServer): """ @@ -39,5 +41,9 @@ def _build_oltpbench(self): Raises an exception if anything goes wrong. Assumes that _download_oltpbench() has already been run. """ - for command in constants.OLTPBENCH_ANT_COMMANDS: - expect_command(command) + old_dir = os.getcwd() + os.chdir(constants.OLTPBENCH_GIT_LOCAL_PATH) + expect_command("./mvnw package") + os.chdir(constants.OLTPBENCH_GIT_TARGET_PATH) + expect_command(f"tar xvzf {constants.OLTPBENCH_VERSION}.tgz") + os.chdir(old_dir) From 2191d557702e3e31976dbd8e67b9a2cea92c2177 Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Mon, 26 Jul 2021 06:00:08 -0400 Subject: [PATCH 03/16] Format... --- src/traffic_cop/traffic_cop.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index e446ab87a8..c8c2234a70 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -222,10 +222,9 @@ TrafficCopResult TrafficCop::ExecuteSetStatement(common::ManagedPointer connection_ctx, - common::ManagedPointer out, - common::ManagedPointer statement) const { +TrafficCopResult TrafficCop::ExecuteShowStatement(common::ManagedPointer connection_ctx, + common::ManagedPointer out, + common::ManagedPointer statement) const { NOISEPAGE_ASSERT(connection_ctx->TransactionState() == network::NetworkTransactionStateType::IDLE, "This is a non-transactional operation and we should not be in a transaction."); NOISEPAGE_ASSERT(statement->GetQueryType() == network::QueryType::QUERY_SHOW, From e06c2ffb1782da0f0e319d2f4813cab5743ee1bd Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Mon, 26 Jul 2021 06:14:13 -0400 Subject: [PATCH 04/16] Add format header... --- src/network/postgres/postgres_packet_util.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/network/postgres/postgres_packet_util.cpp b/src/network/postgres/postgres_packet_util.cpp index 71950e1e83..1b2bb94b14 100644 --- a/src/network/postgres/postgres_packet_util.cpp +++ b/src/network/postgres/postgres_packet_util.cpp @@ -10,6 +10,7 @@ #include "network/postgres/postgres_defs.h" #include "network/postgres/postgres_protocol_util.h" #include "parser/expression/constant_value_expression.h" +#include "spdlog/fmt/fmt.h" namespace noisepage::network { From ee64eb3521690060c5b83d75bce2b7fd4ad1b6e6 Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Wed, 28 Jul 2021 01:44:26 -0400 Subject: [PATCH 05/16] Bunch of fixes/hacks around the test framework stuff. --- script/testing/reporting/constants.py | 11 +++++++++-- .../reporting/parsers/oltpbench/res_parser.py | 4 ++-- .../parsers/oltpbench/summary_parser.py | 6 +++--- .../testing/reporting/parsers/parse_data.py | 19 +++++++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/script/testing/reporting/constants.py b/script/testing/reporting/constants.py index 174c49265a..b34307ec0e 100644 --- a/script/testing/reporting/constants.py +++ b/script/testing/reporting/constants.py @@ -2,5 +2,12 @@ LATENCY_ATTRIBUTE_MAPPING = [ # key = key in publish result json, value= string to search OLTPBench results for # TODO(WAN): this mapping could probably be a.. map? {}? - ('l_25', '25'), ('l_75', '75'), ('l_90', '90'), ('l_95', '95'), ('l_99', '99'), - ('avg', 'av'), ('median', 'median'), ('min', 'min'), ('max', 'max')] + ('l_25', '25th Percentile Latency (microseconds)'), + ('l_75', '75th Percentile Latency (microseconds)'), + ('l_90', '90th Percentile Latency (microseconds)'), + ('l_95', '95th Percentile Latency (microseconds)'), + ('l_99', '99th Percentile Latency (microseconds)'), + ('avg', 'Average Latency (microseconds)'), + ('median', 'Median Latency (microseconds)'), + ('min', 'Minimum Latency (microseconds)'), + ('max', 'Maximum Latency (microseconds)')] diff --git a/script/testing/reporting/parsers/oltpbench/res_parser.py b/script/testing/reporting/parsers/oltpbench/res_parser.py index ee26470c09..c6dc6dbb18 100644 --- a/script/testing/reporting/parsers/oltpbench/res_parser.py +++ b/script/testing/reporting/parsers/oltpbench/res_parser.py @@ -29,8 +29,8 @@ def get_latency_val(row, pattern): reader = csv.DictReader(csvfile, delimiter=',') for row in reader: incremental_metrics.append({ - "time": float(gvbp(row, 'time', None)), - "throughput": float(gvbp(row, 'throughput', None)), + "time": float(gvbp(row, 'time(sec)', None)), + "throughput": float(gvbp(row, 'throughput(req/sec)', None)), "latency": {key: get_latency_val(row, pat) for key, pat in LATENCY_ATTRIBUTE_MAPPING} }) diff --git a/script/testing/reporting/parsers/oltpbench/summary_parser.py b/script/testing/reporting/parsers/oltpbench/summary_parser.py index 39e5c35b3c..0b8400aa4c 100644 --- a/script/testing/reporting/parsers/oltpbench/summary_parser.py +++ b/script/testing/reporting/parsers/oltpbench/summary_parser.py @@ -30,7 +30,7 @@ def parse_summary_file(path): """ def get_latency_val(latency_dist, pattern): value = get_value_by_pattern(latency_dist, pattern, None) - return float("{:.4}".format(value)) if value else value + return float("{:.4}".format(float(value))) if value else value with open(path) as summary_file: summary = json.load(summary_file) @@ -41,14 +41,14 @@ def get_latency_val(latency_dist, pattern): 'db_version': summary.get('DBMS Version', UNKNOWN_RESULT) } } - timestamp = int(get_value_by_pattern(summary, 'timestamp', str(time()))) + timestamp = int(get_value_by_pattern(summary, 'Current Timestamp (milliseconds)', str(time()))) benchmark_type = summary.get('Benchmark Type', UNKNOWN_RESULT) parameters = { 'scale_factor': summary.get('scalefactor', '-1.0'), 'terminals': int(summary.get('terminals', -1)) } metrics = { - 'throughput': get_value_by_pattern(summary, 'throughput', '-1.0'), + 'throughput': get_value_by_pattern(summary, 'Throughput (requests/second)', '-1.0'), 'latency': {key: get_latency_val(latency_dist, pattern) for key, pattern in LATENCY_ATTRIBUTE_MAPPING} } diff --git a/script/testing/reporting/parsers/parse_data.py b/script/testing/reporting/parsers/parse_data.py index 743e8265e7..e3132d7852 100644 --- a/script/testing/reporting/parsers/parse_data.py +++ b/script/testing/reporting/parsers/parse_data.py @@ -1,3 +1,4 @@ +import glob import os import re from decimal import Decimal @@ -130,6 +131,24 @@ def parse_oltpbench_files(results_dir): metrics : dict The summary measurements that were gathered from the test. """ + + def hack_rename(old_glob_target, new_name): + """ + Wan wants to avoid a rabbit hole of refactoring. + Therefore the new OLTPBench files are being renamed to match old expectations here. + """ + matches = glob.glob(old_glob_target) + assert len(matches) == 1 + os.rename(matches[0], new_name) + + hack_rename(f'{results_dir}/*.results.csv', f'{results_dir}/oltpbench.res') + hack_rename(f'{results_dir}/*.raw.csv', f'{results_dir}/oltpbench.csv') + hack_rename(f'{results_dir}/*.samples.csv', f'{results_dir}/oltpbench.samples') + hack_rename(f'{results_dir}/*.summary.json', f'{results_dir}/oltpbench.summary') + hack_rename(f'{results_dir}/*.params.json', f'{results_dir}/oltpbench.params') + hack_rename(f'{results_dir}/*.metrics.json', f'{results_dir}/oltpbench.metrics') + hack_rename(f'{results_dir}/*.config.xml', f'{results_dir}/oltpbench.expconfig') + config_parameters = parse_config_file(results_dir + '/oltpbench.expconfig') metadata, timestamp, benchmark_type, summary_parameters, metrics = parse_summary_file( results_dir + '/oltpbench.summary') From 3d206c8b2cdcffe06250b878bbc01086329dbcc2 Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Wed, 28 Jul 2021 06:51:52 -0400 Subject: [PATCH 06/16] More test framework hackery. --- script/testing/reporting/constants.py | 12 ------------ .../reporting/parsers/oltpbench/res_parser.py | 13 +++++++++++-- .../reporting/parsers/oltpbench/summary_parser.py | 14 ++++++++++++-- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/script/testing/reporting/constants.py b/script/testing/reporting/constants.py index b34307ec0e..ccda9a0f7c 100644 --- a/script/testing/reporting/constants.py +++ b/script/testing/reporting/constants.py @@ -1,13 +1 @@ UNKNOWN_RESULT = 'unknown' -LATENCY_ATTRIBUTE_MAPPING = [ - # key = key in publish result json, value= string to search OLTPBench results for - # TODO(WAN): this mapping could probably be a.. map? {}? - ('l_25', '25th Percentile Latency (microseconds)'), - ('l_75', '75th Percentile Latency (microseconds)'), - ('l_90', '90th Percentile Latency (microseconds)'), - ('l_95', '95th Percentile Latency (microseconds)'), - ('l_99', '99th Percentile Latency (microseconds)'), - ('avg', 'Average Latency (microseconds)'), - ('median', 'Median Latency (microseconds)'), - ('min', 'Minimum Latency (microseconds)'), - ('max', 'Maximum Latency (microseconds)')] diff --git a/script/testing/reporting/parsers/oltpbench/res_parser.py b/script/testing/reporting/parsers/oltpbench/res_parser.py index c6dc6dbb18..039b15fd2d 100644 --- a/script/testing/reporting/parsers/oltpbench/res_parser.py +++ b/script/testing/reporting/parsers/oltpbench/res_parser.py @@ -1,6 +1,5 @@ import csv -from ...constants import LATENCY_ATTRIBUTE_MAPPING from ...utils import get_value_by_pattern @@ -32,7 +31,17 @@ def get_latency_val(row, pattern): "time": float(gvbp(row, 'time(sec)', None)), "throughput": float(gvbp(row, 'throughput(req/sec)', None)), "latency": {key: get_latency_val(row, pat) - for key, pat in LATENCY_ATTRIBUTE_MAPPING} + for key, pat in [ + ('l_25', '25th_lat(ms)'), + ('l_75', '75th_lat(ms)'), + ('l_90', '90th_lat(ms)'), + ('l_95', '95th_lat(ms)'), + ('l_99', '99th_lat(ms)'), + ('avg', 'avg_lat(ms)'), + ('median', 'median_lat(ms)'), + ('min', 'min_lat(ms)'), + ('max', 'max_lat(ms)') + ]} }) return incremental_metrics diff --git a/script/testing/reporting/parsers/oltpbench/summary_parser.py b/script/testing/reporting/parsers/oltpbench/summary_parser.py index 0b8400aa4c..10e68224c9 100644 --- a/script/testing/reporting/parsers/oltpbench/summary_parser.py +++ b/script/testing/reporting/parsers/oltpbench/summary_parser.py @@ -1,7 +1,7 @@ import json from time import time -from ...constants import LATENCY_ATTRIBUTE_MAPPING, UNKNOWN_RESULT +from ...constants import UNKNOWN_RESULT from ...utils import get_value_by_pattern @@ -50,7 +50,17 @@ def get_latency_val(latency_dist, pattern): metrics = { 'throughput': get_value_by_pattern(summary, 'Throughput (requests/second)', '-1.0'), 'latency': {key: get_latency_val(latency_dist, pattern) - for key, pattern in LATENCY_ATTRIBUTE_MAPPING} + for key, pattern in [ + ('l_25', '25th Percentile Latency (microseconds)'), + ('l_75', '75th Percentile Latency (microseconds)'), + ('l_90', '90th Percentile Latency (microseconds)'), + ('l_95', '95th Percentile Latency (microseconds)'), + ('l_99', '99th Percentile Latency (microseconds)'), + ('avg', 'Average Latency (microseconds)'), + ('median', 'Median Latency (microseconds)'), + ('min', 'Minimum Latency (microseconds)'), + ('max', 'Maximum Latency (microseconds)') + ]} } return metadata, timestamp, benchmark_type, parameters, metrics From 8ef0e01a9c9e8a157a75d539253a37d629477d6a Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Wed, 28 Jul 2021 11:04:53 -0400 Subject: [PATCH 07/16] Replace one stupid hack with another stupid hack. --- .../testing/reporting/parsers/parse_data.py | 20 ++++--------------- src/include/common/version.h | 2 ++ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/script/testing/reporting/parsers/parse_data.py b/script/testing/reporting/parsers/parse_data.py index e3132d7852..784cd7f2ca 100644 --- a/script/testing/reporting/parsers/parse_data.py +++ b/script/testing/reporting/parsers/parse_data.py @@ -187,22 +187,10 @@ def _parse_db_metadata(): Warnings -------- - Giant hack that parses a hardcoded constant NOISEPAGE_VERSION - in src/include/common/version.h. + Giant hack that hardcodes version number. If the hack is unsuccessful, it defaults to UNKNOWN_RESULT. """ - regex = r"NOISEPAGE_VERSION[=\s].*(\d.\d.\d)" - curr_dir = os.path.dirname(os.path.realpath(__file__)) - # TODO(WAN): Don't do this. We support SELECT VERSION(), do that instead. - version_file_relative = '../../../../src/include/common/version.h' - version_file = os.path.join(curr_dir, version_file_relative) - db_metadata = {'noisepage': {'db_version': UNKNOWN_RESULT}} - try: - with open(version_file) as f: - match = re.search(regex, f.read()) - db_metadata['noisepage']['db_version'] = match.group(1) - except Exception as err: - LOG.error(err) - - return db_metadata + return {'noisepage': {'db_version': '1.0.0'}} + + diff --git a/src/include/common/version.h b/src/include/common/version.h index d14d193bf8..beb78bd0de 100644 --- a/src/include/common/version.h +++ b/src/include/common/version.h @@ -5,6 +5,8 @@ namespace noisepage::common { constexpr std::string_view NOISEPAGE_NAME = "NoisePage"; +// TODO(WAN): There used to be a fragile hack in parse_data.py that would try to regex out the version number. +// Please update script/testing/reporting/parsers/parse_data.py manually if you change this version number. constexpr std::string_view NOISEPAGE_VERSION = "1.0.0"; constexpr std::string_view NOISEPAGE_VERSION_STR = "NoisePage 1.0.0"; From 782ef897a4bc0ea76b07b12f877f0a2607cb187b Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Thu, 29 Jul 2021 00:23:23 -0400 Subject: [PATCH 08/16] Print out result. --- script/testing/reporting/report_result.py | 1 + 1 file changed, 1 insertion(+) diff --git a/script/testing/reporting/report_result.py b/script/testing/reporting/report_result.py index ccc1ef04a3..d7db5b573f 100644 --- a/script/testing/reporting/report_result.py +++ b/script/testing/reporting/report_result.py @@ -119,6 +119,7 @@ def _send_result(env, path, username, password, result): """ url = f"{PERFORMANCE_STORAGE_SERVICE_API.get(env)}{path}" LOG.debug(f"Sending results to: {url}") + LOG.info(f"Uploading result: {result}") try: result = requests.post(url, json=result, auth=(username, password)) From e1be29b8aae361aae0a7d3e0a64848adc0a1175f Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Thu, 29 Jul 2021 04:15:30 -0400 Subject: [PATCH 09/16] Version is being weird. --- script/testing/reporting/parsers/oltpbench/summary_parser.py | 2 +- src/include/common/version.h | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/script/testing/reporting/parsers/oltpbench/summary_parser.py b/script/testing/reporting/parsers/oltpbench/summary_parser.py index 10e68224c9..ec49dd3370 100644 --- a/script/testing/reporting/parsers/oltpbench/summary_parser.py +++ b/script/testing/reporting/parsers/oltpbench/summary_parser.py @@ -38,7 +38,7 @@ def get_latency_val(latency_dist, pattern): metadata = { 'noisepage': { - 'db_version': summary.get('DBMS Version', UNKNOWN_RESULT) + 'db_version': '1.0.0' } } timestamp = int(get_value_by_pattern(summary, 'Current Timestamp (milliseconds)', str(time()))) diff --git a/src/include/common/version.h b/src/include/common/version.h index beb78bd0de..3befce5689 100644 --- a/src/include/common/version.h +++ b/src/include/common/version.h @@ -7,6 +7,7 @@ namespace noisepage::common { constexpr std::string_view NOISEPAGE_NAME = "NoisePage"; // TODO(WAN): There used to be a fragile hack in parse_data.py that would try to regex out the version number. // Please update script/testing/reporting/parsers/parse_data.py manually if you change this version number. +// And also script/testing/reporting/parsers/summary_parser.py. constexpr std::string_view NOISEPAGE_VERSION = "1.0.0"; constexpr std::string_view NOISEPAGE_VERSION_STR = "NoisePage 1.0.0"; From 016eec9afc2536dc04f51964e9ca23a09d6f30b3 Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Fri, 6 Aug 2021 05:28:40 -0400 Subject: [PATCH 10/16] Suppress Maven download progress. --- script/testing/oltpbench/test_oltpbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/testing/oltpbench/test_oltpbench.py b/script/testing/oltpbench/test_oltpbench.py index d9430f2865..ebd6b2fd40 100644 --- a/script/testing/oltpbench/test_oltpbench.py +++ b/script/testing/oltpbench/test_oltpbench.py @@ -43,7 +43,7 @@ def _build_oltpbench(self): """ old_dir = os.getcwd() os.chdir(constants.OLTPBENCH_GIT_LOCAL_PATH) - expect_command("./mvnw package") + expect_command("./mvnw package --no-transfer-progress") os.chdir(constants.OLTPBENCH_GIT_TARGET_PATH) expect_command(f"tar xvzf {constants.OLTPBENCH_VERSION}.tgz") os.chdir(old_dir) From e0a35d4bb6821d2dc30eea3c6e7ed3f9ac59efb7 Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Fri, 6 Aug 2021 05:28:57 -0400 Subject: [PATCH 11/16] Change folder for Jenkins to benchbase. This might be the bug? --- script/testing/oltpbench/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/testing/oltpbench/constants.py b/script/testing/oltpbench/constants.py index f0203e9f99..8b36cd3558 100755 --- a/script/testing/oltpbench/constants.py +++ b/script/testing/oltpbench/constants.py @@ -5,7 +5,7 @@ # git settings for OLTPBench. OLTPBENCH_VERSION = "oltpbench2-20.1.3-SNAPSHOT" OLTPBENCH_GIT_URL = "https://github.com/oltpbenchmark/oltpbench.git" -OLTPBENCH_GIT_LOCAL_PATH = os.path.join(DIR_TMP, "oltpbench") +OLTPBENCH_GIT_LOCAL_PATH = os.path.join(DIR_TMP, "benchbase") OLTPBENCH_GIT_TARGET_PATH = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "target") OLTPBENCH_GIT_FINAL_PATH = os.path.join(OLTPBENCH_GIT_TARGET_PATH, OLTPBENCH_VERSION) OLTPBENCH_GIT_CLEAN_COMMAND = "rm -rf {}".format(OLTPBENCH_GIT_LOCAL_PATH) From 03a9d77fe9275e7d5c283963ae59e5e4d4745182 Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Tue, 10 Aug 2021 15:11:23 -0400 Subject: [PATCH 12/16] Flip over to benchbase. --- script/testing/oltpbench/constants.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/testing/oltpbench/constants.py b/script/testing/oltpbench/constants.py index 8b36cd3558..ce909babb1 100755 --- a/script/testing/oltpbench/constants.py +++ b/script/testing/oltpbench/constants.py @@ -3,13 +3,13 @@ from ..util.constants import DIR_TMP # git settings for OLTPBench. -OLTPBENCH_VERSION = "oltpbench2-20.1.3-SNAPSHOT" -OLTPBENCH_GIT_URL = "https://github.com/oltpbenchmark/oltpbench.git" +OLTPBENCH_VERSION = "benchbase-2021-SNAPSHOT" +OLTPBENCH_GIT_URL = "https://github.com/cmu-db/benchbase.git" OLTPBENCH_GIT_LOCAL_PATH = os.path.join(DIR_TMP, "benchbase") OLTPBENCH_GIT_TARGET_PATH = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "target") OLTPBENCH_GIT_FINAL_PATH = os.path.join(OLTPBENCH_GIT_TARGET_PATH, OLTPBENCH_VERSION) OLTPBENCH_GIT_CLEAN_COMMAND = "rm -rf {}".format(OLTPBENCH_GIT_LOCAL_PATH) -OLTPBENCH_GIT_CLONE_COMMAND = "git clone --single-branch --branch oltpbench_tim --depth 1 {} {}".format( +OLTPBENCH_GIT_CLONE_COMMAND = "git clone --depth 1 {} {}".format( OLTPBENCH_GIT_URL, OLTPBENCH_GIT_LOCAL_PATH) From 9d37fec6f7bd1fd95a3c21b9d83bfcbc17ce6968 Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Tue, 10 Aug 2021 23:34:58 -0400 Subject: [PATCH 13/16] Fix typo (oltpbench2 -> benchbase). --- script/testing/oltpbench/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/testing/oltpbench/constants.py b/script/testing/oltpbench/constants.py index ce909babb1..15b6797c8c 100755 --- a/script/testing/oltpbench/constants.py +++ b/script/testing/oltpbench/constants.py @@ -25,7 +25,7 @@ OLTPBENCH_DEFAULT_DBTYPE = "noisepage" OLTPBENCH_DEFAULT_DRIVER = "org.postgresql.Driver" OLTPBENCH_DEFAULT_RATE = "unlimited" -OLTPBENCH_DEFAULT_BIN = "java -jar oltpbench2.jar " +OLTPBENCH_DEFAULT_BIN = "java -jar benchbase.jar " OLTPBENCH_DEFAULT_DATABASE_RESTART = True OLTPBENCH_DEFAULT_DATABASE_CREATE = True OLTPBENCH_DEFAULT_DATABASE_LOAD = True From 7f8d68c4050eb30abfefcf3cc19cf4cc0a4591ff Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Wed, 11 Aug 2021 16:09:36 -0400 Subject: [PATCH 14/16] Don't run BenchBase tests. BenchBase can test itself. --- script/testing/oltpbench/test_oltpbench.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/testing/oltpbench/test_oltpbench.py b/script/testing/oltpbench/test_oltpbench.py index ebd6b2fd40..4941893abc 100644 --- a/script/testing/oltpbench/test_oltpbench.py +++ b/script/testing/oltpbench/test_oltpbench.py @@ -43,7 +43,9 @@ def _build_oltpbench(self): """ old_dir = os.getcwd() os.chdir(constants.OLTPBENCH_GIT_LOCAL_PATH) - expect_command("./mvnw package --no-transfer-progress") + # --no-transfer-progress: don't show download progress, too noisy + # -Dmaven.test.skip=true: we're not in the business of testing BenchBase, we just want to use it + expect_command("./mvnw package --no-transfer-progress -Dmaven.test.skip=true") os.chdir(constants.OLTPBENCH_GIT_TARGET_PATH) expect_command(f"tar xvzf {constants.OLTPBENCH_VERSION}.tgz") os.chdir(old_dir) From dac568c891f129a2cb15576ac42a5027f169144d Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Thu, 12 Aug 2021 11:14:30 -0400 Subject: [PATCH 15/16] Temporary test with Wan's branch of benchbase. --- Jenkinsfile | 186 +++++++++++++------------- script/testing/oltpbench/constants.py | 4 +- 2 files changed, 95 insertions(+), 95 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 351f8899f6..72e0e54c89 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,99 +13,99 @@ pipeline { parallelsAlwaysFailFast() } stages { - stage('Ready For CI') { - agent { docker { image 'noisepage:focal' } } - when { not { branch 'master' } } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageGithub() } } - post { cleanup { deleteDir() } } - } - stage('Check') { - parallel { - stage('ubuntu-20.04/gcc-9.3 (Debug/format/lint/censored)') { - agent { docker { image 'noisepage:focal' } } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } - post { cleanup { deleteDir() } } - } - stage('ubuntu-20.04/clang-8.0 (Debug/format/lint/censored)') { - agent { docker { image 'noisepage:focal' } } - environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } - post { cleanup { deleteDir() } } - } - } - } - - stage('Build-only checks') { - parallel { - stage('Benchmarks (debug build only)') { - agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ - buildCommand: 'ninja', - cmake: '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_BENCHMARKS=ON', - ] ) } } - post { cleanup { deleteDir() } } - } - stage('Logging disabled (release build only)') { - agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ - buildCommand: 'ninja', - cmake: '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_BENCHMARKS=ON -DNOISEPAGE_BUILD_TESTS=ON -NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS=ON -DNOISEPAGE_USE_LOGGING=OFF' - ] ) } } - post { cleanup { deleteDir() } } - } - } - } - - stage('Test') { - parallel { - // The first argument to utils.stageTest() indicates whether pipeline metrics should be gathered. - // Pipeline metrics take a while to run and don't need to be run in every stage. The pipeline metrics are only run in one arbitrary stage to check for breakage. - - stage('ubuntu-20.04/gcc-9.3 (Debug/ASAN/jumbotests)') { - agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(true, [cmake: - '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' - ] ) } } - post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } - } - - stage('ubuntu-20.04/gcc-9.3 (Debug/Coverage/unittest)') { - agent { docker { image 'noisepage:focal' ; label 'dgb' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } - environment { CODECOV_TOKEN=credentials('codecov-token') } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: - // Note that unity builds mess with coverage. - '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_GENERATE_COVERAGE=ON' - ] ) } } - post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } - } - - stage('ubuntu-20.04/clang-8.0 (Debug/ASAN/jumbotests)') { - agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } - environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: - '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' - ] ) } } - post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } - } - - stage('ubuntu-20.04/gcc-9.3 (Release/jumbotests)') { - agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: - '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' - ] ) } } - post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } - } - - stage('ubuntu-20.04/clang-8.0 (Release/jumbotests)') { - agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } - environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } - steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: - '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' - ] ) } } - post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } - } - } - } +// stage('Ready For CI') { +// agent { docker { image 'noisepage:focal' } } +// when { not { branch 'master' } } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageGithub() } } +// post { cleanup { deleteDir() } } +// } +// stage('Check') { +// parallel { +// stage('ubuntu-20.04/gcc-9.3 (Debug/format/lint/censored)') { +// agent { docker { image 'noisepage:focal' } } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } +// post { cleanup { deleteDir() } } +// } +// stage('ubuntu-20.04/clang-8.0 (Debug/format/lint/censored)') { +// agent { docker { image 'noisepage:focal' } } +// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } +// post { cleanup { deleteDir() } } +// } +// } +// } +// +// stage('Build-only checks') { +// parallel { +// stage('Benchmarks (debug build only)') { +// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ +// buildCommand: 'ninja', +// cmake: '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_BENCHMARKS=ON', +// ] ) } } +// post { cleanup { deleteDir() } } +// } +// stage('Logging disabled (release build only)') { +// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ +// buildCommand: 'ninja', +// cmake: '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_BENCHMARKS=ON -DNOISEPAGE_BUILD_TESTS=ON -NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS=ON -DNOISEPAGE_USE_LOGGING=OFF' +// ] ) } } +// post { cleanup { deleteDir() } } +// } +// } +// } +// +// stage('Test') { +// parallel { +// // The first argument to utils.stageTest() indicates whether pipeline metrics should be gathered. +// // Pipeline metrics take a while to run and don't need to be run in every stage. The pipeline metrics are only run in one arbitrary stage to check for breakage. +// +// stage('ubuntu-20.04/gcc-9.3 (Debug/ASAN/jumbotests)') { +// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(true, [cmake: +// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' +// ] ) } } +// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } +// } +// +// stage('ubuntu-20.04/gcc-9.3 (Debug/Coverage/unittest)') { +// agent { docker { image 'noisepage:focal' ; label 'dgb' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } +// environment { CODECOV_TOKEN=credentials('codecov-token') } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: +// // Note that unity builds mess with coverage. +// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_GENERATE_COVERAGE=ON' +// ] ) } } +// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } +// } +// +// stage('ubuntu-20.04/clang-8.0 (Debug/ASAN/jumbotests)') { +// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } +// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: +// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' +// ] ) } } +// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } +// } +// +// stage('ubuntu-20.04/gcc-9.3 (Release/jumbotests)') { +// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: +// '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' +// ] ) } } +// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } +// } +// +// stage('ubuntu-20.04/clang-8.0 (Release/jumbotests)') { +// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } +// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } +// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: +// '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' +// ] ) } } +// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } +// } +// } +// } stage('End-to-End') { parallel { diff --git a/script/testing/oltpbench/constants.py b/script/testing/oltpbench/constants.py index 15b6797c8c..521c1fbfac 100755 --- a/script/testing/oltpbench/constants.py +++ b/script/testing/oltpbench/constants.py @@ -4,12 +4,12 @@ # git settings for OLTPBench. OLTPBENCH_VERSION = "benchbase-2021-SNAPSHOT" -OLTPBENCH_GIT_URL = "https://github.com/cmu-db/benchbase.git" +OLTPBENCH_GIT_URL = "https://github.com/lmwnshn/benchbase.git" OLTPBENCH_GIT_LOCAL_PATH = os.path.join(DIR_TMP, "benchbase") OLTPBENCH_GIT_TARGET_PATH = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "target") OLTPBENCH_GIT_FINAL_PATH = os.path.join(OLTPBENCH_GIT_TARGET_PATH, OLTPBENCH_VERSION) OLTPBENCH_GIT_CLEAN_COMMAND = "rm -rf {}".format(OLTPBENCH_GIT_LOCAL_PATH) -OLTPBENCH_GIT_CLONE_COMMAND = "git clone --depth 1 {} {}".format( +OLTPBENCH_GIT_CLONE_COMMAND = "git clone --single-branch --branch yami --depth 1 {} {}".format( OLTPBENCH_GIT_URL, OLTPBENCH_GIT_LOCAL_PATH) From 714253a1eba2efa30756bb411a594bc2607f325d Mon Sep 17 00:00:00 2001 From: Wan Shen Lim Date: Thu, 12 Aug 2021 12:41:59 -0400 Subject: [PATCH 16/16] Revert "Temporary test with Wan's branch of benchbase." This reverts commit dac568c891f129a2cb15576ac42a5027f169144d. --- Jenkinsfile | 186 +++++++++++++------------- script/testing/oltpbench/constants.py | 4 +- 2 files changed, 95 insertions(+), 95 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 72e0e54c89..351f8899f6 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,99 +13,99 @@ pipeline { parallelsAlwaysFailFast() } stages { -// stage('Ready For CI') { -// agent { docker { image 'noisepage:focal' } } -// when { not { branch 'master' } } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageGithub() } } -// post { cleanup { deleteDir() } } -// } -// stage('Check') { -// parallel { -// stage('ubuntu-20.04/gcc-9.3 (Debug/format/lint/censored)') { -// agent { docker { image 'noisepage:focal' } } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } -// post { cleanup { deleteDir() } } -// } -// stage('ubuntu-20.04/clang-8.0 (Debug/format/lint/censored)') { -// agent { docker { image 'noisepage:focal' } } -// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } -// post { cleanup { deleteDir() } } -// } -// } -// } -// -// stage('Build-only checks') { -// parallel { -// stage('Benchmarks (debug build only)') { -// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ -// buildCommand: 'ninja', -// cmake: '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_BENCHMARKS=ON', -// ] ) } } -// post { cleanup { deleteDir() } } -// } -// stage('Logging disabled (release build only)') { -// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ -// buildCommand: 'ninja', -// cmake: '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_BENCHMARKS=ON -DNOISEPAGE_BUILD_TESTS=ON -NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS=ON -DNOISEPAGE_USE_LOGGING=OFF' -// ] ) } } -// post { cleanup { deleteDir() } } -// } -// } -// } -// -// stage('Test') { -// parallel { -// // The first argument to utils.stageTest() indicates whether pipeline metrics should be gathered. -// // Pipeline metrics take a while to run and don't need to be run in every stage. The pipeline metrics are only run in one arbitrary stage to check for breakage. -// -// stage('ubuntu-20.04/gcc-9.3 (Debug/ASAN/jumbotests)') { -// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(true, [cmake: -// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' -// ] ) } } -// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } -// } -// -// stage('ubuntu-20.04/gcc-9.3 (Debug/Coverage/unittest)') { -// agent { docker { image 'noisepage:focal' ; label 'dgb' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } -// environment { CODECOV_TOKEN=credentials('codecov-token') } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: -// // Note that unity builds mess with coverage. -// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_GENERATE_COVERAGE=ON' -// ] ) } } -// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } -// } -// -// stage('ubuntu-20.04/clang-8.0 (Debug/ASAN/jumbotests)') { -// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } -// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: -// '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' -// ] ) } } -// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } -// } -// -// stage('ubuntu-20.04/gcc-9.3 (Release/jumbotests)') { -// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: -// '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' -// ] ) } } -// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } -// } -// -// stage('ubuntu-20.04/clang-8.0 (Release/jumbotests)') { -// agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } -// environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } -// steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: -// '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' -// ] ) } } -// post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } -// } -// } -// } + stage('Ready For CI') { + agent { docker { image 'noisepage:focal' } } + when { not { branch 'master' } } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageGithub() } } + post { cleanup { deleteDir() } } + } + stage('Check') { + parallel { + stage('ubuntu-20.04/gcc-9.3 (Debug/format/lint/censored)') { + agent { docker { image 'noisepage:focal' } } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } + post { cleanup { deleteDir() } } + } + stage('ubuntu-20.04/clang-8.0 (Debug/format/lint/censored)') { + agent { docker { image 'noisepage:focal' } } + environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageCheck() } } + post { cleanup { deleteDir() } } + } + } + } + + stage('Build-only checks') { + parallel { + stage('Benchmarks (debug build only)') { + agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ + buildCommand: 'ninja', + cmake: '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_BENCHMARKS=ON', + ] ) } } + post { cleanup { deleteDir() } } + } + stage('Logging disabled (release build only)') { + agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageBuildDefault([ + buildCommand: 'ninja', + cmake: '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_BENCHMARKS=ON -DNOISEPAGE_BUILD_TESTS=ON -NOISEPAGE_BUILD_SELF_DRIVING_E2E_TESTS=ON -DNOISEPAGE_USE_LOGGING=OFF' + ] ) } } + post { cleanup { deleteDir() } } + } + } + } + + stage('Test') { + parallel { + // The first argument to utils.stageTest() indicates whether pipeline metrics should be gathered. + // Pipeline metrics take a while to run and don't need to be run in every stage. The pipeline metrics are only run in one arbitrary stage to check for breakage. + + stage('ubuntu-20.04/gcc-9.3 (Debug/ASAN/jumbotests)') { + agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(true, [cmake: + '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' + ] ) } } + post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } + } + + stage('ubuntu-20.04/gcc-9.3 (Debug/Coverage/unittest)') { + agent { docker { image 'noisepage:focal' ; label 'dgb' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } + environment { CODECOV_TOKEN=credentials('codecov-token') } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: + // Note that unity builds mess with coverage. + '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_GENERATE_COVERAGE=ON' + ] ) } } + post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } + } + + stage('ubuntu-20.04/clang-8.0 (Debug/ASAN/jumbotests)') { + agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } + environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: + '-DCMAKE_BUILD_TYPE=Debug -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_ASAN=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' + ] ) } } + post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } + } + + stage('ubuntu-20.04/gcc-9.3 (Release/jumbotests)') { + agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: + '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' + ] ) } } + post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } + } + + stage('ubuntu-20.04/clang-8.0 (Release/jumbotests)') { + agent { docker { image 'noisepage:focal' ; args '--cap-add sys_ptrace -v /jenkins/ccache:/home/jenkins/.ccache' } } + environment { CC="/usr/bin/clang-8" ; CXX="/usr/bin/clang++-8" } + steps { script { utils = utils ?: load(utilsFileName) ; utils.stageTest(false, [cmake: + '-DCMAKE_BUILD_TYPE=Release -DNOISEPAGE_BUILD_TESTS=ON -DNOISEPAGE_UNITY_BUILD=ON -DNOISEPAGE_USE_JUMBOTESTS=ON' + ] ) } } + post { always { script { utils = utils ?: load(utilsFileName) ; utils.stageArchive() } } ; cleanup { deleteDir() } } + } + } + } stage('End-to-End') { parallel { diff --git a/script/testing/oltpbench/constants.py b/script/testing/oltpbench/constants.py index 521c1fbfac..15b6797c8c 100755 --- a/script/testing/oltpbench/constants.py +++ b/script/testing/oltpbench/constants.py @@ -4,12 +4,12 @@ # git settings for OLTPBench. OLTPBENCH_VERSION = "benchbase-2021-SNAPSHOT" -OLTPBENCH_GIT_URL = "https://github.com/lmwnshn/benchbase.git" +OLTPBENCH_GIT_URL = "https://github.com/cmu-db/benchbase.git" OLTPBENCH_GIT_LOCAL_PATH = os.path.join(DIR_TMP, "benchbase") OLTPBENCH_GIT_TARGET_PATH = os.path.join(OLTPBENCH_GIT_LOCAL_PATH, "target") OLTPBENCH_GIT_FINAL_PATH = os.path.join(OLTPBENCH_GIT_TARGET_PATH, OLTPBENCH_VERSION) OLTPBENCH_GIT_CLEAN_COMMAND = "rm -rf {}".format(OLTPBENCH_GIT_LOCAL_PATH) -OLTPBENCH_GIT_CLONE_COMMAND = "git clone --single-branch --branch yami --depth 1 {} {}".format( +OLTPBENCH_GIT_CLONE_COMMAND = "git clone --depth 1 {} {}".format( OLTPBENCH_GIT_URL, OLTPBENCH_GIT_LOCAL_PATH)