Skip to content

Commit a1bba9d

Browse files
filipecosta90filipecosta90
authored
Include extra tests for test-glob and introduce test-regex (#395)
1 parent fe69ca1 commit a1bba9d

9 files changed

+191
-14
lines changed

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[tool.poetry]
22
name = "redisbench-admin"
3-
version = "0.9.33"
3+
version = "0.9.44"
44
description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
55
authors = ["filipecosta90 <[email protected]>","Redis Performance Group <[email protected]>"]
66
readme = "README.md"

‎redisbench_admin/run/args.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
ENV = os.getenv("ENV", "oss-standalone,oss-cluster")
2525
SETUP = os.getenv("SETUP", "")
2626
BENCHMARK_GLOB = os.getenv("BENCHMARK_GLOB", "*.yml")
27+
BENCHMARK_REGEX = os.getenv("BENCHMARK_REGEX", ".*")
2728
S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME", "ci.benchmarks.redislabs")
2829
PUSH_S3 = bool(os.getenv("PUSH_S3", False))
2930
REDIS_7 = bool(os.getenv("REDIS_7", True))
@@ -82,6 +83,12 @@ def common_run_args(parser):
8283
default=BENCHMARK_GLOB,
8384
help="specify a test glob pattern to use on the tests directory. by default uses *.yml. If --test is defined this options has no effect.",
8485
)
86+
parser.add_argument(
87+
"--test-regex",
88+
type=str,
89+
default=BENCHMARK_REGEX,
90+
help="specify a test regex pattern to use on the tests directory. by default uses '.*'. If --test is defined this options has no effect.",
91+
)
8592

8693
parser.add_argument(
8794
"--defaults_filename",

‎redisbench_admin/utils/benchmark_config.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,10 @@ def parse_exporter_timemetric(metric_path: str, results_dict: dict):
5555
def prepare_benchmark_definitions(args):
5656
benchmark_definitions = {}
5757
result = True
58-
defaults_filename, files = get_testfiles_to_process(args)
58+
defaults_filename = args.defaults_filename
59+
files = get_testfiles_to_process(
60+
args.test_glob, args.test, defaults_filename, args.test_regex
61+
)
5962

6063
(
6164
default_kpis,
@@ -482,23 +485,29 @@ def min_ver_check(
482485
)
483486

484487

485-
def get_testfiles_to_process(args):
486-
defaults_filename = args.defaults_filename
487-
if args.test == "":
488-
files = pathlib.Path().glob(args.test_glob)
488+
def get_testfiles_to_process(test_glob, test_name, defaults_filename, test_regex=".*"):
489+
if test_name == "":
490+
files = pathlib.Path().glob(test_glob)
489491
files = [str(x) for x in files]
490-
if defaults_filename in files:
491-
files.remove(defaults_filename)
492-
492+
for file in files:
493+
if defaults_filename in file:
494+
files.remove(file)
495+
final_files = []
496+
for file in files:
497+
a = re.search(test_regex, file)
498+
if a is not None:
499+
final_files.append(a.string)
500+
files = final_files
493501
logging.info(
494502
"Running all specified benchmarks: {}".format(
495503
" ".join([str(x) for x in files])
496504
)
497505
)
506+
498507
else:
499-
files = args.test.split(",")
508+
files = test_name.split(",")
500509
logging.info("Running specific benchmark in file: {}".format(files))
501-
return defaults_filename, files
510+
return files
502511

503512

504513
def check_required_modules(module_names, required_modules):

‎tests/test_benchmark_config.py‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
get_termination_timeout_secs,
1414
prepare_benchmark_definitions,
1515
process_benchmark_definitions_remote_timeouts,
16+
get_testfiles_to_process,
1617
)
1718

1819

@@ -196,7 +197,7 @@ def test_prepare_benchmark_definitions():
196197
clusterconfig,
197198
) = prepare_benchmark_definitions(args)
198199
assert result == True
199-
assert len(benchmark_definitions.keys()) == 2
200+
assert len(benchmark_definitions.keys()) == 6
200201

201202

202203
def test_process_benchmark_definitions_remote_timeouts():
@@ -224,10 +225,41 @@ def test_process_benchmark_definitions_remote_timeouts():
224225
clusterconfig,
225226
) = prepare_benchmark_definitions(args)
226227
assert result == True
227-
assert len(benchmark_definitions.keys()) == 2
228+
assert len(benchmark_definitions.keys()) == 6
228229
remote_envs_timeout = process_benchmark_definitions_remote_timeouts(
229230
benchmark_definitions
230231
)
231-
assert len(remote_envs_timeout.keys()) == 1
232+
assert len(remote_envs_timeout.keys()) == 2
232233
# we have the default timeout + the one specified
233234
assert list(remote_envs_timeout.values())[0] == 600 + 1200
235+
236+
237+
def test_get_testfiles_to_process():
238+
test_glob_pattern_all = "./tests/test_data/benchmark_definitions/*.yml"
239+
test_glob_pattern_graph500 = "./tests/test_data/benchmark_definitions/graph500*.yml"
240+
test_files_to_process = get_testfiles_to_process(
241+
test_glob_pattern_all, "", "defaults.yml"
242+
)
243+
assert 6 == len(test_files_to_process)
244+
test_files_to_process_graph500_glob = get_testfiles_to_process(
245+
test_glob_pattern_graph500, "", "defaults.yml"
246+
)
247+
assert 3 == len(test_files_to_process_graph500_glob)
248+
test_files_to_process = get_testfiles_to_process(
249+
"./tests/test_data/benchmark_definitions/*.yml",
250+
"",
251+
"defaults.yml",
252+
"graph500.+.yml",
253+
)
254+
assert 3 == len(test_files_to_process)
255+
assert test_files_to_process_graph500_glob == test_files_to_process
256+
257+
test_files_to_process = get_testfiles_to_process(
258+
"./tests/test_data/benchmark_definitions/*.yml",
259+
"",
260+
"defaults.yml",
261+
"^(?!.*graph500).*.yml",
262+
)
263+
assert 3 == len(test_files_to_process)
264+
for test_graph in test_files_to_process_graph500_glob:
265+
assert test_graph not in test_files_to_process

‎tests/test_data/benchmark_definitions/defaults.yml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
version: 0.1
2+
kpis:
3+
- eq: { $.Totals.Total.Errors: 0 }
4+
exporter:
5+
redistimeseries:
6+
timemetric: "$.StartTime"
7+
metrics:
8+
- "$.OverallClientLatencies.Total.q50"
9+
- "$.OverallClientLatencies.Total.q95"
10+
- "$.OverallClientLatencies.Total.q99"
11+
- "$.OverallClientLatencies.Total.avg"
12+
- "$.OverallGraphInternalLatencies.Total.q50"
13+
- "$.OverallGraphInternalLatencies.Total.q95"
14+
- "$.OverallGraphInternalLatencies.Total.q99"
15+
- "$.OverallGraphInternalLatencies.Total.avg"
16+
- "$.OverallRelativeInternalExternalLatencyDiff.avg"
17+
- "$.OverallRelativeInternalExternalLatencyDiff.q50"
18+
- "$.OverallQueryRates.Total"
19+
comparison:
20+
metrics:
21+
- "$.OverallQueryRates.Total"
22+
mode: higher-better
23+
baseline-branch: master

‎tests/test_data/benchmark_definitions/explicit_edge_deletion.yml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: "EXPLICIT-EDGE-DELETION"
2+
remote:
3+
- setup: redisgraph-r5
4+
- type: oss-standalone
5+
dbconfig:
6+
- init_commands:
7+
- '"GRAPH.QUERY" "g" "UNWIND range(0, 1000000) AS x CREATE (:N)-[:R]->(:N)"'
8+
clientconfig:
9+
- tool: redisgraph-benchmark-go
10+
- parameters:
11+
- graph: "g"
12+
- rps: 0
13+
- clients: 1
14+
- threads: 4
15+
- connections: 1
16+
- requests: 1000
17+
- queries:
18+
- { q: "MATCH (n:N)-[e]->() DELETE e RETURN 1 LIMIT 1", ratio: 1 }
19+
kpis:
20+
- le: { $.OverallGraphInternalLatencies.Total.q50: 40.0 }

‎tests/test_data/benchmark_definitions/graph500-scale18-ef16_1hop.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: "GRAPH500-SCALE_18-EF_16-1_HOP"
2+
description: "Dataset: Synthetic graph500 network of scale 18 (262144x262144, 4194304 edges)
3+
- 262017 nodes with label 'Node'
4+
- 4194304 relations of type 'IS_CONNECTED'
5+
- Indexed properties:
6+
- exact-match: Node; [external_id]
7+
"
8+
remote:
9+
- setup: redisgraph-r5
10+
- type: oss-standalone
11+
timeout_seconds: 3600
12+
dbconfig:
13+
- dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisgraph/datasets/graph500-scale18-ef16_v2.4.7_dump.rdb"
14+
- dataset_load_timeout_secs: 180
15+
clientconfig:
16+
- tool: redisgraph-benchmark-go
17+
- parameters:
18+
- graph: "graph500-scale18-ef16"
19+
- rps: 0
20+
- clients: 32
21+
- threads: 4
22+
- connections: 32
23+
- requests: 1000000
24+
- random-int-max: 262016
25+
- random-seed: 12345
26+
- queries:
27+
- { q: "MATCH (n)-[:IS_CONNECTED]->(z) WHERE ID(n) = __rand_int__ RETURN ID(n), count(z)", ratio: 1.0 }
28+
kpis:
29+
- le: { $.OverallClientLatencies.Total.q50: 5.0 }
30+
- ge: { $.OverallQueryRates.Total: 8000 }

‎tests/test_data/benchmark_definitions/graph500-scale18-ef16_1hop_MIXED_READ_65_WRITE_25p_DEL_10p.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
name: "GRAPH500-SCALE_18-EF_16-1_HOP_MIXED_READ_65perc_WRITE_25perc_DEL_10perc"
2+
description: "Test: tests mixed read+write+delete queries within the same benchmark
3+
Dataset: Synthetic graph500 network of scale 18 (262144x262144, 4194304 edges)
4+
- 262017 nodes with label 'Node'
5+
- 4194304 relations of type 'IS_CONNECTED'
6+
- Indexed properties:
7+
- exact-match: Node; [external_id]
8+
"
9+
remote:
10+
- setup: redisgraph-r5
11+
- type: oss-standalone
12+
dbconfig:
13+
- dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisgraph/datasets/graph500-scale18-ef16_v2.4.7_dump.rdb"
14+
- dataset_load_timeout_secs: 180
15+
clientconfig:
16+
- tool: redisgraph-benchmark-go
17+
- parameters:
18+
- graph: "graph500-scale18-ef16"
19+
- rps: 0
20+
- clients: 32
21+
- threads: 4
22+
- connections: 32
23+
- requests: 10000
24+
- random-int-max: 262016
25+
- random-seed: 12345
26+
- queries:
27+
- { q: "MATCH (n)-[:IS_CONNECTED]->(z) WHERE ID(n) = __rand_int__ RETURN ID(n), count(z)", ratio: 0.65 }
28+
- { q: "CYPHER Id1=__rand_int__ Id2=__rand_int__ MATCH (n1:Node {external_id: $Id1}) MATCH (n2:Node {external_id: $Id2}) MERGE(n1)-[rel:IS_CONNECTED]->(n2)", ratio: 0.25 }
29+
- { q: "CYPHER Id1=__rand_int__ Id2=__rand_int__ MATCH (n1:Node {external_id: $Id1})-[rel:IS_CONNECTED]->(n2:Node {external_id: $Id2}) DELETE rel", ratio: 0.10 }

‎tests/test_data/benchmark_definitions/graph500-scale18-ef16_1hop_MIXED_READ_75_WRITE_25p.yml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
name: "GRAPH500-SCALE_18-EF_16-1_HOP_MIXED_READ_75perc_WRITE_25perc"
2+
description: "Dataset: Synthetic graph500 network of scale 18 (262144x262144, 4194304 edges)
3+
- 262017 nodes with label 'Node'
4+
- 4194304 relations of type 'IS_CONNECTED'
5+
- Indexed properties:
6+
- exact-match: Node; [external_id]
7+
"
8+
remote:
9+
- setup: redisgraph-r5
10+
- type: oss-standalone
11+
dbconfig:
12+
- dataset: "https://s3.amazonaws.com/benchmarks.redislabs/redisgraph/datasets/graph500-scale18-ef16_v2.4.7_dump.rdb"
13+
- dataset_load_timeout_secs: 180
14+
clientconfig:
15+
- tool: redisgraph-benchmark-go
16+
- parameters:
17+
- graph: "graph500-scale18-ef16"
18+
- rps: 0
19+
- clients: 32
20+
- threads: 4
21+
- connections: 32
22+
- requests: 10000
23+
- random-int-max: 262016
24+
- random-seed: 12345
25+
- queries:
26+
- { q: "MATCH (n)-[:IS_CONNECTED]->(z) WHERE ID(n) = __rand_int__ RETURN ID(n), count(z)", ratio: 0.75 }
27+
- { q: "CYPHER Id1=__rand_int__ Id2=__rand_int__ MATCH (n1:Node {external_id: $Id1}) MATCH (n2:Node {external_id: $Id2}) MERGE (n1)-[rel:IS_CONNECTED]->(n2)", ratio: 0.25 }

0 commit comments

Comments
 (0)