From 5a8d2be356088a16885cac53a1e1eeef90ff1e47 Mon Sep 17 00:00:00 2001 From: sternj Date: Sun, 11 Dec 2022 21:01:53 -0500 Subject: [PATCH] Added updated expensive benchmarks --- benchmarks/new_benchmark.py | 106 ++++++++++++++++++ test/expensive_benchmarks/bm_async_tree_io.py | 18 ++- test/expensive_benchmarks/bm_docutils.py | 8 +- test/expensive_benchmarks/bm_fannukh.py | 8 +- test/expensive_benchmarks/bm_mdp.py | 6 +- test/expensive_benchmarks/bm_pprint.py | 12 +- test/expensive_benchmarks/bm_raytrace.py | 8 +- test/expensive_benchmarks/bm_sympy.py | 8 +- 8 files changed, 143 insertions(+), 31 deletions(-) create mode 100644 benchmarks/new_benchmark.py diff --git a/benchmarks/new_benchmark.py b/benchmarks/new_benchmark.py new file mode 100644 index 000000000..e656ea055 --- /dev/null +++ b/benchmarks/new_benchmark.py @@ -0,0 +1,106 @@ +import json +import subprocess +import re +import statistics +from glob import glob +from collections import defaultdict +import sys + +cmds = { + # "baseline": ["python3"], + # "scalene": ["python3", "-m", "scalene", "--json", "--outfile", "/dev/null"], + # "scalene-cpu": ["python3", "-m", "scalene", "--json", "--cpu", "--outfile", "/dev/null"], + # "scalene-cpu-gpu": ["python3", "-m", "scalene", "--json", "--cpu", "--gpu", "--outfile", "/dev/null"], + # "scalene-5M": ["python3", "-m", "scalene", "--json", "--outfile", "/dev/null", "--allocation-sampling-window", "5242883"], + # "scalene-10M": ["python3", "-m", "scalene", "--json", "--outfile", "/dev/null", "--allocation-sampling-window", "10485767"], + # "scalene-20M": ["python3", "-m", "scalene", "--json", "--outfile", "/dev/null", "--allocation-sampling-window","20971529"], + # "memray": [ + # "python3", + # "-m", + # "memray", + # "run", + # "--trace-python-allocators", + # "-f", + # "-o", + # "/tmp/memray.out", + # ], + # "fil": ["fil-profile", "-o", "/tmp/abc", '--no-browser', "run"], + # "austin_full": ["austin", "-o", "/dev/null", "-f"], + # "austin_cpu": ["austin", "-o", "/dev/null"], + # 'py-spy': ['py-spy', 'record', '-o', '/tmp/profile.svg', '--', 'python3'], + # 'cProfile': ['python3', '-m', 'cProfile', '-o', '/dev/null'], + 'yappi_wall': ['python3', '-m', 'yappi', '-o', '/dev/null', '-c', 'wall'], + 'yappi_cpu': ['python3', '-m', 'yappi', '-o', '/dev/null', '-c', 'cpu'], + # 'pprofile_det': ['pprofile', '-o', '/dev/null'], + # 'pprofile_stat': ['pprofile', '-o', '/dev/null', '-s', '0.001'], + # 'line_profiler': ['kernprof', '-l', '-o', '/dev/null', '-v'], + # 'profile': ['python3', '-m', 'profile', '-o', '/dev/null'] +} +result_regexp = re.compile(r"Time elapsed:\s+([0-9]*\.[0-9]+)") + + +def main(): + out = defaultdict(lambda : {}) + + for progname in [ + # "./test/expensive_benchmarks/bm_mdp.py", + # "./test/expensive_benchmarks/bm_async_tree_io.py none", + # "./test/expensive_benchmarks/bm_async_tree_io.py io", + # "./test/expensive_benchmarks/bm_async_tree_io.py cpu_io_mixed", + # "./test/expensive_benchmarks/bm_async_tree_io.py memoization", + # "./test/expensive_benchmarks/bm_fannukh.py", + # "./test/expensive_benchmarks/bm_pprint.py", + # "./test/expensive_benchmarks/bm_raytrace.py", + # "./test/expensive_benchmarks/bm_sympy.py", + "./test/expensive_benchmarks/bm_docutils.py" + ]: + for profile_name, profile_cmd in cmds.items(): + times = [] + for i in range(5): + print( + f"Running {profile_name} on {progname} using \"{' '.join(profile_cmd + progname.split(' '))}\"...", + end="", + flush=True, + ) + result = subprocess.run( + profile_cmd + progname.split(' '), + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + ) + + output = result.stdout.decode("utf-8") + # print(output) + match = result_regexp.search(output) + if match is not None: + print(f"... {match.group(1)}", end=('\n' if profile_name != 'memray' else '')) + times.append(round(100 * float(match.group(1))) / 100.0) + if profile_name == 'memray': + res2 = subprocess.run( + ['time', + sys.executable, + '-m', + 'memray', + 'flamegraph', + '-f', + '/tmp/memray.out'], + capture_output=True, + env={'TIME': 'Time elapsed: %e'} + ) + output2 = res2.stderr.decode("utf-8") + match2 = result_regexp.search(output2) + if match2 is not None: + print(f"... {match2.group(1)}") + times[-1] += round(100 * float(match2.group(1))) / 100.0 + else: + print("... RUN FAILED") + # exit(1) + else: + print("RUN FAILED") + # exit(1) + out[profile_name][progname] = times + with open('yappi.json', 'w+') as f: + json.dump(dict(out), f) + + +if __name__ == "__main__": + main() diff --git a/test/expensive_benchmarks/bm_async_tree_io.py b/test/expensive_benchmarks/bm_async_tree_io.py index 47ddd19a6..4608cda39 100755 --- a/test/expensive_benchmarks/bm_async_tree_io.py +++ b/test/expensive_benchmarks/bm_async_tree_io.py @@ -55,13 +55,19 @@ async def recurse(self, recurse_level): await asyncio.gather( *[self.recurse(recurse_level - 1) for _ in range(NUM_RECURSE_BRANCHES)] ) - + async def run(self): - if self.__class__ == IOAsyncTree: + if isinstance(self, IOAsyncTree): num_iters = 9 + elif isinstance(self, MemoizationAsyncTree): + num_iters = 16 + elif isinstance(self, NoneAsyncTree): + num_iters = 22 else: - num_iters = 20 + num_iters = 14 for i in range(num_iters): + if isinstance(self, MemoizationAsyncTree): + self.cache = {} await self.recurse(NUM_RECURSE_LEVELS) @@ -165,9 +171,9 @@ def add_parser_args(parser): async_tree_class = BENCHMARKS[benchmark] async_tree = async_tree_class() - start = time.perf_counter() + start_p = time.perf_counter() asyncio.run(async_tree.run()) - stop = time.perf_counter() - print("Time elapsed: ", stop - start) + stop_p = time.perf_counter() + print("Time elapsed: ", stop_p - start_p) # runner.bench_async_func(f"async_tree_{benchmark}", async_tree.run) diff --git a/test/expensive_benchmarks/bm_docutils.py b/test/expensive_benchmarks/bm_docutils.py index cbe38d765..53b9b329f 100755 --- a/test/expensive_benchmarks/bm_docutils.py +++ b/test/expensive_benchmarks/bm_docutils.py @@ -18,7 +18,7 @@ else: Trace.show = lambda message, channel: ... # don't print to console -DOC_ROOT = (Path(__file__).parent / "docutils_data" / "docs").resolve() +DOC_ROOT = (Path('/home/sam/scalene/test/expensive_benchmarks/bm_docutils.py').parent / "docutils_data" / "docs").resolve() def build_html(doc_root): @@ -48,8 +48,8 @@ def bench_docutils(loops, doc_root): if __name__ == "__main__": # runner = pyperf.Runner() - start = time.perf_counter() + start_p = time.perf_counter() bench_docutils(5, DOC_ROOT) - stop = time.perf_counter() - print("Time elapsed: ", stop - start) + stop_p = time.perf_counter() + print("Time elapsed: ", stop_p - start_p) diff --git a/test/expensive_benchmarks/bm_fannukh.py b/test/expensive_benchmarks/bm_fannukh.py index a827d9e92..faa9d6816 100755 --- a/test/expensive_benchmarks/bm_fannukh.py +++ b/test/expensive_benchmarks/bm_fannukh.py @@ -52,10 +52,10 @@ def fannkuch(n): if __name__ == "__main__": # runner = pyperf.Runner() arg = DEFAULT_ARG - start = time.perf_counter() + start_p = time.perf_counter() # runner.bench_func('fannkuch', fannkuch, arg) - for i in range(2): + for i in range(3): fannkuch(arg) - stop = time.perf_counter() + stop_p = time.perf_counter() - print("Time elapsed: ", stop - start) \ No newline at end of file + print("Time elapsed: ", stop_p - start_p) \ No newline at end of file diff --git a/test/expensive_benchmarks/bm_mdp.py b/test/expensive_benchmarks/bm_mdp.py index b772eb085..fa49fb707 100755 --- a/test/expensive_benchmarks/bm_mdp.py +++ b/test/expensive_benchmarks/bm_mdp.py @@ -266,8 +266,8 @@ def bench_mdp(loops): if __name__ == "__main__": runner = pyperf.Runner() runner.metadata['description'] = "MDP benchmark" - start = time.perf_counter() + start_p = time.perf_counter() # runner.bench_time_func('mdp', bench_mdp) bench_mdp(5) - stop = time.perf_counter() - print("Time elapsed: ", stop - start) + stop_p = time.perf_counter() + print("Time elapsed: ", stop_p - start_p) diff --git a/test/expensive_benchmarks/bm_pprint.py b/test/expensive_benchmarks/bm_pprint.py index e4003eef8..eb42d50e2 100755 --- a/test/expensive_benchmarks/bm_pprint.py +++ b/test/expensive_benchmarks/bm_pprint.py @@ -10,14 +10,14 @@ from pprint import PrettyPrinter -printable = [('string', (1, 2), [3, 4], {5: 6, 7: 8})] * 100_000 * 8 +printable = [('string', (1, 2), [3, 4], {5: 6, 7: 8})] * 100_000 p = PrettyPrinter() if __name__ == '__main__': - start = perf_counter() - # for i in range(7): - p.pformat(printable) - stop = perf_counter() - print("Time elapsed: ", stop - start) + start_p = perf_counter() + for i in range(7): + p.pformat(printable) + stop_p = perf_counter() + print("Time elapsed: ", stop_p - start_p) diff --git a/test/expensive_benchmarks/bm_raytrace.py b/test/expensive_benchmarks/bm_raytrace.py index 8b130b00b..9ea0fd66d 100755 --- a/test/expensive_benchmarks/bm_raytrace.py +++ b/test/expensive_benchmarks/bm_raytrace.py @@ -402,7 +402,7 @@ def add_cmdline_args(cmd, args): cmd.add_argument("--filename", metavar="FILENAME.PPM", help="Output filename of the PPM picture") args = cmd.parse_args() - start = perf_counter() - bench_raytrace(20, args.width, args.height, args.filename) - stop = perf_counter() - print("Time elapsed: ", stop - start) + start_p = perf_counter() + bench_raytrace(25, args.width, args.height, args.filename) + stop_p = perf_counter() + print("Time elapsed: ", stop_p - start_p) diff --git a/test/expensive_benchmarks/bm_sympy.py b/test/expensive_benchmarks/bm_sympy.py index 2a8def457..a04e88657 100755 --- a/test/expensive_benchmarks/bm_sympy.py +++ b/test/expensive_benchmarks/bm_sympy.py @@ -56,11 +56,11 @@ def add_cmdline_args(cmd, args): import gc gc.disable() - start = time.perf_counter() - for _ in range(20): + start_p = time.perf_counter() + for _ in range(25): # Don't benchmark clear_cache(), exclude it of the benchmark clear_cache() bench_expand() - stop = time.perf_counter() - print("Time elapsed: ", stop - start) + stop_p = time.perf_counter() + print("Time elapsed: ", stop_p - start_p)