test: few basic microbenchmarks and structure around it

README.md

@@ -12,18 +12,6 @@ Approximately, this boils down to:
 - Make deployments resource efficient by tuning various knobs.
 
 
-### Terminology
-
-Speedup is always defined as follows:
-- throughput: after/before. E.g. 20rps / 10rps = 2x
-- latency: before/after. E.g. 20ms / 10ms = 2x
-
-Slowdown can be reported as follows:
-- latency: before/after. E.g. 5ms / 10ms = 0.5x a.k.a "2x slowdown"
-
-Efficiency improvements are defined similarly:
-- memory usage improvement w/ same performance (Pss): before/after. E.g. 200MB/100MB = 2x
-
 ### Contributing
 
 At present, this repo is not accepting any external contributions.

caffeination/microbenchmarks/bench_database.py

@@ -0,0 +1,42 @@
+from functools import lru_cache
+
+import frappe
+
+
+def bench_get_value_simple():
+	status = []
+	for role in get_all_roles():
+		status.append(frappe.db.get_value("Role", role, "disabled"))
+
+
+def bench_get_cached_value_simple():
+	status = []
+	for _ in range(10):
+		for role in get_all_roles():
+			status.append(frappe.db.get_value("Role", role, "disabled", cache=True))
+
+
+def bench_empty_transaction_cycling():
+	frappe.db.rollback()
+	frappe.db.commit()
+
+
+def bench_get_single_value():
+	country = frappe.db.get_single_value("System Settings", "country", cache=False)
+	# Requires Casting
+	telemetry = frappe.db.get_single_value("System Settings", "enable_telemetry", cache=False)
+	return country, telemetry
+
+
+def bench_select_star():
+	kwargs = [{}, {"as_list": True}, {"as_dict": True}]
+	results = []
+	for kw in kwargs:
+		results.append(frappe.db.sql("select * from tabRole limit 10", kw))
+
+	return results
+
+
+@lru_cache
+def get_all_roles():
+	return frappe.get_all("Role", order_by="creation asc", limit=10, pluck="name")

caffeination/microbenchmarks/bench_orm.py

@@ -0,0 +1,39 @@
+from functools import lru_cache
+
+import frappe
+
+
+def bench_get_doc():
+	return [frappe.get_doc("Role", r) for r in get_all_roles()]
+
+
+def bench_get_user():
+	"""Complex version of get_doc - involves child documents to init"""
+	guest = frappe.get_doc("User", "Guest")
+	admin = frappe.get_doc("User", "Administrator")
+	return guest, admin
+
+
+def bench_get_cached_doc():
+	docs = []
+	for role in get_all_roles():
+		doctype = "Role"
+
+		docs.append(frappe.get_cached_doc(doctype, role))
+
+	# Clear "local" cache to avoid testing basically nothing.
+	frappe.local.cache.clear()
+	return docs
+
+
+def bench_get_local_cached_doc():
+	docs = []
+	for role in get_all_roles():
+		doctype = "Role"
+		docs.append(frappe.get_cached_doc(doctype, role))
+	return docs
+
+
+@lru_cache
+def get_all_roles():
+	return frappe.get_all("Role", order_by="creation asc", limit=10, pluck="name")

caffeination/microbenchmarks/bench_redis.py

@@ -0,0 +1,29 @@
+from functools import lru_cache
+
+import frappe
+
+
+def bench_make_key():
+	keys = []
+	for dt in get_all_doctypes():
+		keys.append(frappe.cache.make_key(dt))
+	return keys
+
+
+def bench_set_value():
+	for dt in get_all_doctypes():
+		key = f"_test_set_value:{dt}"
+		frappe.cache.set_value(key, cached_get_doc(dt), expires_in_sec=30)
+		assert frappe.cache.exists(key)
+		assert frappe.cache.get_value(key).name == dt
+	frappe.local.cache.clear()
+
+
+@lru_cache
+def get_all_doctypes():
+	return frappe.get_all("DocType", order_by="creation asc", limit=100, pluck="name")
+
+
+@lru_cache
+def cached_get_doc(doctype):
+	return frappe.get_doc("DocType", doctype)

caffeination/microbenchmarks/run_benchmarks.py

@@ -0,0 +1,48 @@
+#!/bin/env python3
+
+import inspect
+import os
+from types import FunctionType
+
+import frappe
+import pyperf
+
+from caffeination.microbenchmarks import bench_database, bench_orm, bench_redis
+
+BENCHMARK_PREFIX = "bench_"
+BENCHMARK_SITE = os.environ.get("FRAPPE_BENCHMARK_SITE") or "bench.localhost"
+
+
+def run_microbenchmarks():
+	benchmarks = discover_benchmarks()
+
+	frappe.init(BENCHMARK_SITE)
+	frappe.connect()
+
+	runner = pyperf.Runner()
+	for name, func in benchmarks:
+		runner.bench_func(name, func)
+
+	frappe.destroy()
+
+
+def discover_benchmarks():
+	benchmark_modules = [
+		bench_orm,
+		bench_database,
+		bench_redis,
+	]
+
+	benchmarks = []
+	for module in benchmark_modules:
+		module_name = module.__name__.split(".")[-1]
+		for fn_name, fn in inspect.getmembers(module, predicate=lambda x: isinstance(x, FunctionType)):
+			if fn_name.startswith(BENCHMARK_PREFIX):
+				unique_name = f"{module_name}_{fn.__name__.removeprefix(BENCHMARK_PREFIX)}"
+				benchmarks.append((unique_name, fn))
+
+	return sorted(benchmarks, key=lambda x: x[0])
+
+
+if __name__ == "__main__":
+	run_microbenchmarks()

pyproject.toml

@@ -8,7 +8,7 @@ requires-python = ">=3.10"
 readme = "README.md"
 dynamic = ["version"]
 dependencies = [
-    "pyperf==2.8.1" # Installed and managed by bench.
+    "pyperf==2.8.1"
 ]
 
 [build-system]