Skip to content

Commit ad065a8

Browse files
vlad-scherbichvlad-scherbich
committed
feat(profiling): add memory allocator testing variants
1 parent de41d2a commit ad065a8

File tree

8 files changed

+291
-14
lines changed

8 files changed

+291
-14
lines changed

‎.riot/requirements/111cc29.txt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.10
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/111cc29.in
6+
#
7+
attrs==25.3.0
8+
coverage[toml]==7.10.6
9+
exceptiongroup==1.3.0
10+
gunicorn==23.0.0
11+
hypothesis==6.45.0
12+
iniconfig==2.1.0
13+
jsonschema==4.25.1
14+
jsonschema-specifications==2025.9.1
15+
lz4==4.4.4
16+
mock==5.2.0
17+
opentracing==2.4.0
18+
packaging==25.0
19+
pluggy==1.6.0
20+
py-cpuinfo==8.0.0
21+
pygments==2.19.2
22+
pytest==8.4.2
23+
pytest-asyncio==0.21.1
24+
pytest-benchmark==5.1.0
25+
pytest-cov==6.3.0
26+
pytest-cpp==2.6.0
27+
pytest-mock==3.15.0
28+
pytest-randomly==3.16.0
29+
referencing==0.36.2
30+
rpds-py==0.27.1
31+
sortedcontainers==2.4.0
32+
tomli==2.2.1
33+
typing-extensions==4.15.0
34+
uwsgi==2.0.29

‎.riot/requirements/133fa5b.txt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.11
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/133fa5b.in
6+
#
7+
attrs==25.3.0
8+
coverage[toml]==7.10.6
9+
gunicorn==23.0.0
10+
hypothesis==6.45.0
11+
iniconfig==2.1.0
12+
jsonschema==4.25.1
13+
jsonschema-specifications==2025.9.1
14+
lz4==4.4.4
15+
mock==5.2.0
16+
opentracing==2.4.0
17+
packaging==25.0
18+
pluggy==1.6.0
19+
py-cpuinfo==8.0.0
20+
pygments==2.19.2
21+
pytest==8.4.2
22+
pytest-asyncio==0.21.1
23+
pytest-benchmark==5.1.0
24+
pytest-cov==6.3.0
25+
pytest-cpp==2.6.0
26+
pytest-mock==3.15.0
27+
pytest-randomly==3.16.0
28+
referencing==0.36.2
29+
rpds-py==0.27.1
30+
sortedcontainers==2.4.0
31+
typing-extensions==4.15.0
32+
uwsgi==2.0.29

‎.riot/requirements/145beea.txt‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.8
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/145beea.in
6+
#
7+
attrs==25.3.0
8+
coverage[toml]==7.6.1
9+
exceptiongroup==1.3.0
10+
gunicorn==23.0.0
11+
hypothesis==6.45.0
12+
importlib-metadata==8.5.0
13+
importlib-resources==6.4.5
14+
iniconfig==2.1.0
15+
jsonschema==4.23.0
16+
jsonschema-specifications==2023.12.1
17+
lz4==4.3.3
18+
mock==5.2.0
19+
opentracing==2.4.0
20+
packaging==25.0
21+
pkgutil-resolve-name==1.3.10
22+
pluggy==1.5.0
23+
py-cpuinfo==8.0.0
24+
pytest==8.3.5
25+
pytest-asyncio==0.21.1
26+
pytest-benchmark==4.0.0
27+
pytest-cov==5.0.0
28+
pytest-cpp==2.6.0
29+
pytest-mock==3.14.1
30+
pytest-randomly==3.15.0
31+
referencing==0.35.1
32+
rpds-py==0.20.1
33+
sortedcontainers==2.4.0
34+
tomli==2.2.1
35+
typing-extensions==4.13.2
36+
uwsgi==2.0.29
37+
zipp==3.20.2

‎.riot/requirements/1e66025.txt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.13
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e66025.in
6+
#
7+
attrs==25.3.0
8+
coverage[toml]==7.10.6
9+
gunicorn==23.0.0
10+
hypothesis==6.45.0
11+
iniconfig==2.1.0
12+
jsonschema==4.25.1
13+
jsonschema-specifications==2025.9.1
14+
lz4==4.4.4
15+
mock==5.2.0
16+
opentracing==2.4.0
17+
packaging==25.0
18+
pluggy==1.6.0
19+
py-cpuinfo==8.0.0
20+
pygments==2.19.2
21+
pytest==8.4.2
22+
pytest-asyncio==0.21.1
23+
pytest-benchmark==5.1.0
24+
pytest-cov==6.3.0
25+
pytest-cpp==2.6.0
26+
pytest-mock==3.15.0
27+
pytest-randomly==3.16.0
28+
referencing==0.36.2
29+
rpds-py==0.27.1
30+
sortedcontainers==2.4.0
31+
uwsgi==2.0.29

‎.riot/requirements/8a26afe.txt‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.9
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/8a26afe.in
6+
#
7+
attrs==25.3.0
8+
coverage[toml]==7.10.6
9+
exceptiongroup==1.3.0
10+
gunicorn==23.0.0
11+
hypothesis==6.45.0
12+
importlib-metadata==8.7.0
13+
iniconfig==2.1.0
14+
jsonschema==4.25.1
15+
jsonschema-specifications==2025.9.1
16+
lz4==4.4.4
17+
mock==5.2.0
18+
opentracing==2.4.0
19+
packaging==25.0
20+
pluggy==1.6.0
21+
py-cpuinfo==8.0.0
22+
pygments==2.19.2
23+
pytest==8.4.2
24+
pytest-asyncio==0.21.1
25+
pytest-benchmark==5.1.0
26+
pytest-cov==6.3.0
27+
pytest-cpp==2.6.0
28+
pytest-mock==3.15.0
29+
pytest-randomly==3.16.0
30+
referencing==0.36.2
31+
rpds-py==0.27.1
32+
sortedcontainers==2.4.0
33+
tomli==2.2.1
34+
typing-extensions==4.15.0
35+
uwsgi==2.0.29
36+
zipp==3.23.0

‎.riot/requirements/ba22367.txt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.12
3+
# by the following command:
4+
#
5+
# pip-compile --allow-unsafe --no-annotate .riot/requirements/ba22367.in
6+
#
7+
attrs==25.3.0
8+
coverage[toml]==7.10.6
9+
gunicorn==23.0.0
10+
hypothesis==6.45.0
11+
iniconfig==2.1.0
12+
jsonschema==4.25.1
13+
jsonschema-specifications==2025.9.1
14+
lz4==4.4.4
15+
mock==5.2.0
16+
opentracing==2.4.0
17+
packaging==25.0
18+
pluggy==1.6.0
19+
py-cpuinfo==8.0.0
20+
pygments==2.19.2
21+
pytest==8.4.2
22+
pytest-asyncio==0.21.1
23+
pytest-benchmark==5.1.0
24+
pytest-cov==6.3.0
25+
pytest-cpp==2.6.0
26+
pytest-mock==3.15.0
27+
pytest-randomly==3.16.0
28+
referencing==0.36.2
29+
rpds-py==0.27.1
30+
sortedcontainers==2.4.0
31+
typing-extensions==4.15.0
32+
uwsgi==2.0.29

‎riotfile.py‎

Lines changed: 88 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,85 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
7979
return [version_to_str(version) for version in SUPPORTED_PYTHON_VERSIONS if min_version <= version <= max_version]
8080

8181

82+
def create_memory_allocator_venvs() -> List[Venv]:
83+
"""Create venv variants for testing different Python memory allocators."""
84+
import ctypes
85+
86+
allocator_configs = [
87+
{
88+
"name_suffix": "pymalloc",
89+
"env": {"PYTHONMALLOC": "pymalloc"},
90+
"description": "Python's default small object allocator"
91+
},
92+
{
93+
"name_suffix": "malloc",
94+
"env": {"PYTHONMALLOC": "malloc"},
95+
"description": "System malloc allocator"
96+
},
97+
{
98+
"name_suffix": "malloc-debug",
99+
"env": {"PYTHONMALLOC": "malloc_debug"},
100+
"description": "System malloc with debug hooks"
101+
},
102+
{
103+
"name_suffix": "pymalloc-debug",
104+
"env": {"PYTHONMALLOC": "pymalloc_debug"},
105+
"description": "Python malloc with debug hooks"
106+
}
107+
]
108+
109+
# Conditionally add jemalloc variant if the library is available
110+
jemalloc_names = [
111+
"libjemalloc.so.2", # Ubuntu/Debian
112+
"libjemalloc.so.1", # Older versions
113+
"libjemalloc.so", # Generic
114+
"libjemalloc.dylib", # macOS (though less common)
115+
]
116+
117+
for lib_name in jemalloc_names:
118+
try:
119+
# Try to load jemalloc library (OSError is raised if not found)
120+
ctypes.CDLL(lib_name)
121+
122+
allocator_configs.append({
123+
"name_suffix": "jemalloc",
124+
"env": {
125+
"PYTHONMALLOC": "malloc",
126+
"LD_PRELOAD": lib_name
127+
},
128+
"description": "jemalloc high-performance allocator"
129+
})
130+
131+
break
132+
except (OSError, AttributeError):
133+
# Library not found or ctypes issue - try next name
134+
continue
135+
136+
venvs = []
137+
for config in allocator_configs:
138+
venvs.append(Venv(
139+
name=f"profile-v2-{config['name_suffix']}",
140+
command="python -m tests.profiling.run pytest -v --no-cov --capture=no --benchmark-disable {cmdargs} tests/profiling_v2",
141+
pys=select_pys(),
142+
env=config["env"],
143+
pkgs=_profile_v2_pkgs
144+
))
145+
146+
return venvs
147+
148+
149+
# Common package configuration for profiling v2 tests
150+
_profile_v2_pkgs = {
151+
"gunicorn": latest,
152+
"jsonschema": latest,
153+
"lz4": latest,
154+
"pytest-cpp": latest,
155+
"pytest-benchmark": latest,
156+
"py-cpuinfo": "~=8.0.0",
157+
"pytest-asyncio": "==0.21.1",
158+
"pytest-randomly": latest,
159+
}
160+
82161
# Common venv configurations for appsec threats testing
83162
_appsec_threats_iast_variants = [
84163
Venv(
@@ -3291,21 +3370,17 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT
32913370
env={
32923371
"DD_PROFILING_ENABLE_ASSERTS": "1",
32933372
"CPUCOUNT": "12",
3373+
# Default values for memory allocator testing - can be overridden with --pass-env
3374+
"PYTHONMALLOC": "pymalloc",
32943375
},
3295-
pkgs={
3296-
"gunicorn": latest,
3297-
"jsonschema": latest,
3298-
"lz4": latest,
3299-
"pytest-cpp": latest,
3300-
#
3301-
# pytest-benchmark depends on cpuinfo which dropped support for Python<=3.6 in 9.0
3302-
# See https://github.com/workhorsy/py-cpuinfo/issues/177
3303-
"pytest-benchmark": latest,
3304-
"py-cpuinfo": "~=8.0.0",
3305-
"pytest-asyncio": "==0.21.1",
3306-
"pytest-randomly": latest,
3307-
},
3376+
pkgs=_profile_v2_pkgs,
33083377
venvs=[
3378+
Venv(
3379+
command="python -m pytest {cmdargs} tests/profiling_v2/test_uwsgi.py",
3380+
pys=select_pys(),
3381+
pkgs={"uwsgi": "<2.0.30"},
3382+
),
3383+
*create_memory_allocator_venvs(),
33093384
# Python 3.8 + 3.9
33103385
Venv(
33113386
pys=["3.8", "3.9"],

‎tests/profiling_v2/collector/test_memalloc.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ def test_memory_collector(tmp_path):
5050
pprof_prefix = str(tmp_path / test_name)
5151
output_filename = pprof_prefix + "." + str(os.getpid())
5252

53-
ddup.config(
53+
ddup.config( # pyright: ignore[reportCallIssue]
5454
service=test_name,
5555
version="test",
5656
env="test",

0 commit comments

Comments
 (0)