Skip to content

Commit 1d473b6

Browse files
sentrivanasentrivana
authored
toxgen: Add variants & move OpenAI under toxgen (#4730)
Adds supports for variants, i.e., the same test suite running with a slightly different setup (for instance, a different set of dependencies, like `openai` and `openai_notiktoken`). To add a variant, simply add a new test suite to the config. The tricky part is naming. I had to rename `openai` to `openai_base` since otherwise the `openai_notiktoken` and `openai_agents` test suite would be run with `tox -e py<version>-openai` / `./scripts/runtox.sh py<version>-openai` due to how tox works. They should be treated as three different suites. Closes #4507
1 parent 173df64 commit 1d473b6

File tree

7 files changed

+77
-53
lines changed

7 files changed

+77
-53
lines changed

‎.github/workflows/test-integrations-ai.yml

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,14 @@ jobs:
6262
run: |
6363
set -x # print commands that are executed
6464
./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-latest"
65-
- name: Test openai latest
65+
- name: Test openai_base latest
6666
run: |
6767
set -x # print commands that are executed
68-
./scripts/runtox.sh "py${{ matrix.python-version }}-openai-latest"
68+
./scripts/runtox.sh "py${{ matrix.python-version }}-openai_base-latest"
69+
- name: Test openai_notiktoken latest
70+
run: |
71+
set -x # print commands that are executed
72+
./scripts/runtox.sh "py${{ matrix.python-version }}-openai_notiktoken-latest"
6973
- name: Test openai_agents latest
7074
run: |
7175
set -x # print commands that are executed
@@ -141,10 +145,14 @@ jobs:
141145
run: |
142146
set -x # print commands that are executed
143147
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain"
144-
- name: Test openai pinned
148+
- name: Test openai_base pinned
149+
run: |
150+
set -x # print commands that are executed
151+
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai_base"
152+
- name: Test openai_notiktoken pinned
145153
run: |
146154
set -x # print commands that are executed
147-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai"
155+
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai_notiktoken"
148156
- name: Test openai_agents pinned
149157
run: |
150158
set -x # print commands that are executed

‎scripts/populate_tox/README.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,14 @@ be expressed like so:
153153
}
154154
```
155155

156+
### `integration_name`
157+
158+
Sometimes, the name of the test suite doesn't match the name of the integration.
159+
For example, we have the `openai_base` and `openai_notiktoken` test suites, both
160+
of which are actually testing the `openai` integration. If this is the case, you can use the `integration_name` key to define the name of the integration. If not provided, it will default to the name of the test suite.
161+
162+
Linking an integration to a test suite allows the script to access integration configuration like for example the minimum version defined in `sentry_sdk/integrations/__init__.py`.
163+
156164

157165
## How-Tos
158166

‎scripts/populate_tox/config.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,24 @@
139139
"loguru": {
140140
"package": "loguru",
141141
},
142+
"openai_base": {
143+
"package": "openai",
144+
"integration_name": "openai",
145+
"deps": {
146+
"*": ["pytest-asyncio", "tiktoken"],
147+
"<1.55": ["httpx<0.28"],
148+
},
149+
"python": ">=3.8",
150+
},
151+
"openai_notiktoken": {
152+
"package": "openai",
153+
"integration_name": "openai",
154+
"deps": {
155+
"*": ["pytest-asyncio"],
156+
"<1.55": ["httpx<0.28"],
157+
},
158+
"python": ">=3.8",
159+
},
142160
"openai_agents": {
143161
"package": "openai-agents",
144162
"deps": {

‎scripts/populate_tox/populate_tox.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,6 @@
7676
"httpx",
7777
"langchain",
7878
"langchain_notiktoken",
79-
"openai",
80-
"openai_notiktoken",
8179
"pure_eval",
8280
"quart",
8381
"ray",
@@ -141,7 +139,11 @@ def _prefilter_releases(
141139
- the list of prefiltered releases
142140
- an optional prerelease if there is one that should be tested
143141
"""
144-
min_supported = _MIN_VERSIONS.get(integration)
142+
integration_name = (
143+
TEST_SUITE_CONFIG[integration].get("integration_name") or integration
144+
)
145+
146+
min_supported = _MIN_VERSIONS.get(integration_name)
145147
if min_supported is not None:
146148
min_supported = Version(".".join(map(str, min_supported)))
147149
else:

‎scripts/populate_tox/tox.jinja

Lines changed: 2 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -83,13 +83,6 @@ envlist =
8383
{py3.9,py3.11,py3.12}-langchain-latest
8484
{py3.9,py3.11,py3.12}-langchain-notiktoken
8585

86-
# OpenAI
87-
{py3.9,py3.11,py3.12}-openai-v1.0
88-
{py3.9,py3.11,py3.12}-openai-v1.22
89-
{py3.9,py3.11,py3.12}-openai-v1.55
90-
{py3.9,py3.11,py3.12}-openai-latest
91-
{py3.9,py3.11,py3.12}-openai-notiktoken
92-
9386
# OpenTelemetry (OTel)
9487
{py3.7,py3.9,py3.12,py3.13}-opentelemetry
9588

@@ -252,20 +245,6 @@ deps =
252245
langchain-{latest,notiktoken}: openai>=1.6.1
253246
langchain-latest: tiktoken~=0.6.0
254247
255-
# OpenAI
256-
openai: pytest-asyncio
257-
openai-v1.0: openai~=1.0.0
258-
openai-v1.0: tiktoken
259-
openai-v1.0: httpx<0.28.0
260-
openai-v1.22: openai~=1.22.0
261-
openai-v1.22: tiktoken
262-
openai-v1.22: httpx<0.28.0
263-
openai-v1.55: openai~=1.55.0
264-
openai-v1.55: tiktoken
265-
openai-latest: openai
266-
openai-latest: tiktoken~=0.6.0
267-
openai-notiktoken: openai
268-
269248
# OpenTelemetry (OTel)
270249
opentelemetry: opentelemetry-distro
271250
@@ -401,7 +380,8 @@ setenv =
401380
launchdarkly: TESTPATH=tests/integrations/launchdarkly
402381
litestar: TESTPATH=tests/integrations/litestar
403382
loguru: TESTPATH=tests/integrations/loguru
404-
openai: TESTPATH=tests/integrations/openai
383+
openai_base: TESTPATH=tests/integrations/openai
384+
openai_notiktoken: TESTPATH=tests/integrations/openai
405385
openai_agents: TESTPATH=tests/integrations/openai_agents
406386
openfeature: TESTPATH=tests/integrations/openfeature
407387
opentelemetry: TESTPATH=tests/integrations/opentelemetry

‎scripts/split_tox_gh_actions/split_tox_gh_actions.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@
6262
"anthropic",
6363
"cohere",
6464
"langchain",
65-
"openai",
65+
"openai_base",
66+
"openai_notiktoken",
6667
"openai_agents",
6768
"huggingface_hub",
6869
],

‎tox.ini

Lines changed: 30 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
# The file (and all resulting CI YAMLs) then need to be regenerated via
1111
# "scripts/generate-test-files.sh".
1212
#
13-
# Last generated: 2025-09-01T12:08:33.833560+00:00
13+
# Last generated: 2025-09-01T14:09:50.564158+00:00
1414

1515
[tox]
1616
requires =
@@ -83,13 +83,6 @@ envlist =
8383
{py3.9,py3.11,py3.12}-langchain-latest
8484
{py3.9,py3.11,py3.12}-langchain-notiktoken
8585

86-
# OpenAI
87-
{py3.9,py3.11,py3.12}-openai-v1.0
88-
{py3.9,py3.11,py3.12}-openai-v1.22
89-
{py3.9,py3.11,py3.12}-openai-v1.55
90-
{py3.9,py3.11,py3.12}-openai-latest
91-
{py3.9,py3.11,py3.12}-openai-notiktoken
92-
9386
# OpenTelemetry (OTel)
9487
{py3.7,py3.9,py3.12,py3.13}-opentelemetry
9588

@@ -145,6 +138,16 @@ envlist =
145138
{py3.9,py3.11,py3.12}-cohere-v5.13.12
146139
{py3.9,py3.11,py3.12}-cohere-v5.17.0
147140

141+
{py3.8,py3.11,py3.12}-openai_base-v1.0.1
142+
{py3.8,py3.11,py3.12}-openai_base-v1.35.15
143+
{py3.8,py3.11,py3.12}-openai_base-v1.69.0
144+
{py3.8,py3.12,py3.13}-openai_base-v1.102.0
145+
146+
{py3.8,py3.11,py3.12}-openai_notiktoken-v1.0.1
147+
{py3.8,py3.11,py3.12}-openai_notiktoken-v1.35.15
148+
{py3.8,py3.11,py3.12}-openai_notiktoken-v1.69.0
149+
{py3.8,py3.12,py3.13}-openai_notiktoken-v1.102.0
150+
148151
{py3.10,py3.11,py3.12}-openai_agents-v0.0.19
149152
{py3.10,py3.12,py3.13}-openai_agents-v0.1.0
150153
{py3.10,py3.12,py3.13}-openai_agents-v0.2.10
@@ -423,20 +426,6 @@ deps =
423426
langchain-{latest,notiktoken}: openai>=1.6.1
424427
langchain-latest: tiktoken~=0.6.0
425428

426-
# OpenAI
427-
openai: pytest-asyncio
428-
openai-v1.0: openai~=1.0.0
429-
openai-v1.0: tiktoken
430-
openai-v1.0: httpx<0.28.0
431-
openai-v1.22: openai~=1.22.0
432-
openai-v1.22: tiktoken
433-
openai-v1.22: httpx<0.28.0
434-
openai-v1.55: openai~=1.55.0
435-
openai-v1.55: tiktoken
436-
openai-latest: openai
437-
openai-latest: tiktoken~=0.6.0
438-
openai-notiktoken: openai
439-
440429
# OpenTelemetry (OTel)
441430
opentelemetry: opentelemetry-distro
442431

@@ -527,6 +516,23 @@ deps =
527516
cohere-v5.13.12: cohere==5.13.12
528517
cohere-v5.17.0: cohere==5.17.0
529518

519+
openai_base-v1.0.1: openai==1.0.1
520+
openai_base-v1.35.15: openai==1.35.15
521+
openai_base-v1.69.0: openai==1.69.0
522+
openai_base-v1.102.0: openai==1.102.0
523+
openai_base: pytest-asyncio
524+
openai_base: tiktoken
525+
openai_base-v1.0.1: httpx<0.28
526+
openai_base-v1.35.15: httpx<0.28
527+
528+
openai_notiktoken-v1.0.1: openai==1.0.1
529+
openai_notiktoken-v1.35.15: openai==1.35.15
530+
openai_notiktoken-v1.69.0: openai==1.69.0
531+
openai_notiktoken-v1.102.0: openai==1.102.0
532+
openai_notiktoken: pytest-asyncio
533+
openai_notiktoken-v1.0.1: httpx<0.28
534+
openai_notiktoken-v1.35.15: httpx<0.28
535+
530536
openai_agents-v0.0.19: openai-agents==0.0.19
531537
openai_agents-v0.1.0: openai-agents==0.1.0
532538
openai_agents-v0.2.10: openai-agents==0.2.10
@@ -831,7 +837,8 @@ setenv =
831837
launchdarkly: TESTPATH=tests/integrations/launchdarkly
832838
litestar: TESTPATH=tests/integrations/litestar
833839
loguru: TESTPATH=tests/integrations/loguru
834-
openai: TESTPATH=tests/integrations/openai
840+
openai_base: TESTPATH=tests/integrations/openai
841+
openai_notiktoken: TESTPATH=tests/integrations/openai
835842
openai_agents: TESTPATH=tests/integrations/openai_agents
836843
openfeature: TESTPATH=tests/integrations/openfeature
837844
opentelemetry: TESTPATH=tests/integrations/opentelemetry

0 commit comments

Comments
 (0)