Skip to content

Commit df55761

Browse files
create the trt_api test suite for tensorrt api level test case (#4715)
1 parent 2ace5ac commit df55761

23 files changed

Lines changed: 246 additions & 57 deletions

‎.github/workflows/_decide.yml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ name: _decide
66
#
77
# lane: fast (PR push) | full (approval / `ci: full` / main push) |
88
# nightly (nightly or release-branch push) | skip (non-approval review)
9+
# suite: trt-api when the opt-in `ci:trt-api` label is present; otherwise empty
910
# backend: standard | rtx | both — from the `backend: TensorRT[-RTX]` labels
1011
# (or the workflow_dispatch `backend` input)
1112

@@ -18,13 +19,17 @@ on:
1819
backend:
1920
description: "standard | rtx | both"
2021
value: ${{ jobs.decide.outputs.backend }}
22+
suite:
23+
description: "Explicit manual suite, or empty for lane selection"
24+
value: ${{ jobs.decide.outputs.suite }}
2125

2226
jobs:
2327
decide:
2428
runs-on: ubuntu-latest
2529
outputs:
2630
lane: ${{ steps.pick.outputs.lane }}
2731
backend: ${{ steps.pick.outputs.backend }}
32+
suite: ${{ steps.pick.outputs.suite }}
2833
steps:
2934
- id: pick
3035
env:
@@ -36,6 +41,7 @@ jobs:
3641
# smoke) on PR pushes. (The old `Force All Tests[L0+L1+L2]` label is gone.)
3742
HAS_FULL_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: full') }}"
3843
HAS_NIGHTLY_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci: nightly') }}"
44+
HAS_TRT_API_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'ci:trt-api') }}"
3945
# Backend labels narrow the engine; on a full/nightly run their absence
4046
# means BOTH. Exact match: 'backend: TensorRT' != 'backend: TensorRT-RTX'.
4147
HAS_RTX_LABEL: "${{ contains(github.event.pull_request.labels.*.name, 'backend: TensorRT-RTX') }}"
@@ -62,6 +68,11 @@ jobs:
6268
*) lane=fast ;;
6369
esac
6470
echo "lane=$lane" >> "$GITHUB_OUTPUT"
71+
suite=""
72+
if [ "$EVENT" = "pull_request" ] && [ "$HAS_TRT_API_LABEL" = "true" ]; then
73+
suite=trt-api
74+
fi
75+
echo "suite=$suite" >> "$GITHUB_OUTPUT"
6576
# Backend: explicit labels win; otherwise a fast PR push stays
6677
# standard-only (cheap), while full/nightly default to BOTH engines.
6778
case "$EVENT" in
@@ -70,9 +81,10 @@ jobs:
7081
if [ "$HAS_RTX_LABEL" = "true" ] && [ "$HAS_STD_LABEL" = "true" ]; then backend=both
7182
elif [ "$HAS_RTX_LABEL" = "true" ]; then backend=rtx
7283
elif [ "$HAS_STD_LABEL" = "true" ]; then backend=standard
84+
elif [ -n "$suite" ]; then backend=both
7385
elif [ "$lane" = "fast" ]; then backend=standard
7486
else backend=both; fi ;;
7587
*) backend=both ;; # push / approval / schedule
7688
esac
7789
echo "backend=$backend" >> "$GITHUB_OUTPUT"
78-
echo "Resolved lane='$lane' backend='$backend' (event=$EVENT)."
90+
echo "Resolved lane='$lane' suite='$suite' backend='$backend' (event=$EVENT)."

‎.github/workflows/_test-linux.yml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@ on:
1717
description: "fast | full | nightly"
1818
required: true
1919
type: string
20+
suite:
21+
description: "Explicit named suite; empty selects by lane"
22+
type: string
23+
default: ""
2024
use-rtx:
2125
description: "Test against TensorRT-RTX (x86_64 only)"
2226
type: boolean
@@ -126,9 +130,14 @@ jobs:
126130
run: |
127131
set -euo pipefail
128132
variant=${{ inputs.use-rtx && 'rtx' || 'standard' }}
129-
json="$(python -m tests.ci matrix --platform linux-x86_64 --lane '${{ inputs.lane }}' --variant "${variant}")"
133+
suite='${{ inputs.suite }}'
134+
if [ -n "${suite}" ]; then
135+
json="$(python -m tests.ci matrix --platform linux-x86_64 --suite "${suite}" --variant "${variant}")"
136+
else
137+
json="$(python -m tests.ci matrix --platform linux-x86_64 --lane '${{ inputs.lane }}' --variant "${variant}")"
138+
fi
130139
echo "matrix=${json}" >> "$GITHUB_OUTPUT"
131-
echo "Lane '${{ inputs.lane }}' (${variant}) suites:"; echo "${json}" | python -m json.tool
140+
echo "Selection '${suite:-${{ inputs.lane }}}' (${variant}) suites:"; echo "${json}" | python -m json.tool
132141
133142
# One test job per suite (auto-skips when suite-matrix is skipped, i.e. SBSA).
134143
test:

‎.github/workflows/_test-windows.yml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ on:
1313
description: "fast | full | nightly"
1414
required: true
1515
type: string
16+
suite:
17+
description: "Explicit named suite; empty selects by lane"
18+
type: string
19+
default: ""
1620
use-rtx:
1721
description: "Test against TensorRT-RTX"
1822
type: boolean
@@ -123,9 +127,14 @@ jobs:
123127
run: |
124128
set -euo pipefail
125129
variant=${{ inputs.use-rtx && 'rtx' || 'standard' }}
126-
json="$(python -m tests.ci matrix --platform windows --lane '${{ inputs.lane }}' --variant "${variant}")"
130+
suite='${{ inputs.suite }}'
131+
if [ -n "${suite}" ]; then
132+
json="$(python -m tests.ci matrix --platform windows --suite "${suite}" --variant "${variant}")"
133+
else
134+
json="$(python -m tests.ci matrix --platform windows --lane '${{ inputs.lane }}' --variant "${variant}")"
135+
fi
127136
echo "matrix=${json}" >> "$GITHUB_OUTPUT"
128-
echo "Lane '${{ inputs.lane }}' (${variant}) windows suites:"; echo "${json}" | python -m json.tool
137+
echo "Selection '${suite:-${{ inputs.lane }}}' (${variant}) windows suites:"; echo "${json}" | python -m json.tool
129138
130139
test:
131140
needs: [substitute-runner, build, suite-matrix]

‎.github/workflows/ci-linux-x86_64.yml‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ concurrency:
3232
# Non-CI labels and non-approved reviews get isolated groups, so automation and
3333
# review comments cannot cancel the active pipeline. CI-control labels, approved
3434
# reviews, and real pushes share the PR group and supersede it cleanly.
35-
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'pull_request_review' && github.event.review.state != 'approved' && format('-review-{0}', github.event.review.id) || '' }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}"
35+
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'pull_request_review' && github.event.review.state != 'approved' && format('-review-{0}', github.event.review.id) || '' }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"ci:trt-api\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}"
3636
cancel-in-progress: true
3737

3838
permissions:
@@ -46,7 +46,7 @@ jobs:
4646
# start a full run once ci:full is present. Non-label events always proceed.
4747
if: >-
4848
github.event.action != 'labeled' ||
49-
contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name)
49+
contains(fromJSON('["ci: full", "ci: nightly", "ci:trt-api", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name)
5050
uses: ./.github/workflows/_decide.yml
5151

5252
# Generate the build matrix ONCE. generate_binary_build_matrix's concurrency
@@ -72,6 +72,7 @@ jobs:
7272
uses: ./.github/workflows/_test-linux.yml
7373
with:
7474
lane: ${{ needs.decide.outputs.lane }}
75+
suite: ${{ needs.decide.outputs.suite }}
7576
use-rtx: false
7677
raw-matrix: ${{ needs.generate-matrix.outputs.matrix }}
7778

@@ -81,6 +82,7 @@ jobs:
8182
uses: ./.github/workflows/_test-linux.yml
8283
with:
8384
lane: ${{ needs.decide.outputs.lane }}
85+
suite: ${{ needs.decide.outputs.suite }}
8486
use-rtx: true
8587
name-prefix: "RTX - "
8688
raw-matrix: ${{ needs.generate-matrix.outputs.matrix }}
@@ -89,7 +91,7 @@ jobs:
8991
# so the wheel builds cheaply and each push smoke-tests the no-C++-runtime path.
9092
python-only:
9193
needs: [decide, generate-matrix]
92-
if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'rtx'
94+
if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.suite == '' && needs.decide.outputs.backend != 'rtx'
9395
uses: ./.github/workflows/_test-linux.yml
9496
with:
9597
lane: python-only
@@ -101,7 +103,7 @@ jobs:
101103
# Runs on any non-skip lane, but only when an RTX backend is selected.
102104
python-only-rtx:
103105
needs: [decide, generate-matrix]
104-
if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.backend != 'standard'
106+
if: needs.decide.outputs.lane != 'skip' && needs.decide.outputs.suite == '' && needs.decide.outputs.backend != 'standard'
105107
uses: ./.github/workflows/_test-linux.yml
106108
with:
107109
lane: python-only
@@ -129,6 +131,7 @@ jobs:
129131
needs.decide.result == 'success' &&
130132
needs.generate-matrix.result == 'success' &&
131133
needs.decide.outputs.lane != 'skip' &&
134+
needs.decide.outputs.suite == '' &&
132135
needs.decide.outputs.backend != 'rtx' &&
133136
needs.standard.result != 'cancelled'
134137
uses: ./.github/workflows/executorch-test-linux.yml

‎.github/workflows/ci-windows.yml‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ concurrency:
3232
# Non-CI labels and non-approved reviews get isolated groups, so automation and
3333
# review comments cannot cancel the active pipeline. CI-control labels, approved
3434
# reviews, and real pushes share the PR group and supersede it cleanly.
35-
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'pull_request_review' && github.event.review.state != 'approved' && format('-review-{0}', github.event.review.id) || '' }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}"
35+
group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}${{ github.event_name == 'pull_request_review' && github.event.review.state != 'approved' && format('-review-{0}', github.event.review.id) || '' }}${{ (github.event.action == 'labeled' || github.event.action == 'unlabeled') && !contains(fromJSON('[\"ci: full\", \"ci: nightly\", \"ci:trt-api\", \"backend: TensorRT\", \"backend: TensorRT-RTX\"]'), github.event.label.name) && format('-label-{0}', github.event.label.name) || '' }}"
3636
cancel-in-progress: true
3737

3838
permissions:
@@ -46,7 +46,7 @@ jobs:
4646
# start a full run once ci:full is present. Non-label events always proceed.
4747
if: >-
4848
github.event.action != 'labeled' ||
49-
contains(fromJSON('["ci: full", "ci: nightly", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name)
49+
contains(fromJSON('["ci: full", "ci: nightly", "ci:trt-api", "backend: TensorRT", "backend: TensorRT-RTX"]'), github.event.label.name)
5050
uses: ./.github/workflows/_decide.yml
5151

5252
# Generate the windows build matrix once (shared across channels;
@@ -66,28 +66,30 @@ jobs:
6666

6767
standard:
6868
needs: [decide, generate-matrix]
69-
if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx'
69+
if: (needs.decide.outputs.suite != '' || needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx'
7070
uses: ./.github/workflows/_test-windows.yml
7171
with:
7272
lane: ${{ needs.decide.outputs.lane }}
73+
suite: ${{ needs.decide.outputs.suite }}
7374
build-flavor: standard
7475
use-rtx: false
7576
raw-matrix: ${{ needs.generate-matrix.outputs.matrix }}
7677

7778
rtx:
7879
needs: [decide, generate-matrix]
79-
if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard'
80+
if: (needs.decide.outputs.suite != '' || needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard'
8081
uses: ./.github/workflows/_test-windows.yml
8182
with:
8283
lane: ${{ needs.decide.outputs.lane }}
84+
suite: ${{ needs.decide.outputs.suite }}
8385
build-flavor: rtx
8486
use-rtx: true
8587
name-prefix: "RTX - "
8688
raw-matrix: ${{ needs.generate-matrix.outputs.matrix }}
8789

8890
python-only:
8991
needs: [decide, generate-matrix]
90-
if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx'
92+
if: needs.decide.outputs.suite == '' && (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'rtx'
9193
uses: ./.github/workflows/_test-windows.yml
9294
with:
9395
lane: python-only
@@ -99,7 +101,7 @@ jobs:
99101
# python-only against TensorRT-RTX (so backend=both runs BOTH python-only variants).
100102
python-only-rtx:
101103
needs: [decide, generate-matrix]
102-
if: (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard'
104+
if: needs.decide.outputs.suite == '' && (needs.decide.outputs.lane == 'full' || needs.decide.outputs.lane == 'nightly') && needs.decide.outputs.backend != 'standard'
103105
uses: ./.github/workflows/_test-windows.yml
104106
with:
105107
lane: python-only

‎pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,8 @@ markers = [
158158
"critical: a smoke/critical-path case run in the L1 tier (pytest -m critical); the L2 tier runs the complement (-m 'not critical')",
159159
"smoke: the fast-lane subset of a subsystem (pytest -m smoke); run on every push by the `fast` lane",
160160
"flaky: a known-flaky test, quarantined out of the gating lanes and tracked by an issue; still run in nightly",
161+
"trt_api: directly validates a TensorRT or TensorRT-RTX API contract used by Torch-TensorRT",
162+
"trt_rtx_only: requires the TensorRT-RTX backend",
161163
]
162164
norecursedirs = [
163165
"bazel-*",

‎tests/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Torch-TensorRT test infrastructure."""

‎tests/ci/__main__.py‎

Lines changed: 43 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@
66
list all suites, tiers, lanes, variants
77
show <name> a suite's resolved command per variant
88
run <name> [opts] [-- ...] run one suite (the call CI + just both make)
9-
matrix [--lane|--tier] JSON matrix `include` for GitHub Actions
9+
matrix [--lane|--tier|--suite] JSON matrix `include` for GitHub Actions
1010
doctor validate the manifest (CI lints this)
1111
"""
1212

1313
from __future__ import annotations
1414

1515
import argparse
1616
import json
17+
import re
1718
import sys
1819

1920
from .runner import REPO_ROOT, describe, junit_path, matrix, run_suite, select
@@ -26,9 +27,10 @@ def _cmd_list(_: argparse.Namespace) -> int:
2627
f"{'SUITE'.ljust(width)} TIER LANES VARIANTS PLATFORMS"
2728
)
2829
for s in SUITES:
30+
lane_display = "manual" if s.manual else ",".join(s.lanes)
2931
print(
3032
f"{s.name.ljust(width)} {s.tier:<4} "
31-
f"{','.join(s.lanes):<21} {','.join(s.variants):<15} {','.join(s.platforms)}"
33+
f"{lane_display:<21} {','.join(s.variants):<15} {','.join(s.platforms)}"
3234
)
3335
print(
3436
f"\n{len(SUITES)} suites. "
@@ -39,7 +41,8 @@ def _cmd_list(_: argparse.Namespace) -> int:
3941

4042
def _cmd_show(args: argparse.Namespace) -> int:
4143
s = by_name(args.name)
42-
print(f"# {s.name} (tier={s.tier}, lanes={','.join(s.lanes)})")
44+
lane_display = "manual" if s.manual else ",".join(s.lanes)
45+
print(f"# {s.name} (tier={s.tier}, lanes={lane_display})")
4346
for var in s.variants:
4447
print(f"\n## variant: {var} junit: {junit_path(s).name}")
4548
print(describe(s, var))
@@ -100,6 +103,7 @@ def _cmd_matrix(args: argparse.Namespace) -> int:
100103
include = matrix(
101104
lane=args.lane,
102105
tier=args.tier,
106+
names=[args.suite] if args.suite else None,
103107
variant=args.variant,
104108
platform=args.platform,
105109
changed=_read_changed(args),
@@ -112,7 +116,7 @@ def _cmd_matrix(args: argparse.Namespace) -> int:
112116

113117
def _cmd_doctor(_: argparse.Namespace) -> int:
114118
"""Static checks CI can gate on: unique names, unique junit paths, valid setup
115-
steps, declared cwd dirs exist, every suite is reachable by some lane."""
119+
steps, declared cwd dirs exist, and scheduled suites have at least one lane."""
116120
problems: list[str] = []
117121
names = [s.name for s in SUITES]
118122
dupes = {n for n in names if names.count(n) > 1}
@@ -129,7 +133,9 @@ def _cmd_doctor(_: argparse.Namespace) -> int:
129133
for step in s.setup:
130134
if step not in valid_setup:
131135
problems.append(f"{s.name}: unknown setup step {step!r}")
132-
if not s.lanes:
136+
if s.manual and s.lanes:
137+
problems.append(f"{s.name}: manual suite must not belong to a lane")
138+
if not s.manual and not s.lanes:
133139
problems.append(f"{s.name}: belongs to no lane")
134140
if not s.variants:
135141
problems.append(f"{s.name}: runs on no variant")
@@ -142,7 +148,33 @@ def _cmd_doctor(_: argparse.Namespace) -> int:
142148
if var not in (s.overrides.keys() | {"standard", "rtx"}):
143149
problems.append(f"{s.name}: bad variant {var!r}")
144150

145-
# Every suite should be exercised by some lane and some tier path.
151+
# TRT contract markers imply membership in the manual trt-api umbrella.
152+
# Keep this structural so doctor can run without importing torch/TRT or
153+
# collecting GPU tests.
154+
trt_suite = by_name("trt-api")
155+
trt_root = REPO_ROOT / trt_suite.cwd
156+
trt_paths = tuple(trt_root / path.rstrip("/") for path in trt_suite.paths)
157+
contract_markers = ("trt_api", "trt_rtx_only")
158+
for test_file in (REPO_ROOT / "tests/py").rglob("test_*.py"):
159+
source = test_file.read_text(encoding="utf-8")
160+
used_trt_markers = set(re.findall(r"pytest\.mark\.(trt_[a-z0-9_]+)", source))
161+
unsupported = used_trt_markers.difference(contract_markers)
162+
if unsupported:
163+
problems.append(
164+
f"{test_file.relative_to(REPO_ROOT)}: unsupported fine-grained "
165+
f"TRT markers {sorted(unsupported)}"
166+
)
167+
if not any(f"pytest.mark.{marker}" in source for marker in contract_markers):
168+
continue
169+
if not any(
170+
test_file == path or (path.is_dir() and path in test_file.parents)
171+
for path in trt_paths
172+
):
173+
problems.append(
174+
f"{test_file.relative_to(REPO_ROOT)}: TRT API contract test is "
175+
"not included in trt-api"
176+
)
177+
146178
if problems:
147179
for p in problems:
148180
print(f"✗ {p}", file=sys.stderr)
@@ -199,6 +231,11 @@ def main(argv: list[str] | None = None) -> int:
199231
g = sp.add_mutually_exclusive_group()
200232
g.add_argument("--lane", choices=("fast", "full", "nightly", "python-only"))
201233
g.add_argument("--tier", choices=("l0", "l1", "l2"))
234+
g.add_argument(
235+
"--suite",
236+
choices=tuple(s.name for s in SUITES),
237+
help="select one named suite, including a manual suite",
238+
)
202239
sp.add_argument("--variant", choices=("standard", "rtx"))
203240
sp.add_argument("--platform", choices=("linux-x86_64", "windows"))
204241
sp.add_argument(

0 commit comments

Comments
 (0)