|
| 1 | +# SPDX-License-Identifier: Apache-2.0 |
| 2 | +# SPDX-FileCopyrightText: 2026 devqubit |
| 3 | + |
1 | 4 | """ |
2 | 5 | devqubit: Experiment tracking for quantum computing. |
3 | 6 |
|
|
15 | 18 | ... backend = run.wrap(AerSimulator()) |
16 | 19 | ... job = backend.run(circuit, shots=1000) |
17 | 20 |
|
18 | | -Comparison |
19 | | ----------- |
20 | | ->>> from devqubit import diff |
| 21 | +Comparison & Verification |
| 22 | +------------------------- |
| 23 | +>>> from devqubit.compare import diff, verify_baseline |
21 | 24 | >>> result = diff("run_id_a", "run_id_b") |
22 | 25 | >>> print(result.identical) |
23 | 26 |
|
24 | | -Verification (High-Level) |
25 | | -------------------------- |
26 | | ->>> from devqubit import verify_baseline |
27 | 27 | >>> result = verify_baseline("candidate_run_id", project="my_project") |
28 | 28 | >>> if result.ok: |
29 | 29 | ... print("Verification passed!") |
30 | | ->>> else: |
31 | | -... print(result.verdict.summary) |
32 | | -
|
33 | | -Verification (Custom Policy) |
34 | | ----------------------------- |
35 | | ->>> from devqubit import verify_baseline |
36 | | ->>> from devqubit.compare import VerifyPolicy, ProgramMatchMode |
37 | | ->>> policy = VerifyPolicy( |
38 | | -... program_match_mode=ProgramMatchMode.STRUCTURAL, |
39 | | -... noise_factor=1.2, |
40 | | -... ) |
41 | | ->>> result = verify_baseline("candidate_run_id", project="my_project", policy=policy) |
42 | 30 |
|
43 | 31 | Run Navigation |
44 | 32 | -------------- |
45 | 33 | >>> from devqubit.runs import list_runs, search_runs, get_baseline |
46 | 34 | >>> runs = list_runs(project="my_project", limit=10) |
47 | | ->>> high_fidelity = search_runs("metric.fidelity > 0.95") |
48 | 35 | >>> baseline = get_baseline("my_project") |
49 | 36 |
|
| 37 | +Bundling |
| 38 | +-------- |
| 39 | +>>> from devqubit.bundle import pack_run, unpack_bundle, Bundle |
| 40 | +>>> pack_run("run_id", "experiment.zip") |
| 41 | +>>> with Bundle("experiment.zip") as bundle: |
| 42 | +... print(bundle.run_id) |
| 43 | +
|
50 | 44 | Submodules |
51 | 45 | ---------- |
52 | 46 | - devqubit.runs: Run navigation and baseline management |
53 | | -- devqubit.compare: Comparison types (ProgramMatchMode, Verdict, etc.) |
54 | | -- devqubit.ci: CI/CD integration (JUnit, GitHub annotations) |
| 47 | +- devqubit.compare: Comparison, verification, and diff utilities |
55 | 48 | - devqubit.bundle: Run packaging utilities |
| 49 | +- devqubit.ci: CI/CD integration (JUnit, GitHub annotations) |
56 | 50 | - devqubit.config: Configuration management |
57 | 51 | - devqubit.uec: UEC snapshot schemas |
58 | | -- devqubit.storage: Storage backends |
| 52 | +- devqubit.storage: Storage backends (advanced) |
59 | 53 | - devqubit.adapters: SDK adapter extension API |
60 | 54 | - devqubit.errors: Public exception types |
61 | 55 | - devqubit.ui: Web UI (optional, requires devqubit[ui]) |
|
64 | 58 | from __future__ import annotations |
65 | 59 |
|
66 | 60 | from importlib.metadata import PackageNotFoundError, version |
67 | | -from pathlib import Path |
68 | 61 | from typing import TYPE_CHECKING, Any |
69 | 62 |
|
70 | 63 |
|
71 | 64 | __all__ = [ |
72 | | - # Version |
73 | 65 | "__version__", |
74 | 66 | # Core tracking |
75 | 67 | "Run", |
76 | 68 | "track", |
77 | 69 | "wrap_backend", |
78 | | - # Comparison |
79 | | - "diff", |
80 | | - # Verification |
81 | | - "verify_baseline", |
82 | | - # Bundle |
83 | | - "pack_run", |
84 | | - "unpack_bundle", |
85 | | - "Bundle", |
86 | | - # Config |
| 70 | + # Configuration |
87 | 71 | "Config", |
88 | 72 | "get_config", |
89 | 73 | "set_config", |
|
97 | 81 |
|
98 | 82 |
|
99 | 83 | if TYPE_CHECKING: |
100 | | - from devqubit_engine.bundle.pack import pack_run, unpack_bundle |
101 | | - from devqubit_engine.bundle.reader import Bundle |
102 | | - from devqubit_engine.compare.diff import diff |
103 | | - from devqubit_engine.compare.results import VerifyResult |
104 | | - from devqubit_engine.compare.verify import VerifyPolicy |
105 | 84 | from devqubit_engine.config import Config, get_config, set_config |
106 | | - from devqubit_engine.storage.types import ObjectStoreProtocol, RegistryProtocol |
107 | | - from devqubit_engine.tracking.record import RunRecord |
108 | 85 | from devqubit_engine.tracking.run import Run, track, wrap_backend |
109 | 86 |
|
110 | 87 |
|
|
113 | 90 | "Run": ("devqubit_engine.tracking.run", "Run"), |
114 | 91 | "track": ("devqubit_engine.tracking.run", "track"), |
115 | 92 | "wrap_backend": ("devqubit_engine.tracking.run", "wrap_backend"), |
116 | | - # Comparison |
117 | | - "diff": ("devqubit_engine.compare.diff", "diff"), |
118 | | - # Bundle |
119 | | - "pack_run": ("devqubit_engine.bundle.pack", "pack_run"), |
120 | | - "unpack_bundle": ("devqubit_engine.bundle.pack", "unpack_bundle"), |
121 | | - "Bundle": ("devqubit_engine.bundle.reader", "Bundle"), |
122 | 93 | # Config |
123 | 94 | "Config": ("devqubit_engine.config", "Config"), |
124 | 95 | "get_config": ("devqubit_engine.config", "get_config"), |
125 | 96 | "set_config": ("devqubit_engine.config", "set_config"), |
126 | 97 | } |
127 | 98 |
|
128 | 99 |
|
129 | | -def verify_baseline( |
130 | | - candidate: str | Path | RunRecord, |
131 | | - *, |
132 | | - project: str, |
133 | | - policy: VerifyPolicy | dict[str, Any] | None = None, |
134 | | - store: ObjectStoreProtocol | None = None, |
135 | | - registry: RegistryProtocol | None = None, |
136 | | - promote_on_pass: bool = False, |
137 | | -) -> "VerifyResult": |
138 | | - """ |
139 | | - Verify a candidate run against the stored baseline for a project. |
140 | | -
|
141 | | - This is the recommended high-level API for CI/CD verification. |
142 | | - It automatically loads the candidate run, baseline, and storage |
143 | | - backends from the global configuration. |
144 | | -
|
145 | | - Parameters |
146 | | - ---------- |
147 | | - candidate : str, Path, or RunRecord |
148 | | - Candidate to verify. Can be: |
149 | | - - A run ID (str) |
150 | | - - A path to a bundle file (Path or str ending in .zip) |
151 | | - - A RunRecord instance (already loaded) |
152 | | - project : str |
153 | | - Project name to look up baseline for. |
154 | | - policy : VerifyPolicy or dict or None, optional |
155 | | - Verification policy configuration. Uses defaults if not provided. |
156 | | - Can be a VerifyPolicy instance or a dict with policy options. |
157 | | - store : ObjectStoreProtocol or None, optional |
158 | | - Object store to use. If None, uses the default from config. |
159 | | - Required when candidate is a RunRecord from a different workspace. |
160 | | - registry : RegistryProtocol or None, optional |
161 | | - Registry to use. If None, uses the default from config. |
162 | | - promote_on_pass : bool, default=False |
163 | | - If True and verification passes, promote candidate to new baseline. |
164 | | -
|
165 | | - Returns |
166 | | - ------- |
167 | | - VerifyResult |
168 | | - Verification result with ``ok`` status, ``failures``, ``comparison``, |
169 | | - and ``verdict`` (root-cause analysis if failed). |
170 | | -
|
171 | | - Raises |
172 | | - ------ |
173 | | - ValueError |
174 | | - If no baseline is set for the project and ``allow_missing_baseline`` |
175 | | - is False in the policy. |
176 | | - RunNotFoundError |
177 | | - If the candidate run does not exist. |
178 | | -
|
179 | | - Examples |
180 | | - -------- |
181 | | - Basic verification: |
182 | | -
|
183 | | - >>> from devqubit import verify_baseline |
184 | | - >>> result = verify_baseline("candidate_run_id", project="my_project") |
185 | | - >>> if result.ok: |
186 | | - ... print("Verification passed!") |
187 | | - ... else: |
188 | | - ... print(f"Failed: {result.failures}") |
189 | | - ... print(f"Root cause: {result.verdict.summary}") |
190 | | -
|
191 | | - With custom policy: |
192 | | -
|
193 | | - >>> from devqubit import verify_baseline |
194 | | - >>> from devqubit.compare import VerifyPolicy, ProgramMatchMode |
195 | | - >>> policy = VerifyPolicy( |
196 | | - ... program_match_mode=ProgramMatchMode.STRUCTURAL, |
197 | | - ... noise_factor=1.2, |
198 | | - ... allow_missing_baseline=True, |
199 | | - ... ) |
200 | | - >>> result = verify_baseline( |
201 | | - ... "candidate_run_id", |
202 | | - ... project="my_project", |
203 | | - ... policy=policy, |
204 | | - ... promote_on_pass=True, |
205 | | - ... ) |
206 | | -
|
207 | | - With bundle file: |
208 | | -
|
209 | | - >>> result = verify_baseline( |
210 | | - ... "experiment.zip", |
211 | | - ... project="my_project", |
212 | | - ... ) |
213 | | -
|
214 | | - CI/CD integration: |
215 | | -
|
216 | | - >>> from devqubit import verify_baseline |
217 | | - >>> from devqubit.ci import write_junit |
218 | | - >>> result = verify_baseline("candidate_run_id", project="my_project") |
219 | | - >>> write_junit(result, "results.xml") |
220 | | - >>> assert result.ok, f"Verification failed: {result.failures}" |
221 | | -
|
222 | | - Cross-workspace verification (explicit stores): |
223 | | -
|
224 | | - >>> from devqubit.storage import create_store, create_registry |
225 | | - >>> store = create_store("s3://my-bucket/objects") |
226 | | - >>> registry = create_registry("s3://my-bucket") |
227 | | - >>> result = verify_baseline( |
228 | | - ... "candidate_run_id", |
229 | | - ... project="my_project", |
230 | | - ... store=store, |
231 | | - ... registry=registry, |
232 | | - ... ) |
233 | | - """ |
234 | | - from devqubit_engine.bundle.reader import Bundle, is_bundle_path |
235 | | - from devqubit_engine.compare.verify import ( |
236 | | - verify_against_baseline as _verify_against_baseline, |
237 | | - ) |
238 | | - from devqubit_engine.config import get_config |
239 | | - from devqubit_engine.storage.factory import create_registry, create_store |
240 | | - from devqubit_engine.storage.types import ArtifactRef |
241 | | - from devqubit_engine.tracking.record import RunRecord |
242 | | - |
243 | | - # Get default store/registry from config if not provided |
244 | | - if store is None or registry is None: |
245 | | - cfg = get_config() |
246 | | - if store is None: |
247 | | - store = create_store(config=cfg) |
248 | | - if registry is None: |
249 | | - registry = create_registry(config=cfg) |
250 | | - |
251 | | - # Handle different candidate types |
252 | | - candidate_record: RunRecord |
253 | | - candidate_store = store |
254 | | - |
255 | | - # Case 1: Already a RunRecord |
256 | | - if isinstance(candidate, RunRecord): |
257 | | - candidate_record = candidate |
258 | | - |
259 | | - # Case 2: Bundle file path |
260 | | - elif is_bundle_path(candidate): |
261 | | - with Bundle(Path(candidate)) as bundle: |
262 | | - record_dict = bundle.run_record |
263 | | - artifacts = [ |
264 | | - ArtifactRef.from_dict(a) |
265 | | - for a in record_dict.get("artifacts", []) |
266 | | - if isinstance(a, dict) |
267 | | - ] |
268 | | - candidate_record = RunRecord(record=record_dict, artifacts=artifacts) |
269 | | - # Use bundle's store for artifacts |
270 | | - candidate_store = bundle.store |
271 | | - |
272 | | - return _verify_against_baseline( |
273 | | - candidate_record, |
274 | | - project=project, |
275 | | - store=candidate_store, |
276 | | - registry=registry, |
277 | | - policy=policy, |
278 | | - promote_on_pass=promote_on_pass, |
279 | | - ) |
280 | | - |
281 | | - # Case 3: Run ID string |
282 | | - else: |
283 | | - candidate_record = registry.load(str(candidate)) |
284 | | - |
285 | | - return _verify_against_baseline( |
286 | | - candidate_record, |
287 | | - project=project, |
288 | | - store=candidate_store, |
289 | | - registry=registry, |
290 | | - policy=policy, |
291 | | - promote_on_pass=promote_on_pass, |
292 | | - ) |
293 | | - |
294 | | - |
295 | 100 | def __getattr__(name: str) -> Any: |
296 | 101 | """Lazy import handler for module-level attributes.""" |
297 | 102 | if name in _LAZY_IMPORTS: |
|
0 commit comments