diff --git a/docs/concepts.md b/docs/concepts.md index 7c563b3..63f6cce 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -8,7 +8,7 @@ Python coverage is experimental and degraded-honest. Graph-backed structure, unt ## Findings -Findings are named signals such as `typescript.type_errors`, `rust.source_hygiene`, `python.syntax`, `python.source-hygiene`, `python.types`, `python.import-graph`, `python.dead-code`, `python.relevant-tests`, and `coverage.unsupported_stacks`. Counts come from validation diagnostics, graph evidence when available, optional-tool degradation, and repository census data. Opcore does not blend them into a single rating. +Findings are named signals such as `typescript.type_errors`, `rust.source_hygiene`, `python.syntax`, `python.source-hygiene`, `python.types`, `python.import-graph`, `python.dead-code`, `python.relevant-tests`, `python.pytest`, and `coverage.unsupported_stacks`. Counts come from validation diagnostics, graph evidence when available, optional-tool degradation, and repository census data. Opcore does not blend them into a single rating. ## Artifacts diff --git a/packages/contracts/schemas/opcore-contracts.schema.json b/packages/contracts/schemas/opcore-contracts.schema.json index 8073d8d..eb0bf0b 100644 --- a/packages/contracts/schemas/opcore-contracts.schema.json +++ b/packages/contracts/schemas/opcore-contracts.schema.json @@ -2394,7 +2394,19 @@ "properties": { "schemaId": { "const": "opcore.python.project-context.v1" }, "schemaVersion": { "const": 1 }, - "target": { "allOf": [{ "$ref": "#/$defs/RepoRelativePath" }, { "pattern": "\\.pyi?$" }] }, + "target": { + "allOf": [ + { "$ref": "#/$defs/RepoRelativePath" }, + { + "anyOf": [ + { "pattern": "\\.pyi?$" }, + { + "pattern": "(^|/)(pyproject\\.toml|Pipfile|Pipfile\\.lock|poetry\\.lock|pdm\\.lock|uv\\.lock|setup\\.cfg|setup\\.py|pytest\\.ini|tox\\.ini|pyrightconfig\\.json|ruff\\.toml|\\.ruff\\.toml|mypy\\.ini|\\.mypy\\.ini|conftest\\.py|requirements.*\\.txt)$" + } + ] + } + ] + }, "repositoryRoot": { "type": "string", "minLength": 1 }, "projectRoot": { "$ref": "#/$defs/PythonProjectRoot" }, "projectBoundary": { "$ref": "#/$defs/PythonProjectRoot" }, @@ -2548,6 +2560,102 @@ } ] }, + "PythonCapabilityCounts": { + "type": "object", + "additionalProperties": false, + "required": [ + "candidateCount", + "collectedCount", + "executedCount", + "passedCount", + "failedCount", + "skippedCount", + "xfailedCount", + "xpassedCount", + "errorCount" + ], + "properties": { + "candidateCount": { "type": "integer", "minimum": 0 }, + "collectedCount": { "type": "integer", "minimum": 0 }, + "executedCount": { "type": "integer", "minimum": 0 }, + "passedCount": { "type": "integer", "minimum": 0 }, + "failedCount": { "type": "integer", "minimum": 0 }, + "skippedCount": { "type": "integer", "minimum": 0 }, + "xfailedCount": { "type": "integer", "minimum": 0 }, + "xpassedCount": { "type": "integer", "minimum": 0 }, + "errorCount": { "type": "integer", "minimum": 0 } + } + }, + "PythonCapabilityCleanupEvidence": { + "type": "object", + "additionalProperties": false, + "required": ["attempted", "ok"], + "properties": { + "attempted": { "type": "boolean" }, + "ok": { "type": "boolean" }, + "failureMessage": { "type": "string", "minLength": 1 } + } + }, + "PythonCapabilityInvocation": { + "type": "object", + "additionalProperties": false, + "required": [ + "stage", + "command", + "argsDigest", + "argCount", + "selectionMode", + "durationMs", + "termination", + "outputBytes" + ], + "properties": { + "stage": { "enum": ["collection", "execution"] }, + "command": { "type": "string", "minLength": 1 }, + "argsDigest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "argCount": { "type": "integer", "minimum": 0 }, + "selectionMode": { "enum": ["none", "direct_argv", "manifest"] }, + "selectionDigest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "durationMs": { "type": "number", "minimum": 0 }, + "termination": { "enum": ["exited", "timeout", "signal", "spawn_error"] }, + "exitCode": { "type": "integer", "minimum": 0 }, + "signal": { "type": "string", "minLength": 1 }, + "outputBytes": { "type": "integer", "minimum": 0 }, + "stdoutDigest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "stderrDigest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" } + } + }, + "PythonValidationCapabilityRun": { + "type": "object", + "additionalProperties": false, + "required": ["capability", "checkId", "activation", "outcome", "message"], + "properties": { + "capability": { "const": "pytest" }, + "checkId": { "$ref": "#/$defs/ValidationCheckId" }, + "activation": { "enum": ["enabled", "disabled", "not_applicable"] }, + "outcome": { "type": "string", "minLength": 1 }, + "message": { "type": "string", "minLength": 1 }, + "projectKey": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "projectRoot": { "$ref": "#/$defs/RepoRelativePath" }, + "configFile": { "$ref": "#/$defs/RepoRelativePath" }, + "targetCount": { "type": "integer", "minimum": 0 }, + "candidatePaths": { + "type": "array", + "items": { "$ref": "#/$defs/RepoRelativePath" } + }, + "collectedNodeIds": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + }, + "afterStateFingerprint": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "selectionMode": { "enum": ["none", "direct_argv", "manifest"] }, + "selectionDigest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }, + "counts": { "$ref": "#/$defs/PythonCapabilityCounts" }, + "collection": { "$ref": "#/$defs/PythonCapabilityInvocation" }, + "execution": { "$ref": "#/$defs/PythonCapabilityInvocation" }, + "cleanup": { "$ref": "#/$defs/PythonCapabilityCleanupEvidence" } + } + }, "ValidationResult": { "type": "object", "additionalProperties": false, @@ -2760,6 +2868,10 @@ "type": "array", "items": { "$ref": "#/$defs/PythonProjectContext" } }, + "pythonCapabilityRuns": { + "type": "array", + "items": { "$ref": "#/$defs/PythonValidationCapabilityRun" } + }, "manifest": { "type": "object", "additionalProperties": false, diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index dd5e13f..2fd1b89 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1330,6 +1330,70 @@ export interface ValidationResultManifest { skippedChecks?: readonly ValidationSkippedCheck[]; } +export const pythonCapabilityActivations = ["enabled", "disabled", "not_applicable"] as const; +export type PythonCapabilityActivation = (typeof pythonCapabilityActivations)[number]; + +export const pythonPytestSelectionModes = ["none", "direct_argv", "manifest"] as const; +export type PythonPytestSelectionMode = (typeof pythonPytestSelectionModes)[number]; + +export const pythonCapabilityProcessTerminations = ["exited", "timeout", "signal", "spawn_error"] as const; +export type PythonCapabilityProcessTermination = (typeof pythonCapabilityProcessTerminations)[number]; + +export interface PythonCapabilityCounts { + candidateCount: number; + collectedCount: number; + executedCount: number; + passedCount: number; + failedCount: number; + skippedCount: number; + xfailedCount: number; + xpassedCount: number; + errorCount: number; +} + +export interface PythonCapabilityCleanupEvidence { + attempted: boolean; + ok: boolean; + failureMessage?: string; +} + +export interface PythonCapabilityInvocation { + stage: "collection" | "execution"; + command: string; + argsDigest: string; + argCount: number; + selectionMode: PythonPytestSelectionMode; + selectionDigest?: string; + durationMs: number; + termination: PythonCapabilityProcessTermination; + exitCode?: number; + signal?: string; + outputBytes: number; + stdoutDigest?: string; + stderrDigest?: string; +} + +export interface PythonValidationCapabilityRun { + capability: "pytest"; + checkId: string; + activation: PythonCapabilityActivation; + outcome: string; + message: string; + projectKey?: string; + projectRoot?: string; + configFile?: string; + targetCount?: number; + candidatePaths?: readonly string[]; + collectedNodeIds?: readonly string[]; + afterStateFingerprint?: string; + selectionMode?: PythonPytestSelectionMode; + selectionDigest?: string; + counts?: PythonCapabilityCounts; + collection?: PythonCapabilityInvocation; + execution?: PythonCapabilityInvocation; + cleanup?: PythonCapabilityCleanupEvidence; +} + export interface ValidationResult { ok: boolean; status: ValidationResultStatus; @@ -1339,6 +1403,7 @@ export interface ValidationResult { refusal?: EditRefusal; manifest?: ValidationResultManifest; pythonProjectContexts?: readonly PythonProjectContext[]; + pythonCapabilityRuns?: readonly PythonValidationCapabilityRun[]; } export interface RequiredContextDocPolicy { @@ -6621,9 +6686,65 @@ export function validateValidationResultPayload(result: ValidationResult): Valid validateValidationResultManifest(result.manifest); } if (result.pythonProjectContexts !== undefined) validatePythonProjectContexts(result.pythonProjectContexts); + if (result.pythonCapabilityRuns !== undefined) validatePythonValidationCapabilityRuns(result.pythonCapabilityRuns); return result; } +export function validatePythonValidationCapabilityRuns( + runs: readonly PythonValidationCapabilityRun[] +): readonly PythonValidationCapabilityRun[] { + if (!Array.isArray(runs)) throw new Error("Python capability runs must be an array"); + for (const run of runs) validatePythonValidationCapabilityRun(run); + return runs; +} + +export function validatePythonValidationCapabilityRun(run: PythonValidationCapabilityRun): PythonValidationCapabilityRun { + if (!run || typeof run !== "object") throw new Error("Python capability run is required"); + validateExactObjectKeys(run, [ + "capability", + "checkId", + "activation", + "outcome", + "message", + "projectKey", + "projectRoot", + "configFile", + "targetCount", + "candidatePaths", + "collectedNodeIds", + "afterStateFingerprint", + "selectionMode", + "selectionDigest", + "counts", + "collection", + "execution", + "cleanup" + ], "Python capability run"); + if (run.capability !== "pytest") throw new Error("Python capability run capability must be pytest"); + validateNonEmptyString(run.checkId, "Python capability run checkId"); + if (!includesString(pythonCapabilityActivations, run.activation)) { + throw new Error(`Unknown Python capability activation: ${String(run.activation)}`); + } + validateNonEmptyString(run.outcome, "Python capability run outcome"); + validateNonEmptyString(run.message, "Python capability run message"); + if (run.projectKey !== undefined) validateSha256Identity(run.projectKey, "Python capability run projectKey"); + if (run.projectRoot !== undefined) validatePythonProjectRoot(run.projectRoot, "Python capability run projectRoot"); + if (run.configFile !== undefined) validateRepoRelativePath(run.configFile); + if (run.targetCount !== undefined) validateNonNegativeInteger(run.targetCount, "Python capability run targetCount"); + if (run.candidatePaths !== undefined) validateRepoPathArray(run.candidatePaths, "Python capability run candidatePaths"); + if (run.collectedNodeIds !== undefined) validateStringArray(run.collectedNodeIds, "Python capability run collectedNodeIds", { allowEmpty: true }); + if (run.afterStateFingerprint !== undefined) validateSha256Identity(run.afterStateFingerprint, "Python capability run afterStateFingerprint"); + if (run.selectionMode !== undefined && !includesString(pythonPytestSelectionModes, run.selectionMode)) { + throw new Error(`Unknown Python capability selection mode: ${String(run.selectionMode)}`); + } + if (run.selectionDigest !== undefined) validateSha256Identity(run.selectionDigest, "Python capability run selectionDigest"); + if (run.counts !== undefined) validatePythonCapabilityCounts(run.counts); + if (run.collection !== undefined) validatePythonCapabilityInvocation(run.collection, "Python capability run collection"); + if (run.execution !== undefined) validatePythonCapabilityInvocation(run.execution, "Python capability run execution"); + if (run.cleanup !== undefined) validatePythonCapabilityCleanupEvidence(run.cleanup); + return run; +} + export function validatePythonProjectContext(context: PythonProjectContext): PythonProjectContext { if (!context || typeof context !== "object") throw new Error("Python project context is required"); validateExactObjectKeys(context, [ @@ -6631,31 +6752,56 @@ export function validatePythonProjectContext(context: PythonProjectContext): Pyt "layout", "evidence", "targetRuntime", "managers", "buildSystem", "interpreter", "tools", "projectKey", "contextFingerprint", "outcome", "reasons" ], "Python project context"); + validatePythonProjectContextIdentity(context); + validatePythonProjectContextLayout(context.layout); + validatePythonProjectContextEvidence(context.evidence); + validatePythonProjectTarget(context.targetRuntime); + validatePythonProjectManagerEvidenceEntries(context.managers); + validatePythonProjectBuildSystem(context.buildSystem); + if (context.interpreter !== undefined) validatePythonInterpreterProvenance(context.interpreter); + validatePythonProjectToolEntries(context.tools); + validateSha256Identity(context.projectKey, "Python project context projectKey"); + validateSha256Identity(context.contextFingerprint, "Python project context contextFingerprint"); + validatePythonProjectContextOutcome(context.outcome, context.reasons); + return context; +} + +function validatePythonProjectContextIdentity(context: PythonProjectContext): void { if (context.schemaId !== PYTHON_PROJECT_CONTEXT_SCHEMA_ID) { throw new Error(`Python project context schemaId must be ${PYTHON_PROJECT_CONTEXT_SCHEMA_ID}`); } if (context.schemaVersion !== 1) throw new Error("Python project context schemaVersion must be 1"); validateRepoRelativePath(context.target); - if (!/\.pyi?$/u.test(context.target)) throw new Error("Python project context target must be a .py or .pyi path"); + if (!isPythonProjectContextTarget(context.target)) { + throw new Error("Python project context target must be Python source or config"); + } validateNonEmptyString(context.repositoryRoot, "Python project context repositoryRoot"); validatePythonProjectRoot(context.projectRoot, "Python project context projectRoot"); validatePythonProjectRoot(context.projectBoundary, "Python project context projectBoundary"); validatePythonProjectRoots(context.sourceRoots, "Python project context sourceRoots"); - if (!context.layout || typeof context.layout !== "object") throw new Error("Python project context layout is required"); - validateExactObjectKeys(context.layout, ["kinds", "paths"], "Python project context layout"); - validateExactEnumArray(context.layout.kinds, pythonProjectLayoutKinds, "Python project context layout kinds", false); - validatePythonProjectRoots(context.layout.paths, "Python project context layout paths"); - if (!Array.isArray(context.evidence)) throw new Error("Python project context evidence must be an array"); - for (const entry of context.evidence) { +} + +function validatePythonProjectContextLayout(layout: PythonProjectContext["layout"]): void { + if (!layout || typeof layout !== "object") throw new Error("Python project context layout is required"); + validateExactObjectKeys(layout, ["kinds", "paths"], "Python project context layout"); + validateExactEnumArray(layout.kinds, pythonProjectLayoutKinds, "Python project context layout kinds", false); + validatePythonProjectRoots(layout.paths, "Python project context layout paths"); +} + +function validatePythonProjectContextEvidence(evidence: PythonProjectContext["evidence"]): void { + if (!Array.isArray(evidence)) throw new Error("Python project context evidence must be an array"); + for (const entry of evidence) { validateExactObjectKeys(entry, ["path", "role"], "Python project context evidence"); validateRepoRelativePath(entry.path); if (!includesString(["boundary", "config", "lock", "requirements", "build", "layout"] as const, entry.role)) { throw new Error(`Unknown Python project evidence role: ${String(entry.role)}`); } } - validatePythonProjectTarget(context.targetRuntime); - if (!Array.isArray(context.managers)) throw new Error("Python project context managers must be an array"); - for (const manager of context.managers) { +} + +function validatePythonProjectManagerEvidenceEntries(managers: PythonProjectContext["managers"]): void { + if (!Array.isArray(managers)) throw new Error("Python project context managers must be an array"); + for (const manager of managers) { validateExactObjectKeys(manager, ["kind", "configFiles", "lockFiles"], "Python project manager evidence"); if (!includesString(pythonProjectManagerKinds, manager.kind)) { throw new Error(`Unknown Python project manager kind: ${String(manager.kind)}`); @@ -6663,17 +6809,21 @@ export function validatePythonProjectContext(context: PythonProjectContext): Pyt validateRepoPathArray(manager.configFiles, "Python project manager configFiles"); validateRepoPathArray(manager.lockFiles, "Python project manager lockFiles"); } - if (context.buildSystem !== undefined) { - validateExactObjectKeys(context.buildSystem, ["configFile", "backend", "requires"], "Python project buildSystem"); - validateRepoRelativePath(context.buildSystem.configFile); - if (context.buildSystem.backend !== undefined) { - validateNonEmptyString(context.buildSystem.backend, "Python project buildSystem backend"); - } - validateStringArray(context.buildSystem.requires, "Python project buildSystem requires", { allowEmpty: true }); +} + +function validatePythonProjectBuildSystem(buildSystem: PythonProjectContext["buildSystem"]): void { + if (buildSystem === undefined) return; + validateExactObjectKeys(buildSystem, ["configFile", "backend", "requires"], "Python project buildSystem"); + validateRepoRelativePath(buildSystem.configFile); + if (buildSystem.backend !== undefined) { + validateNonEmptyString(buildSystem.backend, "Python project buildSystem backend"); } - if (context.interpreter !== undefined) validatePythonInterpreterProvenance(context.interpreter); - if (!Array.isArray(context.tools)) throw new Error("Python project context tools must be an array"); - for (const tool of context.tools) { + validateStringArray(buildSystem.requires, "Python project buildSystem requires", { allowEmpty: true }); +} + +function validatePythonProjectToolEntries(tools: PythonProjectContext["tools"]): void { + if (!Array.isArray(tools)) throw new Error("Python project context tools must be an array"); + for (const tool of tools) { validateExactObjectKeys( tool, ["tool", "available", "executable", "argv", "cwd", "source", "version", "configFile"], @@ -6686,13 +6836,17 @@ export function validatePythonProjectContext(context: PythonProjectContext): Pyt throw new Error(`Available Python project tool ${tool.tool} must include version provenance`); } } - validateSha256Identity(context.projectKey, "Python project context projectKey"); - validateSha256Identity(context.contextFingerprint, "Python project context contextFingerprint"); - if (!includesString(pythonProjectContextOutcomes, context.outcome)) { - throw new Error(`Unknown Python project context outcome: ${String(context.outcome)}`); +} + +function validatePythonProjectContextOutcome( + outcome: PythonProjectContext["outcome"], + reasons: PythonProjectContext["reasons"] +): void { + if (!includesString(pythonProjectContextOutcomes, outcome)) { + throw new Error(`Unknown Python project context outcome: ${String(outcome)}`); } - if (!Array.isArray(context.reasons)) throw new Error("Python project context reasons must be an array"); - for (const reason of context.reasons) { + if (!Array.isArray(reasons)) throw new Error("Python project context reasons must be an array"); + for (const reason of reasons) { validateExactObjectKeys(reason, ["code", "message", "path", "tool"], "Python project context reason"); if (!includesString(pythonProjectContextReasonCodes, reason.code)) { throw new Error(`Unknown Python project context reason: ${String(reason.code)}`); @@ -6701,13 +6855,31 @@ export function validatePythonProjectContext(context: PythonProjectContext): Pyt if (reason.path !== undefined) validateRepoRelativePath(reason.path); if (reason.tool !== undefined) validateNonEmptyString(reason.tool, "Python project context reason tool"); } - if (context.outcome === "resolved" && context.reasons.length > 0) { - throw new Error("Resolved Python project context must not include reasons"); - } - if (context.outcome !== "resolved" && context.reasons.length === 0) { - throw new Error("Non-resolved Python project context must include reasons"); - } - return context; + if (outcome === "resolved" && reasons.length > 0) throw new Error("Resolved Python project context must not include reasons"); + if (outcome !== "resolved" && reasons.length === 0) throw new Error("Non-resolved Python project context must include reasons"); +} + +function isPythonProjectContextTarget(path: string): boolean { + if (/\.pyi?$/u.test(path)) return true; + const name = path.slice(path.lastIndexOf("/") + 1); + return [ + "pyproject.toml", + "Pipfile", + "Pipfile.lock", + "poetry.lock", + "pdm.lock", + "uv.lock", + "setup.cfg", + "setup.py", + "pytest.ini", + "tox.ini", + "pyrightconfig.json", + "ruff.toml", + ".ruff.toml", + "mypy.ini", + ".mypy.ini", + "conftest.py" + ].includes(name) || /^requirements.*\.txt$/u.test(name); } export function validatePythonProjectContexts(contexts: readonly PythonProjectContext[]): readonly PythonProjectContext[] { @@ -6721,6 +6893,72 @@ export function validatePythonProjectContexts(contexts: readonly PythonProjectCo return contexts; } +function validatePythonCapabilityCounts(counts: PythonCapabilityCounts): void { + if (!counts || typeof counts !== "object") throw new Error("Python capability counts are required"); + validateExactObjectKeys(counts, [ + "candidateCount", + "collectedCount", + "executedCount", + "passedCount", + "failedCount", + "skippedCount", + "xfailedCount", + "xpassedCount", + "errorCount" + ], "Python capability counts"); + for (const key of Object.keys(counts) as (keyof PythonCapabilityCounts)[]) { + validateNonNegativeInteger(counts[key], `Python capability counts ${key}`); + } +} + +function validatePythonCapabilityCleanupEvidence(cleanup: PythonCapabilityCleanupEvidence): void { + if (!cleanup || typeof cleanup !== "object") throw new Error("Python capability cleanup evidence is required"); + validateExactObjectKeys(cleanup, ["attempted", "ok", "failureMessage"], "Python capability cleanup evidence"); + if (typeof cleanup.attempted !== "boolean") throw new Error("Python capability cleanup attempted must be boolean"); + if (typeof cleanup.ok !== "boolean") throw new Error("Python capability cleanup ok must be boolean"); + if (cleanup.failureMessage !== undefined) { + validateNonEmptyString(cleanup.failureMessage, "Python capability cleanup failureMessage"); + } +} + +function validatePythonCapabilityInvocation(invocation: PythonCapabilityInvocation, label: string): void { + if (!invocation || typeof invocation !== "object") throw new Error(`${label} is required`); + validateExactObjectKeys(invocation, [ + "stage", + "command", + "argsDigest", + "argCount", + "selectionMode", + "selectionDigest", + "durationMs", + "termination", + "exitCode", + "signal", + "outputBytes", + "stdoutDigest", + "stderrDigest" + ], label); + if (!includesString(["collection", "execution"] as const, invocation.stage)) { + throw new Error(`${label} stage must be collection or execution`); + } + validateNonEmptyString(invocation.command, `${label} command`); + validateSha256Identity(invocation.argsDigest, `${label} argsDigest`); + validateNonNegativeInteger(invocation.argCount, `${label} argCount`); + if (!includesString(pythonPytestSelectionModes, invocation.selectionMode)) { + throw new Error(`Unknown ${label} selectionMode: ${String(invocation.selectionMode)}`); + } + if (invocation.selectionDigest !== undefined) validateSha256Identity(invocation.selectionDigest, `${label} selectionDigest`); + validateNonNegativeInteger(invocation.durationMs, `${label} durationMs`); + if (!includesString(pythonCapabilityProcessTerminations, invocation.termination)) { + throw new Error(`Unknown ${label} termination: ${String(invocation.termination)}`); + } + if (invocation.exitCode !== undefined) validateNonNegativeInteger(invocation.exitCode, `${label} exitCode`); + if (invocation.signal !== undefined) validateNonEmptyString(invocation.signal, `${label} signal`); + validateNonNegativeInteger(invocation.outputBytes, `${label} outputBytes`); + if (invocation.stdoutDigest !== undefined) validateSha256Identity(invocation.stdoutDigest, `${label} stdoutDigest`); + if (invocation.stderrDigest !== undefined) validateSha256Identity(invocation.stderrDigest, `${label} stderrDigest`); +} + function validatePythonProjectTarget(target: PythonProjectTarget): void { if (!target || typeof target !== "object") throw new Error("Python project targetRuntime is required"); validateExactObjectKeys( diff --git a/packages/fixtures/src/index.ts b/packages/fixtures/src/index.ts index eba1d69..f4f83cf 100644 --- a/packages/fixtures/src/index.ts +++ b/packages/fixtures/src/index.ts @@ -609,7 +609,8 @@ export const conformanceFixtureMetadata = [ "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ], degradedTools: ["mypy", "pyright", "ruff", "pytest"] } diff --git a/packages/fixtures/validation-python/README.md b/packages/fixtures/validation-python/README.md index 8c4f036..4824faa 100644 --- a/packages/fixtures/validation-python/README.md +++ b/packages/fixtures/validation-python/README.md @@ -6,3 +6,4 @@ Synthetic fixtures for Python validation conformance: - `failing`: syntax, source-hygiene, and import-graph diagnostics. - `degraded-tools`: type-check degradation when optional Python type tools are absent. - `compiler-truth`: valid multiline/stub grammar plus compiler-only invalid control-flow and future-import fixtures. Files use a `.fixture` suffix so repository validation does not mistake intentionally synthetic inputs for product source; tests remove the suffix before validation. +- `pytest-authority`: opt-in real-tool pytest authority fixtures. Includes pass/fail, collection-fail, timeout, manifest-fallback, and workspace-isolation cases. All synthetic source, test, and packaging files use a `.fixture` suffix so repository validation and provenance checks keep the repository clean room; the authority proof script materializes canonical filenames before execution. diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/pyproject.toml.fixture new file mode 100644 index 0000000..35c01ab --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-collection-fail" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/conftest.py.fixture new file mode 100644 index 0000000..5801bea --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/conftest.py.fixture @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/test_app.py.fixture new file mode 100644 index 0000000..837d47e --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/collection-fail/tests/test_app.py.fixture @@ -0,0 +1,6 @@ +from src.app import answer +from missing_fixture_dependency import NEVER_DEFINED + + +def test_answer() -> None: + assert answer() == NEVER_DEFINED diff --git a/packages/fixtures/validation-python/pytest-authority/fail/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/fail/pyproject.toml.fixture new file mode 100644 index 0000000..ff678d3 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-fail" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/fail/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/fail/src/app.py.fixture new file mode 100644 index 0000000..ffa919e --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/src/app.py.fixture @@ -0,0 +1,3 @@ +def answer() -> int: + return 42 + diff --git a/packages/fixtures/validation-python/pytest-authority/fail/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/fail/tests/conftest.py.fixture new file mode 100644 index 0000000..6ee195b --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/tests/conftest.py.fixture @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + diff --git a/packages/fixtures/validation-python/pytest-authority/fail/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/fail/tests/test_app.py.fixture new file mode 100644 index 0000000..7291019 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/fail/tests/test_app.py.fixture @@ -0,0 +1,5 @@ +from src.app import answer + + +def test_answer() -> None: + assert answer() == 0 diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/pyproject.toml.fixture new file mode 100644 index 0000000..e9ac466 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-manifest" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/tests/conftest.py.fixture new file mode 100644 index 0000000..5801bea --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/tests/conftest.py.fixture @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/packages/fixtures/validation-python/pytest-authority/manifest/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/manifest/tests/test_app.py.fixture new file mode 100644 index 0000000..455177c --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/manifest/tests/test_app.py.fixture @@ -0,0 +1,11 @@ +import pytest + +from src.app import answer + +VALUES = [f"case-{index:03d}-{'x' * 120}" for index in range(180)] + + +@pytest.mark.parametrize("label", VALUES, ids=VALUES) +def test_answer(label: str) -> None: + assert label.startswith("case-") + assert answer() == 42 diff --git a/packages/fixtures/validation-python/pytest-authority/pass/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/pass/pyproject.toml.fixture new file mode 100644 index 0000000..5bc5e17 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-pass" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/pass/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/pass/src/app.py.fixture new file mode 100644 index 0000000..ffa919e --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/src/app.py.fixture @@ -0,0 +1,3 @@ +def answer() -> int: + return 42 + diff --git a/packages/fixtures/validation-python/pytest-authority/pass/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/pass/tests/conftest.py.fixture new file mode 100644 index 0000000..6ee195b --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/tests/conftest.py.fixture @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + diff --git a/packages/fixtures/validation-python/pytest-authority/pass/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/pass/tests/test_app.py.fixture new file mode 100644 index 0000000..ca059ed --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/pass/tests/test_app.py.fixture @@ -0,0 +1,6 @@ +from src.app import answer + + +def test_answer() -> None: + assert answer() == 42 + diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/pyproject.toml.fixture new file mode 100644 index 0000000..a0a2e34 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-timeout" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/tests/conftest.py.fixture new file mode 100644 index 0000000..5801bea --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/tests/conftest.py.fixture @@ -0,0 +1,4 @@ +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) diff --git a/packages/fixtures/validation-python/pytest-authority/timeout/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/timeout/tests/test_app.py.fixture new file mode 100644 index 0000000..3fa8fcc --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/timeout/tests/test_app.py.fixture @@ -0,0 +1,8 @@ +import time + +from src.app import answer + + +def test_answer() -> None: + time.sleep(31) + assert answer() == 42 diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/pyproject.toml.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/pyproject.toml.fixture new file mode 100644 index 0000000..bd362b6 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/pyproject.toml.fixture @@ -0,0 +1,3 @@ +[project] +name = "pytest-authority-workspace-isolation" +version = "0.0.0" diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/src/app.py.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/src/app.py.fixture new file mode 100644 index 0000000..4860f40 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/src/app.py.fixture @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/conftest.py.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/conftest.py.fixture new file mode 100644 index 0000000..b3a5741 --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/conftest.py.fixture @@ -0,0 +1,12 @@ +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + + +def pytest_collection_modifyitems(session, config, items): + (Path(config.rootpath) / "src" / "app.py").write_text( + "def answer() -> int:\n return 0\n", + encoding="utf-8" + ) diff --git a/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/test_app.py.fixture b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/test_app.py.fixture new file mode 100644 index 0000000..913d39a --- /dev/null +++ b/packages/fixtures/validation-python/pytest-authority/workspace-isolation/tests/test_app.py.fixture @@ -0,0 +1,5 @@ +from src.app import answer + + +def test_answer() -> None: + assert answer() == 42 diff --git a/packages/opcore-graph-core-darwin-arm64/metadata.json b/packages/opcore-graph-core-darwin-arm64/metadata.json index 65279a0..1cbe56a 100644 --- a/packages/opcore-graph-core-darwin-arm64/metadata.json +++ b/packages/opcore-graph-core-darwin-arm64/metadata.json @@ -4,6 +4,6 @@ "targetPlatform": "darwin-arm64", "binaryPath": "opcore-graph-core", "checksumPath": "opcore-graph-core.sha256", - "checksumSha256": "3fb851c8e057f085c0dd1fb15a7e3a9796a6176974a8fb6e1f34a5222b1271ae", + "checksumSha256": "82178ce4cae6892dff3263250bcf0885e1344ac80310184e4c330e7238a3f249", "buildProfile": "release" } diff --git a/packages/opcore-graph-core-darwin-arm64/opcore-graph-core b/packages/opcore-graph-core-darwin-arm64/opcore-graph-core index d94c85e..684e610 100755 Binary files a/packages/opcore-graph-core-darwin-arm64/opcore-graph-core and b/packages/opcore-graph-core-darwin-arm64/opcore-graph-core differ diff --git a/packages/opcore-graph-core-darwin-arm64/opcore-graph-core.sha256 b/packages/opcore-graph-core-darwin-arm64/opcore-graph-core.sha256 index c00a462..17c0122 100644 --- a/packages/opcore-graph-core-darwin-arm64/opcore-graph-core.sha256 +++ b/packages/opcore-graph-core-darwin-arm64/opcore-graph-core.sha256 @@ -1 +1 @@ -3fb851c8e057f085c0dd1fb15a7e3a9796a6176974a8fb6e1f34a5222b1271ae opcore-graph-core +82178ce4cae6892dff3263250bcf0885e1344ac80310184e4c330e7238a3f249 opcore-graph-core diff --git a/packages/opcore/src/init.ts b/packages/opcore/src/init.ts index fc3d821..8e82c2e 100644 --- a/packages/opcore/src/init.ts +++ b/packages/opcore/src/init.ts @@ -755,7 +755,8 @@ function checksForLanguage(language: string, validation: OpcoreInitLanguageSetti "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]; } return []; diff --git a/packages/opcore/src/reporting.ts b/packages/opcore/src/reporting.ts index b278141..9fa688a 100644 --- a/packages/opcore/src/reporting.ts +++ b/packages/opcore/src/reporting.ts @@ -228,11 +228,11 @@ export function createOpcoreMetricReport(input: CreateOpcoreMetricReportInput): }); addDiagnosticSignal(signals, { id: "python.untested_modules", - title: "Python modules without TESTED_BY graph evidence", + title: "Python modules without TESTED_BY graph candidate evidence", category: "python", severity: "warning", checkId: pythonRelevantTestsCheckId, - diagnostics: diagnostics.filter((diagnostic) => diagnostic.code === "PY_RELEVANT_TESTS_ABSENT" && hasPath(diagnostic)) + diagnostics: diagnostics.filter((diagnostic) => diagnostic.code === "PY_RELEVANT_TEST_CANDIDATES_ABSENT" && hasPath(diagnostic)) }); addDiagnosticSignal(signals, { id: "python.dead_exports", diff --git a/packages/opcore/src/scan-validation-preview.ts b/packages/opcore/src/scan-validation-preview.ts index 9d0a6d6..3f3c3e8 100644 --- a/packages/opcore/src/scan-validation-preview.ts +++ b/packages/opcore/src/scan-validation-preview.ts @@ -1,4 +1,7 @@ -import type { ValidationResult } from "@the-open-engine/opcore-contracts"; +import { + validatePythonValidationCapabilityRun, + type ValidationResult +} from "@the-open-engine/opcore-contracts"; export const scanDiagnosticPreviewLimit = 50; @@ -19,6 +22,11 @@ export function compactScanValidationResult( if (result.pythonProjectContexts !== undefined) { compact.pythonProjectContexts = [...result.pythonProjectContexts]; } + if (result.pythonCapabilityRuns !== undefined) { + compact.pythonCapabilityRuns = result.pythonCapabilityRuns.map((run) => + validatePythonValidationCapabilityRun(run) + ); + } if (result.manifest !== undefined) { compact.manifest = { schemaVersion: result.manifest.schemaVersion, diff --git a/packages/validation-policy/src/factory.ts b/packages/validation-policy/src/factory.ts index ac1ae36..b643195 100644 --- a/packages/validation-policy/src/factory.ts +++ b/packages/validation-policy/src/factory.ts @@ -29,6 +29,13 @@ export function validationChecksForRepoPolicyAndCoverage( return validationChecksForRepoPolicy(repoRoot, options).filter((check) => adapters.has(check.adapter)); } +export function validationChecksForConfigPolicy( + config: OpcoreRepoConfig, + options: OpcoreRepoValidationPolicyOptions = {} +): readonly ValidationCheckDefinition[] { + return applyValidationPolicy(createBuiltInValidationChecks(config, options), config); +} + export function createBuiltInValidationChecks( config?: OpcoreRepoConfig, options: OpcoreRepoValidationPolicyOptions = {}, @@ -69,6 +76,13 @@ function validationChecksForRepoConfig( const builtIns = createBuiltInValidationChecks(config, options, repoRoot); const packChecks = loadRepoCheckPacks(repoRoot, config).flatMap((pack) => pack.checks); const available = [...builtIns, ...packChecks]; + return applyValidationPolicy(available, config); +} + +function applyValidationPolicy( + available: readonly ValidationCheckDefinition[], + config: OpcoreRepoConfig +): readonly ValidationCheckDefinition[] { createValidationCheckRegistry(available); const knownCheckIds = new Set(available.map((check) => check.id)); @@ -84,8 +98,9 @@ function validationChecksForRepoConfig( const filteredContexts = new WeakMap[0]>(); const checks = available .filter((check) => adapters === undefined || adapters.has(check.adapter)) - .filter((check) => !disabled.has(check.id)) .map((check) => applyDefaultScopePolicy(check, defaults)) + .map((check) => applyDisabledPolicy(check, disabled)) + .filter((check): check is ValidationCheckDefinition => check !== undefined) .map((check) => applyPathPolicy(check, config.validation.pathPolicy, filteredContexts)); createValidationCheckRegistry(checks); return checks; @@ -151,6 +166,23 @@ function applyDefaultScopePolicy(check: ValidationCheckDefinition, defaults: Rea return { ...check, defaultScopes: check.supportedScopes }; } +function applyDisabledPolicy( + check: ValidationCheckDefinition, + disabled: ReadonlySet +): ValidationCheckDefinition | undefined { + if (!disabled.has(check.id)) return check; + if (check.inactiveResult === undefined) return undefined; + return { + ...check, + requiresGraph: false, + graphUsage: "none", + graphRequirements: undefined, + defaultScopes: [], + inactiveStateWhenUnselected: "disabled", + run: (context) => check.inactiveResult?.(context, "disabled") + }; +} + function applyPathPolicy( check: ValidationCheckDefinition, pathPolicy: OpcorePathPolicy | undefined, diff --git a/packages/validation-python/src/check-ids.ts b/packages/validation-python/src/check-ids.ts index 49d4060..42f6819 100644 --- a/packages/validation-python/src/check-ids.ts +++ b/packages/validation-python/src/check-ids.ts @@ -4,6 +4,7 @@ export const PYTHON_TYPES_CHECK_ID = "python.types"; export const PYTHON_IMPORT_GRAPH_CHECK_ID = "python.import-graph"; export const PYTHON_DEAD_CODE_CHECK_ID = "python.dead-code"; export const PYTHON_RELEVANT_TESTS_CHECK_ID = "python.relevant-tests"; +export const PYTHON_PYTEST_CHECK_ID = "python.pytest"; export const pythonValidationCheckIds = [ PYTHON_SYNTAX_CHECK_ID, @@ -11,7 +12,8 @@ export const pythonValidationCheckIds = [ PYTHON_TYPES_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, PYTHON_DEAD_CODE_CHECK_ID, - PYTHON_RELEVANT_TESTS_CHECK_ID + PYTHON_RELEVANT_TESTS_CHECK_ID, + PYTHON_PYTEST_CHECK_ID ] as const; export type PythonValidationCheckId = (typeof pythonValidationCheckIds)[number]; diff --git a/packages/validation-python/src/environment-resolution.ts b/packages/validation-python/src/environment-resolution.ts index 8b7badd..5f92757 100644 --- a/packages/validation-python/src/environment-resolution.ts +++ b/packages/validation-python/src/environment-resolution.ts @@ -43,6 +43,12 @@ export interface PythonProjectEnvironmentResolution { fingerprintInput: unknown; } +interface ResolvedPythonTool { + tool?: PythonProjectToolProvenance; + failure?: PythonProjectContextReason; + terminal?: boolean; +} + interface ExecutableCandidate { argv: readonly string[]; source: PythonProjectExecutableSource; @@ -205,81 +211,22 @@ async function resolveTools( managerCandidates: ManagerExecutableCandidates, reasons: PythonProjectContextReason[] ): Promise { - const toolKinds: PythonProjectToolKind[] = ["mypy", "pyright", "ruff", "pytest"]; + const toolKinds: readonly Exclude[] = ["mypy", "pyright", "ruff", "pytest"]; const tools: PythonProjectToolProvenance[] = []; for (const tool of toolKinds) { - const configFile = tool === "mypy" || tool === "pyright" || tool === "ruff" || tool === "pytest" - ? options.toolConfigs[tool] - : undefined; const candidates = toolCandidates(tool, options, managerCandidates, reasons); - if (options.toolArgv?.[tool] !== undefined && candidates.length === 0) { - const override = options.toolArgv[tool]; - if (override === undefined) throw new Error(`Python ${tool} argv override is required`); - tools.push({ - tool, - available: false, - executable: override[0], - argv: [...override], - cwd: projectCwd(options), - source: "explicit_override", - ...(configFile === undefined ? {} : { configFile }) - }); + const overrideTool = explicitOverrideUnavailableTool(tool, options, candidates); + if (overrideTool !== undefined) { + tools.push(overrideTool); continue; } - let resolvedTool: PythonProjectToolProvenance | undefined; - let failure: PythonProjectContextReason | undefined; - for (const candidate of candidates) { - if (!(await candidateAvailable(options.workspace, candidate.argv[0]))) continue; - const command = candidate.argv[0]; - const prefix = candidate.argv.slice(1); - const result = processProbe.run(command, [...prefix, "--version"], { - cwd: projectCwd(options), env, timeoutMs: options.timeoutMs ?? 10000 - }); - if (!result.ok) { - failure = probeFailureReason(result, tool); - if (candidate.source !== "path") break; - continue; - } - const version = firstVersion(result.stdout, result.stderr); - if (version === undefined) { - failure = { - code: "malformed_probe_output", - tool, - message: `${tool} version probe returned malformed metadata` - }; - if (candidate.source !== "path") break; - continue; - } - const executable = candidate.source === "path" - ? processProbe.resolveExecutable?.(command, { env, platform: options.platform ?? process.platform }) ?? command - : command; - resolvedTool = { - tool, - available: true, - executable, - argv: [executable, ...prefix], - cwd: projectCwd(options), - source: candidate.source, - version, - ...(configFile === undefined ? {} : { configFile }) - }; - break; - } - if (resolvedTool !== undefined) { - tools.push(resolvedTool); + const resolved = await resolveAvailableTool(tool, options, processProbe, env, candidates); + if (resolved.tool !== undefined) { + tools.push(resolved.tool); continue; } - const fallback = candidates[0] ?? { argv: [tool], source: "path" as const }; - tools.push({ - tool, - available: false, - executable: fallback.argv[0], - argv: fallback.argv, - cwd: projectCwd(options), - source: fallback.source, - ...(configFile === undefined ? {} : { configFile }) - }); - reasons.push(failure ?? { code: "tool_unavailable", tool, message: `${tool} is unavailable for ${options.projectRoot}` }); + reasons.push(resolved.failure ?? { code: "tool_unavailable", tool, message: `${tool} is unavailable for ${options.projectRoot}` }); + tools.push(unavailableTool(tool, options, candidates)); } if (options.buildSystem !== undefined) { tools.push(resolveBuildTool(options, processProbe, env, interpreter, reasons)); @@ -287,6 +234,127 @@ async function resolveTools( return tools.sort((left, right) => left.tool.localeCompare(right.tool)); } +function explicitOverrideUnavailableTool( + tool: Exclude, + options: PythonProjectEnvironmentOptions, + candidates: readonly ExecutableCandidate[] +): PythonProjectToolProvenance | undefined { + if (options.toolArgv?.[tool] === undefined || candidates.length > 0) return undefined; + const override = options.toolArgv[tool]; + if (override === undefined) throw new Error(`Python ${tool} argv override is required`); + return { + tool, + available: false, + executable: override[0], + argv: [...override], + cwd: projectCwd(options), + source: "explicit_override", + ...(toolConfigFile(tool, options) === undefined ? {} : { configFile: toolConfigFile(tool, options) }) + }; +} + +async function resolveAvailableTool( + tool: Exclude, + options: PythonProjectEnvironmentOptions, + processProbe: PythonProjectProcessProbe, + env: Record, + candidates: readonly ExecutableCandidate[] +): Promise { + let failure: PythonProjectContextReason | undefined; + for (const candidate of candidates) { + const resolved = await resolveAvailableToolCandidate(tool, options, processProbe, env, candidate); + if (resolved.failure !== undefined) failure = resolved.failure; + if (resolved.tool !== undefined || resolved.terminal) return resolved; + } + return failure === undefined ? {} : { failure }; +} + +async function resolveAvailableToolCandidate( + tool: Exclude, + options: PythonProjectEnvironmentOptions, + processProbe: PythonProjectProcessProbe, + env: Record, + candidate: ExecutableCandidate +): Promise { + if (!(await candidateAvailable(options.workspace, candidate.argv[0]))) return {}; + const command = candidate.argv[0]; + const prefix = candidate.argv.slice(1); + const result = await processProbe.run(command, [...toolVersionProbePrefix(tool, prefix), "--version"], { + cwd: projectCwd(options), + env: tool === "pytest" ? { ...env, PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1" } : env, + timeoutMs: options.timeoutMs ?? 10000 + }); + if (!result.ok) return { failure: probeFailureReason(result, tool), terminal: candidate.source !== "path" }; + const version = firstVersion(result.stdout, result.stderr); + if (version === undefined) { + return { + failure: { + code: "malformed_probe_output", + tool, + message: `${tool} version probe returned malformed metadata` + }, + terminal: candidate.source !== "path" + }; + } + const executable = resolvedToolExecutable(command, candidate.source, processProbe, env, options.platform ?? process.platform); + return { + tool: { + tool, + available: true, + executable, + argv: [executable, ...prefix], + cwd: projectCwd(options), + source: candidate.source, + version, + ...(toolConfigFile(tool, options) === undefined ? {} : { configFile: toolConfigFile(tool, options) }) + }, + terminal: true + }; +} + +function unavailableTool( + tool: Exclude, + options: PythonProjectEnvironmentOptions, + candidates: readonly ExecutableCandidate[] +): PythonProjectToolProvenance { + const fallback = candidates[0] ?? { argv: [tool], source: "path" as const }; + return { + tool, + available: false, + executable: fallback.argv[0], + argv: fallback.argv, + cwd: projectCwd(options), + source: fallback.source, + ...(toolConfigFile(tool, options) === undefined ? {} : { configFile: toolConfigFile(tool, options) }) + }; +} + +function toolConfigFile( + tool: Exclude, + options: PythonProjectEnvironmentOptions +): string | undefined { + return options.toolConfigs[tool]; +} + +function toolVersionProbePrefix( + tool: Exclude, + prefix: readonly string[] +): readonly string[] { + return tool === "ruff" ? withoutToolConfigOptions(prefix, tool) : prefix; +} + +function resolvedToolExecutable( + command: string, + source: PythonProjectExecutableSource, + processProbe: PythonProjectProcessProbe, + env: Record, + platform: string +): string { + return source === "path" + ? processProbe.resolveExecutable?.(command, { env, platform }) ?? command + : command; +} + function runnableInterpreterArgv( candidate: ExecutableCandidate, probedExecutable: string, @@ -698,6 +766,24 @@ function safeToolOptionPrefix(tool: Exclude, arg return true; } +function withoutToolConfigOptions( + args: readonly string[], + tool: Exclude +): readonly string[] { + const options = toolConfigOptions[tool]; + const result: string[] = []; + for (let index = 0; index < args.length; index += 1) { + const argument = args[index]; + if (options.includes(argument)) { + index += 1; + continue; + } + if (options.some((option) => argument.startsWith(`${option}=`))) continue; + result.push(argument); + } + return result; +} + function safeToolConfigPaths( tool: Exclude, args: readonly string[], diff --git a/packages/validation-python/src/index.ts b/packages/validation-python/src/index.ts index 1ff1cbc..7180484 100644 --- a/packages/validation-python/src/index.ts +++ b/packages/validation-python/src/index.ts @@ -2,6 +2,7 @@ import type { ValidationCheckDefinition, ValidationCheckResult } from "@the-open import type { ValidationDiagnostic } from "@the-open-engine/opcore-contracts"; import { createDeadCodeCheck } from "./dead-code-check.js"; import { createImportGraphCheck } from "./import-graph-check.js"; +import { createPytestCheck } from "./pytest-check.js"; import { createRelevantTestsCheck } from "./relevant-tests-check.js"; import { createSourceHygieneCheck } from "./source-hygiene-check.js"; import { createSyntaxCheck, type PythonSyntaxCheckOptions } from "./syntax-check.js"; @@ -12,6 +13,7 @@ import type { PythonImportAnalyzer } from "./import-analysis.js"; export { PYTHON_DEAD_CODE_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, + PYTHON_PYTEST_CHECK_ID, PYTHON_RELEVANT_TESTS_CHECK_ID, PYTHON_SOURCE_HYGIENE_CHECK_ID, PYTHON_SYNTAX_CHECK_ID, @@ -37,6 +39,7 @@ export type { PythonProjectProcessProbe } from "./environment-resolution.js"; export { createPythonValidationAdapterStatus, type PythonValidationToolchainOptions } from "./toolchain.js"; export { createSyntaxCheck, type PythonSyntaxCheckOptions } from "./syntax-check.js"; export { createTypeCheck, type PythonTypeCheckOptions } from "./type-check.js"; +export { disabledPytestRun } from "./pytest-check.js"; export interface CreatePythonValidationChecksOptions extends PythonTypeCheckOptions, PythonSyntaxCheckOptions { importAnalyzer?: PythonImportAnalyzer; @@ -63,7 +66,8 @@ export function createPythonValidationChecks( createTypeCheck(options, resolveContexts, resolveSources), createImportGraphCheck(resolveSources), createDeadCodeCheck(resolveRoots), - createRelevantTestsCheck(resolveRoots) + createRelevantTestsCheck(resolveRoots), + createPytestCheck(options, resolveContexts) ].map((check) => withPythonProjectContexts(check, resolveContexts)); } diff --git a/packages/validation-python/src/materialized-workspace.ts b/packages/validation-python/src/materialized-workspace.ts new file mode 100644 index 0000000..6480cb1 --- /dev/null +++ b/packages/validation-python/src/materialized-workspace.ts @@ -0,0 +1,56 @@ +import { chmodSync, existsSync, rmSync, writeFileSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { dirname, relative, resolve, sep } from "node:path"; +import { validateRepoRelativePath } from "@the-open-engine/opcore-contracts"; + +export function resolveMaterializedWorkspacePath(root: string, path: string, label: string): string { + const normalized = validateRepoRelativePath(path); + const absolutePath = resolve(root, normalized); + const relativePath = relative(root, absolutePath); + if (relativePath === "" || relativePath.startsWith("..") || relativePath.split(sep).includes("..")) { + throw new Error(`Repo-relative path escapes ${label}: ${path}`); + } + return absolutePath; +} + +export async function writeMaterializedWorkspaceFile( + root: string, + path: string, + content: string, + label: string, + mode?: number +): Promise { + const absolutePath = resolveMaterializedWorkspacePath(root, path, label); + await mkdir(dirname(absolutePath), { recursive: true }); + writeFileSync(absolutePath, content, "utf8"); + if (mode !== undefined) chmodSync(absolutePath, mode & 0o7777); +} + +export function removeMaterializedWorkspace(root: string, label: string): void { + let lastError: unknown; + for (let attempt = 0; attempt < 6; attempt += 1) { + try { + rmSync(root, { recursive: true, force: true }); + if (!existsSync(root)) return; + lastError = new Error(`${label} cleanup left residual files.`); + } catch (error) { + lastError = error; + } + sleep(25); + } + throw sanitizeWorkspaceCleanupError(lastError, label); +} + +function sanitizeWorkspaceCleanupError(error: unknown, label: string): Error { + const code = typeof error === "object" && error !== null && "code" in error && typeof error.code === "string" + ? error.code + : undefined; + const suffix = error instanceof Error && error.message.includes("residual files") + ? "Residual files remained after cleanup retries." + : (code === undefined ? "Cleanup failed." : `Cleanup failed (${code}).`); + return new Error(`${label} ${suffix}`); +} + +function sleep(milliseconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} diff --git a/packages/validation-python/src/node-shims.d.ts b/packages/validation-python/src/node-shims.d.ts index 9180682..dac787e 100644 --- a/packages/validation-python/src/node-shims.d.ts +++ b/packages/validation-python/src/node-shims.d.ts @@ -30,18 +30,20 @@ declare module "node:crypto" { } declare module "node:fs" { + export function chmodSync(path: string, mode: number): void; export function existsSync(path: string): boolean; + export function readFileSync(path: string, encoding: "utf8"): string; export function realpathSync(path: string): string; export function mkdirSync(path: string, options?: { recursive?: boolean }): void; export function mkdtempSync(prefix: string): string; export function rmSync(path: string, options?: { recursive?: boolean; force?: boolean }): void; - export function writeFileSync(path: string, data: string): void; + export function writeFileSync(path: string, data: string, encoding?: "utf8"): void; } declare module "node:fs/promises" { export function access(path: string): Promise; export function copyFile(source: string, destination: string): Promise; - export function lstat(path: string): Promise<{ isFile(): boolean; isSymbolicLink(): boolean }>; + export function lstat(path: string): Promise<{ mode: number; isFile(): boolean; isSymbolicLink(): boolean }>; export function mkdir(path: string, options?: { recursive?: boolean }): Promise; export function readFile(path: string, encoding: "utf8"): Promise; export function readdir(path: string, options: { recursive: true }): Promise; @@ -68,8 +70,23 @@ interface Buffer { toString(encoding?: BufferEncoding): string; } +declare namespace NodeJS { + type Signals = string; + + interface ErrnoException extends Error { + code?: string; + } +} + +declare const Buffer: { + byteLength(input: string, encoding?: BufferEncoding): number; +}; + declare const process: { env: Record; + execPath: string; + pid: number; cwd(): string; + kill(pid: number, signal?: string): boolean; platform: string; }; diff --git a/packages/validation-python/src/project-config-files.ts b/packages/validation-python/src/project-config-files.ts index ed16565..1201a5c 100644 --- a/packages/validation-python/src/project-config-files.ts +++ b/packages/validation-python/src/project-config-files.ts @@ -1,5 +1,5 @@ export const pythonBoundaryFileNames = [ - "pyproject.toml", "Pipfile", "Pipfile.lock", "poetry.lock", "pdm.lock", "uv.lock", "setup.cfg", "setup.py" + "pyproject.toml", "Pipfile", "Pipfile.lock", "poetry.lock", "pdm.lock", "uv.lock", "setup.cfg", "setup.py", "pytest.ini", "tox.ini" ] as const; const pythonToolConfigFileNames = [ @@ -9,6 +9,7 @@ const pythonToolConfigFileNames = [ export function isRelevantPythonConfig(path: string): boolean { const name = path.slice(path.lastIndexOf("/") + 1); return pythonBoundaryFileNames.includes(name as (typeof pythonBoundaryFileNames)[number]) || + name === "conftest.py" || /^requirements.*\.txt$/u.test(name) || pythonToolConfigFileNames.includes(name as (typeof pythonToolConfigFileNames)[number]); } diff --git a/packages/validation-python/src/project-context.ts b/packages/validation-python/src/project-context.ts index d244440..3a5d988 100644 --- a/packages/validation-python/src/project-context.ts +++ b/packages/validation-python/src/project-context.ts @@ -18,6 +18,7 @@ import { normalizePythonProjectFingerprintInput, pythonProjectDigest } from "./project-fingerprint.js"; +import { isRelevantPythonConfig } from "./project-config-files.js"; import type { PythonProjectWorkspace } from "./project-workspace.js"; import { readPythonStaticProjectConfig } from "./static-config.js"; @@ -177,7 +178,9 @@ function normalizeTargets(targets: readonly string[]): readonly string[] { return [...new Set(targets.map((target) => { try { const normalized = validateRepoRelativePath(target.replaceAll("\\", "/")); - if (!/\.pyi?$/u.test(normalized)) throw new Error(`Python project context target must be .py or .pyi: ${normalized}`); + if (!/\.pyi?$/u.test(normalized) && !isRelevantPythonConfig(normalized)) { + throw new Error(`Python project context target must be Python source or config: ${normalized}`); + } return normalized; } catch (error) { throw new PythonProjectContextResolutionError(error instanceof Error ? error.message : String(error)); diff --git a/packages/validation-python/src/project-workspace.ts b/packages/validation-python/src/project-workspace.ts index 7cdf1b7..4e80bb0 100644 --- a/packages/validation-python/src/project-workspace.ts +++ b/packages/validation-python/src/project-workspace.ts @@ -14,9 +14,11 @@ export interface PythonProjectWorkspaceRealpath { export interface PythonProjectWorkspace { read(path: string): Promise; list(): Promise; + listAll?(): Promise; exists(path: string): Promise; realpath(path: string): Promise; executableExists(path: string): Promise; + statMode?(path: string): Promise; } export function createValidationFileViewPythonWorkspace( @@ -25,36 +27,41 @@ export function createValidationFileViewPythonWorkspace( fullWorkspace?: PythonProjectWorkspace ): PythonProjectWorkspace { const executableAvailable = executableExists ?? fullWorkspace?.executableExists ?? nodeExecutableExists; + const listPaths = async (include: (path: string) => boolean) => { + const fileViewPaths = await fileView.listVisibleFiles(); + const fullWorkspacePaths = fullWorkspace === undefined ? [] : await fullWorkspace.list(); + const fileViewPathSet = new Set(fileViewPaths); + const fullWorkspacePathSet = new Set(fullWorkspacePaths); + const candidates = [...new Set([...fileViewPaths, ...fullWorkspacePaths])].filter(include).sort(); + const visible: string[] = []; + for (const path of candidates) { + if (fileView.defaultReadState === "after") { + const overlay = fileView.overlayFor(path); + if (overlay?.action === "delete") continue; + if (overlay?.action === "write" + || (fileViewPathSet.has(path) && (fullWorkspace === undefined || fullWorkspacePathSet.has(path)))) { + visible.push(path); + continue; + } + } + if (await fileView.exists(path)) visible.push(path); + } + return visible; + }; return { read: async (path) => { const result = await fileView.readAfter(validateRepoRelativePath(path)); return result.status === "found" ? result.content : undefined; }, - list: async () => { - const fileViewPaths = await fileView.listVisibleFiles(); - const fullWorkspacePaths = fullWorkspace === undefined ? [] : await fullWorkspace.list(); - const fileViewPathSet = new Set(fileViewPaths); - const fullWorkspacePathSet = new Set(fullWorkspacePaths); - const candidates = [...new Set([...fileViewPaths, ...fullWorkspacePaths])] - .filter(isPythonProjectWorkspaceInput) - .sort(); - const visible: string[] = []; - for (const path of candidates) { - if (fileView.defaultReadState === "after") { - const overlay = fileView.overlayFor(path); - if (overlay?.action === "delete") continue; - if (overlay?.action === "write" - || (fileViewPathSet.has(path) && (fullWorkspace === undefined || fullWorkspacePathSet.has(path)))) { - visible.push(path); - continue; - } - } - if (await fileView.exists(path)) visible.push(path); - } - return visible; - }, + list: async () => listPaths(isPythonProjectWorkspaceInput), + listAll: async () => listPaths(() => true), exists: (path) => fileView.exists(validateRepoRelativePath(path)), realpath: async (path) => { + if (path === ".") { + return fullWorkspace === undefined + ? { path: ".", symlink: false, unavailable: true } + : fullWorkspace.realpath(path); + } const normalized = validateRepoRelativePath(path); if (fullWorkspace === undefined) return { path: normalized, symlink: false, unavailable: true }; const baseline = await fullWorkspace.realpath(normalized); @@ -63,7 +70,10 @@ export function createValidationFileViewPythonWorkspace( if (baseline.unavailable && await fullWorkspace.exists(normalized)) return baseline; return { path: normalized, symlink: false }; }, - executableExists: executableAvailable + executableExists: executableAvailable, + statMode: fullWorkspace?.statMode === undefined + ? undefined + : async (path) => fullWorkspace.statMode?.(validateRepoRelativePath(path)) }; } function isPythonProjectWorkspaceInput(path: string): boolean { @@ -97,6 +107,7 @@ export function createNodePythonProjectWorkspace(repoRoot: string): PythonProjec } }, realpath: async (path) => { + if (path === ".") return { path: ".", symlink: false }; const normalized = validateRepoRelativePath(path); const absolute = resolveRepoPath(canonicalRoot, normalized); try { @@ -110,7 +121,15 @@ export function createNodePythonProjectWorkspace(repoRoot: string): PythonProjec throw error; } }, - executableExists: nodeExecutableExists + executableExists: nodeExecutableExists, + statMode: async (path) => { + try { + return (await lstat(resolveRepoPath(canonicalRoot, path))).mode; + } catch (error) { + if (isMissing(error)) return undefined; + throw error; + } + } }; } diff --git a/packages/validation-python/src/pytest-check.ts b/packages/validation-python/src/pytest-check.ts new file mode 100644 index 0000000..576889c --- /dev/null +++ b/packages/validation-python/src/pytest-check.ts @@ -0,0 +1,198 @@ +import type { + GraphFactEdge, + PythonProjectContext, + PythonProjectToolProvenance, + PythonValidationCapabilityRun, + ValidationDiagnostic +} from "@the-open-engine/opcore-contracts"; +import type { ValidationCheckContext, ValidationCheckDefinition } from "@the-open-engine/opcore-validation"; +import { PYTHON_PYTEST_CHECK_ID } from "./check-ids.js"; +import { pythonCheckAdapter, pythonCheckOwner, supportedPythonValidationScopes } from "./check-constants.js"; +import { createRelevantTestsGraphRequirements } from "./graph-requirements.js"; +import { isRelevantPythonConfig } from "./project-config-files.js"; +import { executeProjectPytest } from "./pytest-project-runner.js"; +import { candidateFailure, disabledPytestRun, failureRun, notApplicableRun, unsupportedPytestTool, unsupportedRun } from "./pytest-result.js"; +import type { ProjectRunInput } from "./pytest-types.js"; +import { pytestWorkspaceCaps } from "./pytest-workspace.js"; +import { type PythonProjectWorkspace } from "./project-workspace.js"; +import { + isPythonSourcePath, + pythonProjectInputSet, + type PythonProjectContextResolver, + type PythonSourceRootResolver +} from "./source-files.js"; +import type { PythonValidationToolchainOptions } from "./toolchain.js"; + +interface CreatePytestCheckOptions extends Omit {} + +interface CandidateSelectionState { + exactTargets: ReadonlySet; + projectWideRoots: readonly PythonProjectContext[]; +} + +export { disabledPytestRun }; + +export function createPytestCheck( + options: CreatePytestCheckOptions = {}, + resolveContexts?: PythonProjectContextResolver, + resolveRoots?: PythonSourceRootResolver +): ValidationCheckDefinition { + const resolveRootPaths = resolveRoots ?? (async (context) => pythonProjectInputSet(context)); + return { + id: PYTHON_PYTEST_CHECK_ID, + owner: pythonCheckOwner, + adapter: pythonCheckAdapter, + defaultSeverity: "warning", + supportedScopes: supportedPythonValidationScopes, + defaultScopes: [], + inactiveResult: async (context, state) => + !hasRelevantPytestInput(context) + ? undefined + : state === "disabled" + ? disabledPytestRun() + : notApplicableRun("Python pytest execution is opt-in."), + inactiveStateWhenUnselected: "not_applicable", + requiresGraph: true, + graphRequirements: createRelevantTestsGraphRequirements(resolveRootPaths), + run: async (context) => runPytestCheck(context, options.nodeWorkspace, resolveContexts) + }; +} + +async function runPytestCheck( + context: ValidationCheckContext, + nodeWorkspace: PythonProjectWorkspace | undefined, + resolveContexts: PythonProjectContextResolver | undefined +) { + const targets = pythonProjectInputSet(context); + if (targets.length === 0) return notApplicableRun("No Python source or config targets selected."); + if (resolveContexts === undefined) { + return failureRun("PYTHON_PYTEST_CONTEXT_MISSING", "Canonical Python project context resolver is required.", "tool_failure"); + } + const rootContexts = await resolveContexts(context, targets); + const unresolved = rootContexts.find(isUnresolvedPytestContext); + if (unresolved !== undefined) { + return unsupportedRun(unresolved, `Python pytest execution requires a resolved project context for ${unresolved.target}.`); + } + const selectedCandidates = selectCandidatePaths(rootContexts, await context.graph.testedBy()); + if (selectedCandidates.length === 0) { + return candidateFailure(rootContexts, [], "No TESTED_BY graph candidate tests matched the selected Python targets."); + } + if (selectedCandidates.length > pytestWorkspaceCaps.maxCandidateFiles) { + return failureRun("PYTHON_PYTEST_CANDIDATE_OVERFLOW", `Pytest candidate selection exceeded ${pytestWorkspaceCaps.maxCandidateFiles} files.`, "tool_failure", rootContexts, selectedCandidates); + } + const candidateContexts = await resolveContexts(context, selectedCandidates); + const groups = groupProjects(candidateContexts, selectedCandidates); + const diagnostics: ValidationDiagnostic[] = []; + const capabilityRuns: PythonValidationCapabilityRun[] = []; + let overallPassed = true; + for (const group of groups) { + if (group.candidatePaths.some((path) => !candidateContexts.some((entry) => entry.target === path))) { + return failureRun("PYTHON_PYTEST_CANDIDATE_MISSING", "Pytest candidate context resolution omitted at least one candidate test.", "tool_failure", rootContexts, group.candidatePaths); + } + const pytest = selectPytestTool(group.context); + if (pytest === undefined) { + const result = unsupportedPytestTool(group.context, group.candidatePaths); + diagnostics.push(...(result.diagnostics ?? [])); + capabilityRuns.push(...(result.pythonCapabilityRuns ?? [])); + overallPassed = false; + continue; + } + const run = await executeProjectPytest(context, nodeWorkspace, group, pytest); + diagnostics.push(...run.diagnostics); + capabilityRuns.push(run.capabilityRun); + if (run.outcome !== "passed") overallPassed = false; + } + const outcome: "passed" | "findings" = overallPassed && capabilityRuns.some((run) => run.counts?.passedCount && run.cleanup?.ok) + ? "passed" + : "findings"; + return { + diagnostics, + outcome, + pythonProjectContexts: rootContexts, + pythonCapabilityRuns: capabilityRuns + }; +} + +function selectCandidatePaths(rootContexts: readonly PythonProjectContext[], testedBy: readonly GraphFactEdge[]): readonly string[] { + const state: CandidateSelectionState = { + exactTargets: new Set( + rootContexts + .map((context) => context.target) + .filter((target) => isPythonSourcePath(target) && !isRelevantPythonConfig(target)) + ), + projectWideRoots: [] + }; + state.projectWideRoots = rootContexts.filter((context) => !state.exactTargets.has(context.target)); + const selected = new Set(); + for (const edge of testedBy) { + const fromPath = endpointFilePath(edge.from); + const toPath = endpointFilePath(edge.to); + const forward = selectedCandidatePath(edge.from, fromPath, edge.to, toPath, state); + if (forward !== undefined) selected.add(forward); + const reverse = selectedCandidatePath(edge.to, toPath, edge.from, fromPath, state); + if (reverse !== undefined) selected.add(reverse); + } + return [...selected].sort(); +} + +function selectedCandidatePath( + sourceEndpoint: string, + sourcePath: string | undefined, + candidateEndpoint: string, + candidatePath: string | undefined, + state: CandidateSelectionState +): string | undefined { + if (sourcePath === undefined || candidatePath === undefined || !looksLikeTestCandidate(candidateEndpoint, candidatePath)) return undefined; + if (state.exactTargets.has(sourcePath)) return candidatePath; + return state.projectWideRoots.some((context) => withinProject(sourcePath, context.projectRoot) && withinProject(candidatePath, context.projectRoot)) + ? candidatePath + : undefined; +} + +function groupProjects(contexts: readonly PythonProjectContext[], candidatePaths: readonly string[]): readonly ProjectRunInput[] { + const groups = new Map(); + for (const context of contexts) { + const group = groups.get(context.projectKey) ?? { context, candidatePaths: [] }; + if (candidatePaths.includes(context.target)) group.candidatePaths.push(context.target); + groups.set(context.projectKey, group); + } + return [...groups.values()] + .map((entry) => ({ context: entry.context, candidatePaths: [...new Set(entry.candidatePaths)].sort() })) + .filter((entry) => entry.candidatePaths.length > 0) + .sort((left, right) => left.context.projectRoot.localeCompare(right.context.projectRoot)); +} + +function selectPytestTool(context: PythonProjectContext): PythonProjectToolProvenance | undefined { + return context.tools.find((tool) => tool.tool === "pytest" && tool.available); +} + +function isUnresolvedPytestContext(context: PythonProjectContext): boolean { + if (context.interpreter === undefined || context.outcome === "ambiguous" || context.outcome === "unsupported") return true; + return context.reasons.some((reason) => + reason.code === "invalid_config" || + reason.code === "path_refused" || + reason.code === "symlink_refused" || + reason.code === "ambiguous_path" || + reason.tool === "python" + ); +} + +function endpointFilePath(endpoint: string): string | undefined { + const match = /^[^:]+:([^#]+)(?:#.*)?$/u.exec(endpoint); + return match?.[1]; +} + +function hasRelevantPytestInput(context: ValidationCheckContext): boolean { + return pythonProjectInputSet(context).length > 0; +} + +function looksLikeTestCandidate(endpoint: string, path: string): boolean { + if (!path.endsWith(".py")) return false; + if (/^[^:]*test[^:]*:/iu.test(endpoint)) return true; + const name = path.slice(path.lastIndexOf("/") + 1); + return /(^|\/)tests?\//u.test(path) || name.startsWith("test_") || name.endsWith("_test.py"); +} + +function withinProject(path: string, projectRoot: string): boolean { + return projectRoot === "." || path === projectRoot || path.startsWith(`${projectRoot}/`); +} diff --git a/packages/validation-python/src/pytest-hook-source.ts b/packages/validation-python/src/pytest-hook-source.ts new file mode 100644 index 0000000..10c5924 --- /dev/null +++ b/packages/validation-python/src/pytest-hook-source.ts @@ -0,0 +1,47 @@ +export function pytestHookSource(): string { + return [ + "import json", + "import os", + "from pathlib import Path", + "", + "_output_path = Path(os.environ['OPCORE_PYTEST_HOOK_OUTPUT'])", + "_selection_manifest = os.environ.get('OPCORE_PYTEST_SELECTION_MANIFEST')", + "_selected_ids = None", + "if _selection_manifest:", + " _selected_ids = set()", + " for line in Path(_selection_manifest).read_text(encoding='utf-8').splitlines():", + " line = line.strip()", + " if line:", + " _selected_ids.add(line)", + "", + "def _emit(payload):", + " with _output_path.open('a', encoding='utf-8') as handle:", + " handle.write(json.dumps(payload, sort_keys=True) + '\\n')", + "", + "def pytest_collection_modifyitems(session, config, items):", + " if _selected_ids is not None:", + " selected = [item for item in items if item.nodeid in _selected_ids]", + " items[:] = selected", + " for item in items:", + " _emit({'type': 'collected', 'nodeid': item.nodeid})", + "", + "def pytest_collectreport(report):", + " if report.failed:", + " _emit({'type': 'collect_error', 'nodeid': getattr(report, 'nodeid', ''), 'message': str(report.longrepr)})", + "", + "def pytest_runtest_logreport(report):", + " payload = {", + " 'type': 'test_report',", + " 'nodeid': report.nodeid,", + " 'when': report.when,", + " 'outcome': report.outcome", + " }", + " wasxfail = getattr(report, 'wasxfail', None)", + " if wasxfail is not None:", + " payload['wasxfail'] = str(wasxfail)", + " _emit(payload)", + "", + "def pytest_sessionfinish(session, exitstatus):", + " _emit({'type': 'session_finish', 'exitstatus': int(exitstatus)})" + ].join("\n"); +} diff --git a/packages/validation-python/src/pytest-process.ts b/packages/validation-python/src/pytest-process.ts new file mode 100644 index 0000000..3791c66 --- /dev/null +++ b/packages/validation-python/src/pytest-process.ts @@ -0,0 +1,232 @@ +import { createHash } from "node:crypto"; +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { PythonCapabilityInvocation, PythonProjectToolProvenance } from "@the-open-engine/opcore-contracts"; +import { runTool, type PythonToolRunResult } from "./process.js"; +import { pytestHookSource } from "./pytest-hook-source.js"; +import { pytestDiagnostic } from "./pytest-result.js"; +import { + analyzeExecutionEvents, + collectErrorCount, + countResults, + executionFailureDiagnostics, + readHookReport, + uniqueCollectedNodeIds +} from "./pytest-protocol.js"; +import type { CollectionRunResult, ExecutionRunResult, ProjectRunInput } from "./pytest-types.js"; +import { pytestWorkspaceCaps, type MaterializedPytestWorkspace } from "./pytest-workspace.js"; + +export function writePytestRuntimeModule(runtimeRoot: string): string { + const runtimeModule = join(runtimeRoot, "opcore_pytest_hook.py"); + writeFileSync(runtimeModule, pytestHookSource(), "utf8"); + return runtimeModule; +} + +export async function runCollection( + materialized: MaterializedPytestWorkspace, + group: ProjectRunInput, + pytest: PythonProjectToolProvenance +): Promise { + const outputPath = join(materialized.runtimeRoot, "collection.jsonl"); + const selectionDigest = sha256(group.candidatePaths); + const args = [ + ...pytest.argv.slice(1), + "-p", "no:cacheprovider", + "-p", "opcore_pytest_hook", + "--collect-only", + "-q", + ...group.candidatePaths.map((path) => relativeProjectPath(path, group.context.projectRoot)) + ]; + const startedAt = Date.now(); + const result = runTool(pytest.executable, args, { + cwd: materialized.projectCwd, + env: pytestEnv(materialized.runtimeRoot, outputPath), + timeoutMs: 30000, + allowedExitCodes: [0, 1, 2, 3, 4, 5], + maxOutputBytes: pytestWorkspaceCaps.maxProcessOutputBytes + }); + const invocation = invocationSummary("collection", pytest, args, result, "direct_argv", selectionDigest, Date.now() - startedAt); + const report = readHookReport(outputPath, { requireSessionFinish: result.termination === "exited" }); + return collectionOutcome(result, report, invocation, selectionDigest, group.candidatePaths.length); +} + +export async function runExecution( + materialized: MaterializedPytestWorkspace, + group: ProjectRunInput, + pytest: PythonProjectToolProvenance, + nodeIds: readonly string[] +): Promise { + const outputPath = join(materialized.runtimeRoot, "execution.jsonl"); + const selectionDigest = sha256(nodeIds); + const args = [...pytest.argv.slice(1), "-p", "no:cacheprovider", "-p", "opcore_pytest_hook", "-q"]; + const env = pytestEnv(materialized.runtimeRoot, outputPath); + const selectionMode = maybeAttachSelectionManifest(args, env, materialized.runtimeRoot, nodeIds); + if (selectionMode instanceof Error) { + return { + outcome: "tool_failure", + message: selectionMode.message, + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_SELECTION_OVERFLOW", selectionMode.message, group.context.target)], + counts: countResults(group.candidatePaths.length, nodeIds.length, 0, 0, 0, 0, 0, 0, 0), + invocation: invocationSummary("execution", pytest, args, spawnErrorResult(pytest.executable, args, materialized.projectCwd, selectionMode.message), "manifest", selectionDigest, 0), + selectionMode: "manifest", + selectionDigest + }; + } + const startedAt = Date.now(); + const result = runTool(pytest.executable, args, { + cwd: materialized.projectCwd, + env, + timeoutMs: 30000, + allowedExitCodes: [0, 1, 2, 3, 4, 5], + maxOutputBytes: pytestWorkspaceCaps.maxProcessOutputBytes + }); + const invocation = invocationSummary("execution", pytest, args, result, selectionMode, selectionDigest, Date.now() - startedAt); + const report = readHookReport(outputPath, { requireSessionFinish: result.termination === "exited" }); + return executionOutcome(result, report, invocation, selectionMode, selectionDigest, group.context.target, group.candidatePaths.length, nodeIds); +} + +function collectionOutcome( + result: PythonToolRunResult, + report: ReturnType, + invocation: PythonCapabilityInvocation, + selectionDigest: string, + candidateCount: number +): CollectionRunResult { + const nodeIds = report.events.some((event) => event.type === "test_report") ? [] : uniqueCollectedNodeIds(report.events); + const counts = countResults(candidateCount, nodeIds.length, 0, 0, 0, 0, 0, 0, collectErrorCount(report.events)); + if (result.termination === "exited" && report.exitStatus !== result.exitCode) return failedCollection("tool_failure", `Pytest collection hook exit status ${report.exitStatus} did not match process exit code ${result.exitCode}.`, counts, invocation, selectionDigest); + if (report.events.some((event) => event.type === "test_report")) return failedCollection("tool_failure", "Pytest collection emitted test execution reports during --collect-only.", counts, invocation, selectionDigest); + if (result.termination === "timeout") return failedCollection("timeout", result.failureMessage ?? "pytest collection timed out", counts, invocation, selectionDigest); + if (result.termination !== "exited") return failedCollection("tool_failure", result.failureMessage ?? "pytest collection failed", counts, invocation, selectionDigest); + if (result.exitCode === 4) return failedCollection("invalid_config", "Pytest collection reported invalid configuration.", counts, invocation, selectionDigest); + if (result.exitCode !== 0 && result.exitCode !== 1) return failedCollection("tool_failure", `Pytest collection exited with code ${result.exitCode}.`, counts, invocation, selectionDigest); + if (result.exitCode === 1 && collectErrorCount(report.events) === 0) return failedCollection("tool_failure", "Pytest collection exited with code 1 but emitted no collection-error events.", counts, invocation, selectionDigest, nodeIds); + if (result.exitCode === 1) return failedCollection("findings", "Pytest collection reported collection errors.", counts, invocation, selectionDigest, nodeIds); + if (collectErrorCount(report.events) > 0) return failedCollection("tool_failure", "Pytest collection emitted collection-error events despite a zero exit code.", counts, invocation, selectionDigest); + if (nodeIds.length === 0) return failedCollection("findings", "Pytest collection produced zero node ids.", counts, invocation, selectionDigest); + if (nodeIds.length > pytestWorkspaceCaps.maxCollectedNodeIds) return failedCollection("tool_failure", `Pytest collection exceeded ${pytestWorkspaceCaps.maxCollectedNodeIds} node ids.`, counts, invocation, selectionDigest); + if (Buffer.byteLength(nodeIds.join("\n"), "utf8") > pytestWorkspaceCaps.maxNodeIdBytes) return failedCollection("tool_failure", `Pytest collection node ids exceeded ${pytestWorkspaceCaps.maxNodeIdBytes} bytes.`, counts, invocation, selectionDigest); + return { outcome: "passed", message: "Pytest collection succeeded.", nodeIds, counts, invocation, selectionMode: "direct_argv", selectionDigest }; +} + +function executionOutcome( + result: PythonToolRunResult, + report: ReturnType, + invocation: PythonCapabilityInvocation, + selectionMode: "direct_argv" | "manifest", + selectionDigest: string, + path: string, + candidateCount: number, + nodeIds: readonly string[] +): ExecutionRunResult { + if (result.termination === "exited" && report.exitStatus !== result.exitCode) return failedExecution("tool_failure", `Pytest execution hook exit status ${report.exitStatus} did not match process exit code ${result.exitCode}.`, [pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution hook exit status ${report.exitStatus} did not match process exit code ${result.exitCode}.`, path)], countResults(candidateCount, nodeIds.length, 0, 0, 0, 0, 0, 0, 0), invocation, selectionMode, selectionDigest); + const analysis = analyzeExecutionEvents(candidateCount, nodeIds, report.events, path); + if (result.termination === "timeout") return failedExecution("timeout", result.failureMessage ?? "pytest execution timed out", [pytestDiagnostic("PYTHON_PYTEST_TIMEOUT", result.failureMessage ?? "pytest execution timed out", path)], analysis.counts, invocation, selectionMode, selectionDigest); + if (result.termination !== "exited") return failedExecution("tool_failure", result.failureMessage ?? "pytest execution failed", [pytestDiagnostic("PYTHON_PYTEST_TOOL_FAILED", result.failureMessage ?? "pytest execution failed", path)], analysis.counts, invocation, selectionMode, selectionDigest); + if (analysis.diagnostics.length > 0) return failedExecution("tool_failure", analysis.diagnostics[0]?.message ?? "Pytest execution hook protocol mismatch.", analysis.diagnostics, analysis.counts, invocation, selectionMode, selectionDigest); + if (analysis.counts.executedCount === 0) return failedExecution("findings", "Pytest execution ran zero tests.", [pytestDiagnostic("PYTHON_PYTEST_NO_EXECUTION", "Pytest execution ran zero tests.", path)], analysis.counts, invocation, selectionMode, selectionDigest); + if (analysis.counts.passedCount === 0) return failedExecution("findings", "Pytest execution produced no passing tests.", [pytestDiagnostic("PYTHON_PYTEST_NO_PASS", "Pytest execution produced no passing tests.", path)], analysis.counts, invocation, selectionMode, selectionDigest); + const errors = executionFailureDiagnostics(report.events, path); + if (result.exitCode !== 0 || errors.length > 0 || analysis.counts.failedCount > 0 || analysis.counts.xpassedCount > 0 || analysis.counts.errorCount > 0) { + return failedExecution("findings", "Pytest execution reported failing outcomes.", errors.length > 0 ? errors : [pytestDiagnostic("PYTHON_PYTEST_FAILURES", "Pytest execution reported failing outcomes.", path)], analysis.counts, invocation, selectionMode, selectionDigest); + } + return { outcome: "passed", message: "Pytest execution reported passing tests.", diagnostics: [], counts: analysis.counts, invocation, selectionMode, selectionDigest }; +} + +function failedCollection( + outcome: CollectionRunResult["outcome"], + message: string, + counts: CollectionRunResult["counts"], + invocation: CollectionRunResult["invocation"], + selectionDigest: string, + nodeIds: readonly string[] = [] +): CollectionRunResult { + return { outcome, message, nodeIds, counts, invocation, selectionMode: "direct_argv", selectionDigest }; +} + +function failedExecution( + outcome: ExecutionRunResult["outcome"], + message: string, + diagnostics: ExecutionRunResult["diagnostics"], + counts: ExecutionRunResult["counts"], + invocation: ExecutionRunResult["invocation"], + selectionMode: ExecutionRunResult["selectionMode"], + selectionDigest: string +): ExecutionRunResult { + return { outcome, message, diagnostics, counts, invocation, selectionMode, selectionDigest }; +} + +function maybeAttachSelectionManifest( + args: string[], + env: Record, + runtimeRoot: string, + nodeIds: readonly string[] +): "direct_argv" | "manifest" | Error { + if (Buffer.byteLength(nodeIds.join("\0"), "utf8") <= pytestWorkspaceCaps.maxArgvBytes) { + args.push(...nodeIds); + return "direct_argv"; + } + const manifest = `${nodeIds.join("\n")}\n`; + if (Buffer.byteLength(manifest, "utf8") > pytestWorkspaceCaps.maxManifestBytes) { + return new Error(`Pytest selection manifest exceeded ${pytestWorkspaceCaps.maxManifestBytes} bytes.`); + } + const manifestPath = join(runtimeRoot, "selection.txt"); + writeFileSync(manifestPath, manifest, "utf8"); + env.OPCORE_PYTEST_SELECTION_MANIFEST = manifestPath; + return "manifest"; +} + +function pytestEnv(runtimeRoot: string, outputPath: string): Record { + return { ...process.env, PYTHONPATH: runtimeRoot, PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1", OPCORE_PYTEST_HOOK_OUTPUT: outputPath }; +} + +function invocationSummary( + stage: "collection" | "execution", + pytest: PythonProjectToolProvenance, + args: readonly string[], + result: PythonToolRunResult, + selectionMode: "direct_argv" | "manifest", + selectionDigest: string, + durationMs: number +): PythonCapabilityInvocation { + return { + stage, + command: pytest.tool, + argsDigest: sha256([pytest.executable, ...args]), + argCount: args.length, + selectionMode, + selectionDigest, + durationMs, + termination: result.termination, + ...(result.exitCode === null ? {} : { exitCode: result.exitCode }), + ...(result.signal === null ? {} : { signal: result.signal }), + outputBytes: Buffer.byteLength(result.stdout, "utf8") + Buffer.byteLength(result.stderr, "utf8"), + stdoutDigest: sha256(result.stdout), + stderrDigest: sha256(result.stderr) + }; +} + +function relativeProjectPath(path: string, projectRoot: string): string { + return projectRoot === "." ? path : path.slice(`${projectRoot}/`.length); +} + +function sha256(value: unknown): string { + const input = typeof value === "string" ? value : JSON.stringify(value); + return `sha256:${createHash("sha256").update(input, "utf8").digest("hex")}`; +} + +function spawnErrorResult(command: string, args: readonly string[], cwd: string, failureMessage: string): PythonToolRunResult { + return { + ok: false, + termination: "spawn_error", + command, + args: [...args], + cwd, + allowedExitCodes: [0], + exitCode: null, + signal: null, + stdout: "", + stderr: "", + failureMessage + }; +} diff --git a/packages/validation-python/src/pytest-project-runner.ts b/packages/validation-python/src/pytest-project-runner.ts new file mode 100644 index 0000000..b0609b3 --- /dev/null +++ b/packages/validation-python/src/pytest-project-runner.ts @@ -0,0 +1,134 @@ +import { existsSync } from "node:fs"; +import type { PythonProjectToolProvenance, PythonValidationCapabilityRun } from "@the-open-engine/opcore-contracts"; +import type { ValidationCheckContext } from "@the-open-engine/opcore-validation"; +import { removeMaterializedWorkspace } from "./materialized-workspace.js"; +import type { PythonProjectWorkspace } from "./project-workspace.js"; +import { collectPytestProjectPaths, materializePytestWorkspace } from "./pytest-workspace.js"; +import { runCollection, runExecution, writePytestRuntimeModule } from "./pytest-process.js"; +import { cleanupFailureMessage, createCleanupState, finalizeCapabilityRun, pytestDiagnostic, recordCleanup } from "./pytest-result.js"; +import type { ProjectRunInput, ProjectRunResult } from "./pytest-types.js"; +import { PYTHON_PYTEST_CHECK_ID } from "./check-ids.js"; + +export async function executeProjectPytest( + validation: ValidationCheckContext, + nodeWorkspace: PythonProjectWorkspace | undefined, + group: ProjectRunInput, + pytest: PythonProjectToolProvenance +): Promise { + const projectPaths = await collectPytestProjectPaths(validation, group.context, group.candidatePaths, nodeWorkspace); + const capabilityBase = { + capability: "pytest" as const, + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled" as const, + projectKey: group.context.projectKey, + projectRoot: group.context.projectRoot, + configFile: pytest.configFile, + targetCount: group.candidatePaths.length, + candidatePaths: group.candidatePaths + }; + const cleanup = createCleanupState(); + let afterStateFingerprint: string | undefined; + let collectedNodeIds: readonly string[] | undefined; + let collectionSelectionMode: "direct_argv" | "manifest" | "none" = "none"; + let collectionSelectionDigest: string | undefined; + let collectionCounts; + let collectionInvocation; + try { + const collectionWorkspace = await materializePytestWorkspace({ context: validation, project: group.context, paths: projectPaths, nodeWorkspace }); + afterStateFingerprint = collectionWorkspace.afterStateFingerprint; + let collectionFailure: ProjectRunResult | undefined; + try { + writePytestRuntimeModule(collectionWorkspace.runtimeRoot); + const collection = await runCollection(collectionWorkspace, group, pytest); + collectionInvocation = collection.invocation; + collectionCounts = collection.counts; + collectionSelectionMode = collection.selectionMode; + collectionSelectionDigest = collection.selectionDigest; + collectedNodeIds = collection.nodeIds; + if (collection.outcome !== "passed") { + collectionFailure = { + outcome: collection.outcome, + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_COLLECTION_FAILED", collection.message, group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: collection.outcome, message: collection.message, selectionMode: collection.selectionMode, selectionDigest: collection.selectionDigest, counts: collection.counts, collection: collection.invocation } + }; + } + } finally { + cleanupWorkspace(cleanup, collectionWorkspace); + } + if (collectionFailure !== undefined) return finalizeCapabilityRun(cleanup, collectionFailure); + if (!cleanup.ok) return cleanupFailureResult(cleanup, capabilityBase, afterStateFingerprint, collectedNodeIds, collectionSelectionMode, collectionSelectionDigest, collectionCounts, collectionInvocation, group.context.target); + const executionWorkspace = await materializePytestWorkspace({ context: validation, project: group.context, paths: projectPaths, nodeWorkspace }); + if (executionWorkspace.afterStateFingerprint !== afterStateFingerprint) { + recordCleanup(cleanup, executionWorkspace.cleanup); + return finalizeCapabilityRun(cleanup, { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_FINGERPRINT_MISMATCH", "Pytest execution after-state fingerprint did not match collection.", group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message: "Pytest execution after-state fingerprint did not match collection.", collectedNodeIds, selectionMode: collectionSelectionMode, selectionDigest: collectionSelectionDigest, counts: collectionCounts, collection: collectionInvocation } + }); + } + let executionResult: ProjectRunResult | undefined; + try { + writePytestRuntimeModule(executionWorkspace.runtimeRoot); + const execution = await runExecution(executionWorkspace, group, pytest, collectedNodeIds ?? []); + executionResult = { + outcome: execution.outcome, + diagnostics: execution.diagnostics, + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: execution.outcome, message: execution.message, collectedNodeIds, selectionMode: execution.selectionMode, selectionDigest: execution.selectionDigest, counts: execution.counts, collection: collectionInvocation, execution: execution.invocation } + }; + } finally { + cleanupWorkspace(cleanup, executionWorkspace); + } + return finalizeCapabilityRun(cleanup, executionResult ?? { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_TOOL_FAILED", "Pytest execution produced no result.", group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message: "Pytest execution produced no result.", collectedNodeIds, selectionMode: collectionSelectionMode, selectionDigest: collectionSelectionDigest, counts: collectionCounts, collection: collectionInvocation } + }); + } catch (error) { + return finalizeCapabilityRun(cleanup, { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_TOOL_FAILED", errorMessage(error), group.context.target)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message: errorMessage(error), collectedNodeIds, selectionMode: collectionSelectionMode, selectionDigest: collectionSelectionDigest, counts: collectionCounts, collection: collectionInvocation } + }); + } +} + +function cleanupFailureResult( + cleanup: Parameters[0], + capabilityBase: { + capability: "pytest"; + checkId: string; + activation: "enabled"; + projectKey: string; + projectRoot: string; + configFile?: string; + targetCount: number; + candidatePaths: readonly string[]; + }, + afterStateFingerprint: string | undefined, + collectedNodeIds: readonly string[] | undefined, + selectionMode: "direct_argv" | "manifest" | "none", + selectionDigest: string | undefined, + counts: PythonValidationCapabilityRun["counts"], + collection: PythonValidationCapabilityRun["collection"], + path: string +): ProjectRunResult { + const message = cleanupFailureMessage(cleanup); + return finalizeCapabilityRun(cleanup, { + outcome: "tool_failure", + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_CLEANUP_FAILED", message, path)], + capabilityRun: { ...capabilityBase, afterStateFingerprint, outcome: "tool_failure", message, collectedNodeIds, selectionMode, selectionDigest, counts, collection } + }); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function cleanupWorkspace( + cleanup: Parameters[0], + workspace: { root: string; cleanup(): void } +): void { + recordCleanup(cleanup, workspace.cleanup); + if (!existsSync(workspace.root)) return; + recordCleanup(cleanup, () => removeMaterializedWorkspace(workspace.root, "Pytest temporary workspace")); +} diff --git a/packages/validation-python/src/pytest-protocol.ts b/packages/validation-python/src/pytest-protocol.ts new file mode 100644 index 0000000..903f83b --- /dev/null +++ b/packages/validation-python/src/pytest-protocol.ts @@ -0,0 +1,224 @@ +import type { PythonCapabilityCounts, ValidationDiagnostic } from "@the-open-engine/opcore-contracts"; +import { readFileSync } from "node:fs"; +import { pytestWorkspaceCaps } from "./pytest-workspace.js"; +import { pytestDiagnostic } from "./pytest-result.js"; +import type { ExecutionEventAnalysis, HookEvent, HookReport } from "./pytest-types.js"; + +export function readHookReport(path: string, options: { requireSessionFinish?: boolean } = {}): HookReport { + const requireSessionFinish = options.requireSessionFinish ?? true; + try { + const text = readFileSync(path, "utf8"); + if (Buffer.byteLength(text, "utf8") > pytestWorkspaceCaps.maxHookReportBytes) { + throw new Error(`Pytest hook report exceeded ${pytestWorkspaceCaps.maxHookReportBytes} bytes.`); + } + const events: HookEvent[] = []; + let exitStatus: number | undefined; + for (const [index, line] of text.split(/\r?\n/u).filter(Boolean).entries()) { + const payload = JSON.parse(line) as unknown; + if (!isRecord(payload)) throw new Error(`Pytest hook event ${index + 1} is not an object.`); + if (payload.type === "session_finish") { + if (exitStatus !== undefined) throw new Error("Duplicate pytest session_finish event."); + if (!Number.isInteger(payload.exitstatus)) throw new Error("Pytest session_finish event requires integer exitstatus."); + exitStatus = payload.exitstatus as number; + continue; + } + events.push(validateHookEvent(payload, index + 1)); + } + if (exitStatus === undefined) { + if (!requireSessionFinish) return { events, exitStatus: -1 }; + throw new Error("Pytest hook report omitted session_finish."); + } + return { events, exitStatus }; + } catch (error) { + if ((error as { code?: string }).code === "ENOENT") { + if (!requireSessionFinish) return { events: [], exitStatus: -1 }; + throw new Error("Pytest hook report was not created."); + } + throw error; + } +} + +export function uniqueCollectedNodeIds(events: readonly HookEvent[]): readonly string[] { + const nodeIds = new Set(); + for (const event of events) { + if (event.type !== "collected" || event.nodeid === undefined) continue; + if (nodeIds.has(event.nodeid)) throw new Error(`Duplicate pytest collection node id: ${event.nodeid}`); + nodeIds.add(event.nodeid); + } + return [...nodeIds]; +} + +export function collectErrorCount(events: readonly HookEvent[]): number { + return events.filter((event) => event.type === "collect_error").length; +} + +export function analyzeExecutionEvents( + candidateCount: number, + expectedNodeIds: readonly string[], + events: readonly HookEvent[], + path: string +): ExecutionEventAnalysis { + const expected = new Set(expectedNodeIds); + const collected = new Set(); + const terminal = new Map(); + const diagnostics: ValidationDiagnostic[] = []; + for (const event of events) processExecutionEvent(event, expected, collected, terminal, diagnostics, path); + if (collected.size !== expected.size || expectedNodeIds.some((nodeId) => !collected.has(nodeId))) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", "Pytest execution collected node ids did not match the collection selection.", path)); + } + const counts = summarizeExecutionCounts(candidateCount, expectedNodeIds.length, terminal, events); + if (counts.executedCount !== expectedNodeIds.length && collectErrorCount(events) === 0) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", "Pytest execution terminal node ids did not match the collected selection.", path)); + } + return { counts, diagnostics }; +} + +export function executionFailureDiagnostics(events: readonly HookEvent[], path: string) { + return events + .filter((event) => event.type === "collect_error" && event.message !== undefined) + .map((event) => pytestDiagnostic("PYTHON_PYTEST_COLLECTION_ERROR", event.message!, path)); +} + +export function countResults( + candidateCount: number, + collectedCount: number, + executedCount: number, + passedCount: number, + failedCount: number, + skippedCount: number, + xfailedCount: number, + xpassedCount: number, + errorCount: number +): PythonCapabilityCounts { + return { candidateCount, collectedCount, executedCount, passedCount, failedCount, skippedCount, xfailedCount, xpassedCount, errorCount }; +} + +interface NodeOutcomeState { + callSeen: boolean; + passed: boolean; + failed: boolean; + skipped: boolean; + xfailed: boolean; + xpassed: boolean; + error: boolean; + setupSkipped: boolean; +} + +function processExecutionEvent( + event: HookEvent, + expected: ReadonlySet, + collected: Set, + terminal: Map, + diagnostics: ReturnType, + path: string +): void { + if (event.type === "collected") return processCollectedEvent(event, expected, collected, diagnostics, path); + if (event.type !== "test_report" || event.nodeid === undefined || event.when === undefined || event.outcome === undefined) return; + if (!expected.has(event.nodeid)) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution reported unexpected node id: ${event.nodeid}`, path)); + return; + } + const current = terminal.get(event.nodeid) ?? emptyNodeOutcomeState(); + if (event.when === "call") { + processCallOutcome( + event as HookEvent & { nodeid: string; when: "call"; outcome: "passed" | "failed" | "skipped" }, + current, + diagnostics, + path, + terminal + ); + return; + } + if (event.outcome === "failed") current.error = true; + if (event.when === "setup" && event.outcome === "skipped") current.setupSkipped = true; + terminal.set(event.nodeid, current); +} + +function processCollectedEvent( + event: HookEvent, + expected: ReadonlySet, + collected: Set, + diagnostics: ReturnType, + path: string +): void { + if (event.nodeid === undefined || !expected.has(event.nodeid)) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution collected unexpected node id: ${event.nodeid ?? ""}`, path)); + return; + } + if (collected.has(event.nodeid)) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution collected duplicate node id: ${event.nodeid}`, path)); + return; + } + collected.add(event.nodeid); +} + +function processCallOutcome( + event: HookEvent & { nodeid: string; when: "call"; outcome: "passed" | "failed" | "skipped" }, + current: NodeOutcomeState, + diagnostics: ReturnType, + path: string, + terminal: Map +): void { + if (current.callSeen) { + diagnostics.push(pytestDiagnostic("PYTHON_PYTEST_PROTOCOL_MISMATCH", `Pytest execution reported duplicate call outcome for ${event.nodeid}.`, path)); + return; + } + current.callSeen = true; + if (event.outcome === "passed" && event.wasxfail === undefined) current.passed = true; + else if (event.outcome === "failed" && event.wasxfail !== undefined) current.xpassed = true; + else if (event.outcome === "failed") current.failed = true; + else if (event.outcome === "skipped" && event.wasxfail !== undefined) current.xfailed = true; + else if (event.outcome === "skipped") current.skipped = true; + terminal.set(event.nodeid, current); +} + +function summarizeExecutionCounts( + candidateCount: number, + collectedCount: number, + terminal: ReadonlyMap, + events: readonly HookEvent[] +): PythonCapabilityCounts { + let passedCount = 0; + let failedCount = 0; + let skippedCount = 0; + let xfailedCount = 0; + let xpassedCount = 0; + let errorCount = collectErrorCount(events); + for (const counts of terminal.values()) { + if (counts.passed) passedCount += 1; + if (counts.failed) failedCount += 1; + if (counts.skipped || counts.setupSkipped) skippedCount += 1; + if (counts.xfailed) xfailedCount += 1; + if (counts.xpassed) xpassedCount += 1; + if (counts.error) errorCount += 1; + } + return countResults(candidateCount, collectedCount, terminal.size, passedCount, failedCount, skippedCount, xfailedCount, xpassedCount, errorCount); +} + +function emptyNodeOutcomeState(): NodeOutcomeState { + return { callSeen: false, passed: false, failed: false, skipped: false, xfailed: false, xpassed: false, error: false, setupSkipped: false }; +} + +function validateHookEvent(payload: Record, lineNumber: number): HookEvent { + if (payload.type === "collected") { + if (typeof payload.nodeid !== "string" || payload.nodeid.length === 0) throw new Error(`Pytest collected event ${lineNumber} requires non-empty nodeid.`); + return { type: "collected", nodeid: payload.nodeid }; + } + if (payload.type === "collect_error") { + if (typeof payload.message !== "string" || payload.message.length === 0) throw new Error(`Pytest collect_error event ${lineNumber} requires message.`); + if (payload.nodeid !== undefined && typeof payload.nodeid !== "string") throw new Error(`Pytest collect_error event ${lineNumber} has invalid nodeid.`); + return { type: "collect_error", message: payload.message, ...(typeof payload.nodeid === "string" ? { nodeid: payload.nodeid } : {}) }; + } + if (payload.type === "test_report") { + if (typeof payload.nodeid !== "string" || payload.nodeid.length === 0) throw new Error(`Pytest test_report event ${lineNumber} requires non-empty nodeid.`); + if (payload.when !== "setup" && payload.when !== "call" && payload.when !== "teardown") throw new Error(`Pytest test_report event ${lineNumber} has invalid when value.`); + if (payload.outcome !== "passed" && payload.outcome !== "failed" && payload.outcome !== "skipped") throw new Error(`Pytest test_report event ${lineNumber} has invalid outcome.`); + if (payload.wasxfail !== undefined && typeof payload.wasxfail !== "string") throw new Error(`Pytest test_report event ${lineNumber} has invalid wasxfail value.`); + return { type: "test_report", nodeid: payload.nodeid, when: payload.when, outcome: payload.outcome, ...(typeof payload.wasxfail === "string" ? { wasxfail: payload.wasxfail } : {}) }; + } + throw new Error(`Unknown pytest hook event type at line ${lineNumber}: ${String(payload.type)}`); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} diff --git a/packages/validation-python/src/pytest-result.ts b/packages/validation-python/src/pytest-result.ts new file mode 100644 index 0000000..cbcb019 --- /dev/null +++ b/packages/validation-python/src/pytest-result.ts @@ -0,0 +1,170 @@ +import type { PythonProjectContext, PythonValidationCapabilityRun, ValidationDiagnostic } from "@the-open-engine/opcore-contracts"; +import type { ValidationCheckResult } from "@the-open-engine/opcore-validation"; +import { PYTHON_PYTEST_CHECK_ID } from "./check-ids.js"; +import { diagnostic } from "./diagnostics.js"; +import type { CleanupState, ProjectRunResult, PytestRunOutcome } from "./pytest-types.js"; + +export function pytestDiagnostic(code: string, message: string, path: string | undefined): ValidationDiagnostic { + return diagnostic({ + category: "test", + severity: "warning", + code, + message, + ...(path === undefined ? {} : { path }) + }); +} + +export function notApplicableRun(message: string): ValidationCheckResult { + return { + diagnostics: [], + outcome: "passed", + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "not_applicable", + outcome: "not_applicable", + message + }] + }; +} + +export function disabledPytestRun(message = "python.pytest is disabled by repo policy."): ValidationCheckResult { + return { + diagnostics: [], + outcome: "passed", + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "disabled", + outcome: "disabled", + message, + selectionMode: "none" + }] + }; +} + +export function unsupportedRun(context: PythonProjectContext, message: string): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_CONTEXT_UNSUPPORTED", message, context.target)], + outcome: "unsupported_target", + pythonProjectContexts: [context], + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome: "unsupported_target", + message, + projectKey: context.projectKey, + projectRoot: context.projectRoot + }] + }; +} + +export function unsupportedPytestTool(context: PythonProjectContext, candidatePaths: readonly string[]): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_UNSUPPORTED", `pytest is unavailable for project ${context.projectRoot}.`, context.target)], + outcome: "tool_unavailable", + pythonProjectContexts: [context], + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome: "tool_unavailable", + message: `pytest is unavailable for project ${context.projectRoot}.`, + projectKey: context.projectKey, + projectRoot: context.projectRoot, + candidatePaths + }] + }; +} + +export function candidateFailure( + contexts: readonly PythonProjectContext[], + candidatePaths: readonly string[], + message: string +): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic("PYTHON_PYTEST_NO_CANDIDATES", message, contexts[0]?.target)], + outcome: "findings", + pythonProjectContexts: contexts, + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome: "no_candidates", + message, + candidatePaths + }] + }; +} + +export function failureRun( + code: string, + message: string, + outcome: Extract, + contexts: readonly PythonProjectContext[] = [], + candidatePaths: readonly string[] = [] +): ValidationCheckResult { + return { + diagnostics: [pytestDiagnostic(code, message, contexts[0]?.target)], + outcome, + pythonProjectContexts: contexts, + pythonCapabilityRuns: [{ + capability: "pytest", + checkId: PYTHON_PYTEST_CHECK_ID, + activation: "enabled", + outcome, + message, + candidatePaths + }] + }; +} + +export function createCleanupState(): CleanupState { + return { attempted: false, ok: true, failureMessages: [] }; +} + +export function recordCleanup(cleanup: CleanupState, action: () => void): void { + cleanup.attempted = true; + try { + action(); + } catch (error) { + cleanup.ok = false; + cleanup.failureMessages.push(errorMessage(error)); + } +} + +export function cleanupFailureMessage(cleanup: CleanupState): string { + return cleanup.failureMessages.length === 0 + ? "Pytest temporary workspace cleanup failed." + : `Pytest temporary workspace cleanup failed: ${cleanup.failureMessages.join("; ")}`; +} + +export function finalizeCapabilityRun(cleanup: CleanupState, result: ProjectRunResult): ProjectRunResult { + const cleanupEvidence: NonNullable = { + attempted: cleanup.attempted, + ok: cleanup.ok, + ...(cleanup.ok || cleanup.failureMessages.length === 0 ? {} : { failureMessage: cleanup.failureMessages.join("; ") }) + }; + if (cleanup.ok) { + return { ...result, capabilityRun: { ...result.capabilityRun, cleanup: cleanupEvidence } }; + } + const cleanupMessage = cleanupFailureMessage(cleanup); + return { + outcome: "tool_failure", + diagnostics: [ + ...result.diagnostics, + pytestDiagnostic("PYTHON_PYTEST_CLEANUP_FAILED", cleanupMessage, result.diagnostics[0]?.path ?? result.capabilityRun.projectRoot) + ], + capabilityRun: { + ...result.capabilityRun, + outcome: "tool_failure", + message: result.outcome === "passed" ? cleanupMessage : `${result.capabilityRun.message} Cleanup also failed.`, + cleanup: cleanupEvidence + } + }; +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/validation-python/src/pytest-types.ts b/packages/validation-python/src/pytest-types.ts new file mode 100644 index 0000000..c2d1fdd --- /dev/null +++ b/packages/validation-python/src/pytest-types.ts @@ -0,0 +1,65 @@ +import type { + PythonCapabilityCounts, + PythonCapabilityInvocation, + PythonProjectContext, + PythonValidationCapabilityRun, + ValidationDiagnostic +} from "@the-open-engine/opcore-contracts"; + +export type PytestRunOutcome = "passed" | "findings" | "tool_failure" | "timeout" | "invalid_config"; + +export interface ProjectRunInput { + context: PythonProjectContext; + candidatePaths: readonly string[]; +} + +export interface CleanupState { + attempted: boolean; + ok: boolean; + failureMessages: string[]; +} + +export interface HookEvent { + type: "collected" | "collect_error" | "test_report"; + nodeid?: string; + when?: "setup" | "call" | "teardown"; + outcome?: "passed" | "failed" | "skipped"; + wasxfail?: string; + message?: string; +} + +export interface HookReport { + events: readonly HookEvent[]; + exitStatus: number; +} + +export interface ExecutionEventAnalysis { + counts: PythonCapabilityCounts; + diagnostics: ValidationDiagnostic[]; +} + +export interface ProjectRunResult { + outcome: PytestRunOutcome; + diagnostics: ValidationDiagnostic[]; + capabilityRun: PythonValidationCapabilityRun; +} + +export interface CollectionRunResult { + outcome: PytestRunOutcome; + message: string; + nodeIds: readonly string[]; + counts: PythonCapabilityCounts; + invocation: PythonCapabilityInvocation; + selectionMode: "direct_argv"; + selectionDigest: string; +} + +export interface ExecutionRunResult { + outcome: Exclude; + message: string; + diagnostics: ValidationDiagnostic[]; + counts: PythonCapabilityCounts; + invocation: PythonCapabilityInvocation; + selectionMode: "direct_argv" | "manifest"; + selectionDigest: string; +} diff --git a/packages/validation-python/src/pytest-workspace.ts b/packages/validation-python/src/pytest-workspace.ts new file mode 100644 index 0000000..a511a76 --- /dev/null +++ b/packages/validation-python/src/pytest-workspace.ts @@ -0,0 +1,125 @@ +import { mkdtempSync } from "node:fs"; +import { mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ValidationCheckContext } from "@the-open-engine/opcore-validation"; +import type { PythonProjectContext } from "@the-open-engine/opcore-contracts"; +import { + removeMaterializedWorkspace, + resolveMaterializedWorkspacePath, + writeMaterializedWorkspaceFile +} from "./materialized-workspace.js"; +import { pythonProjectDigest } from "./project-fingerprint.js"; +import { createValidationFileViewPythonWorkspace, type PythonProjectWorkspace } from "./project-workspace.js"; +import { isRelevantPythonConfig } from "./project-config-files.js"; + +export const pytestWorkspaceCaps = { + maxCandidateFiles: 128, + maxCollectedNodeIds: 2048, + maxNodeIdBytes: 256 * 1024, + maxManifestBytes: 256 * 1024, + maxHookReportBytes: 512 * 1024, + maxProcessOutputBytes: 512 * 1024, + maxWorkspaceBytes: 4 * 1024 * 1024, + maxArgvBytes: 16 * 1024 +} as const; + +export interface MaterializedPytestWorkspace { + root: string; + repoRoot: string; + projectCwd: string; + runtimeRoot: string; + afterStateFingerprint: string; + totalBytes: number; + cleanup(): void; +} + +export interface MaterializePytestWorkspaceArgs { + context: ValidationCheckContext; + project: PythonProjectContext; + paths: readonly string[]; + nodeWorkspace?: PythonProjectWorkspace; +} + +export async function materializePytestWorkspace( + args: MaterializePytestWorkspaceArgs +): Promise { + const workspace = createValidationFileViewPythonWorkspace(args.context.fileView, undefined, args.nodeWorkspace); + const tempRoot = mkdtempSync(join(tmpdir(), pytestWorkspacePrefix())); + const repoRoot = join(tempRoot, "repo"); + const runtimeRoot = join(tempRoot, "runtime"); + let totalBytes = 0; + try { + await mkdir(repoRoot, { recursive: true }); + await mkdir(runtimeRoot, { recursive: true }); + const normalizedPaths = [...new Set(args.paths)].sort(); + const fingerprintEntries: { path: string; content: string }[] = []; + for (const path of normalizedPaths) { + const real = await workspace.realpath(path); + if (real.unavailable) throw new Error(`Pytest workspace realpath evidence is unavailable: ${path}`); + if (real.symlink || real.path !== path) throw new Error(`Pytest workspace refuses symlinked path: ${path}`); + const content = await workspace.read(path); + if (content === undefined) throw new Error(`Pytest workspace materialization requires visible file: ${path}`); + totalBytes += Buffer.byteLength(content, "utf8"); + if (totalBytes > pytestWorkspaceCaps.maxWorkspaceBytes) { + throw new Error(`Pytest workspace exceeded ${pytestWorkspaceCaps.maxWorkspaceBytes} bytes`); + } + fingerprintEntries.push({ path, content }); + await writeMaterializedWorkspaceFile(repoRoot, path, content, "pytest workspace", await workspace.statMode?.(path) ?? 0o644); + } + const projectCwd = args.project.projectRoot === "." ? repoRoot : writeMaterializedProjectPath(repoRoot, args.project.projectRoot); + return { + root: tempRoot, + repoRoot, + projectCwd, + runtimeRoot, + totalBytes, + afterStateFingerprint: pythonProjectDigest(fingerprintEntries), + cleanup: () => removeMaterializedWorkspace(tempRoot, "Pytest temporary workspace") + }; + } catch (error) { + removeMaterializedWorkspace(tempRoot, "Pytest temporary workspace"); + throw error; + } +} + +export async function collectPytestProjectPaths( + context: ValidationCheckContext, + project: PythonProjectContext, + candidatePaths: readonly string[], + nodeWorkspace?: PythonProjectWorkspace +): Promise { + const workspace = createValidationFileViewPythonWorkspace(context.fileView, undefined, nodeWorkspace); + const visible = workspace.listAll === undefined ? await workspace.list() : await workspace.listAll(); + const relevant = new Set(); + for (const path of visible) { + if (!withinProject(path, project.projectRoot)) continue; + relevant.add(path); + } + for (const candidate of candidatePaths) { + if (withinProject(candidate, project.projectRoot)) relevant.add(candidate); + } + for (const evidence of project.evidence) { + if (withinProject(evidence.path, project.projectRoot) && (isRelevantPythonConfig(evidence.path) || evidence.role === "config")) { + relevant.add(evidence.path); + } + } + return [...relevant].sort(); +} + +function withinProject(path: string, projectRoot: string): boolean { + return projectRoot === "." || path === projectRoot || path.startsWith(`${projectRoot}/`); +} + +function writeMaterializedProjectPath(root: string, path: string): string { + return path === "." ? root : resolveMaterializedWorkspacePath(root, path, "pytest workspace"); +} + +function pytestWorkspacePrefix(): string { + const configured = process.env.OPCORE_INTERNAL_PYTEST_WORKSPACE_PREFIX; + if (configured === undefined) return "opcore-python-pytest-workspace-"; + if (!/^[A-Za-z0-9._-]+$/.test(configured) || !configured.endsWith("-")) { + throw new Error("OPCORE_INTERNAL_PYTEST_WORKSPACE_PREFIX must be a simple basename ending with '-'"); + } + return configured; +} diff --git a/packages/validation-python/src/relevant-tests-check.ts b/packages/validation-python/src/relevant-tests-check.ts index 2a648a5..ddb5ab0 100644 --- a/packages/validation-python/src/relevant-tests-check.ts +++ b/packages/validation-python/src/relevant-tests-check.ts @@ -23,16 +23,16 @@ export function createRelevantTestsCheck(resolveRoots: PythonSourceRootResolver) category: "test", severity: "info", path, - code: "PY_RELEVANT_TESTS_FOUND", - message: `TESTED_BY graph evidence exists for ${path}: ${evidence.map(testEndpoint).sort().join(", ")}` + code: "PY_RELEVANT_TEST_CANDIDATES_FOUND", + message: `TESTED_BY graph candidate evidence for ${path}: ${evidence.map(testEndpoint).sort().join(", ")}` }; } return { category: "test", severity: "info", path, - code: "PY_RELEVANT_TESTS_ABSENT", - message: `No TESTED_BY graph evidence found for ${path}.` + code: "PY_RELEVANT_TEST_CANDIDATES_ABSENT", + message: `No TESTED_BY graph candidate evidence found for ${path}.` }; }); return { diagnostics }; diff --git a/packages/validation-python/src/source-closure.ts b/packages/validation-python/src/source-closure.ts new file mode 100644 index 0000000..24ceb9e --- /dev/null +++ b/packages/validation-python/src/source-closure.ts @@ -0,0 +1,96 @@ +import type { PythonProjectContext } from "@the-open-engine/opcore-contracts"; +import type { ValidationCheckContext } from "@the-open-engine/opcore-validation"; +import type { PythonImportEdge } from "./import-analysis.js"; + +interface PythonPathContent { + path: string; +} + +interface ExpandPythonSourceClosureArgs { + context: ValidationCheckContext; + rootPaths: readonly string[]; + edges: readonly PythonImportEdge[]; + sourceByPath: ReadonlyMap; + resolveContexts: ( + context: ValidationCheckContext, + targets?: readonly string[] + ) => Promise; +} + +export async function expandPythonSourceClosure( + args: ExpandPythonSourceClosureArgs +): Promise { + const projectContexts = new Map(); + let selected = transitiveSourcePaths(args.rootPaths, args.edges); + while (true) { + const unresolvedTargets = selected.filter((path) => !projectContexts.has(path)); + if (unresolvedTargets.length > 0) { + for (const projectContext of await args.resolveContexts(args.context, unresolvedTargets)) { + projectContexts.set(projectContext.target, projectContext); + } + } + const expanded = transitiveSourcePaths( + includePackageInitializers(selected, args.sourceByPath, [...projectContexts.values()]), + args.edges + ); + if (expanded.length === selected.length && expanded.every((path, index) => path === selected[index])) return expanded; + selected = expanded; + } +} + +function includePackageInitializers( + selectedPaths: readonly string[], + sourceByPath: ReadonlyMap, + projectContexts: readonly PythonProjectContext[] +): readonly string[] { + const expanded = new Set(selectedPaths); + for (const path of selectedPaths) { + const sourceRoot = owningSourceRoot(path, projectContexts); + if (sourceRoot === undefined) continue; + let directory = path.slice(0, path.lastIndexOf("/")); + while (directory.length > 0 && directory !== sourceRoot && pathWithinRoot(directory, sourceRoot)) { + for (const initializer of [`${directory}/__init__.py`, `${directory}/__init__.pyi`]) { + if (sourceByPath.has(initializer)) expanded.add(initializer); + } + const separator = directory.lastIndexOf("/"); + if (separator < 0) break; + directory = directory.slice(0, separator); + } + } + return [...expanded].sort(); +} + +function owningSourceRoot(path: string, projectContexts: readonly PythonProjectContext[]): string | undefined { + return projectContexts + .flatMap((projectContext) => projectContext.sourceRoots) + .filter((sourceRoot) => pathWithinRoot(path, sourceRoot)) + .sort((left, right) => right.length - left.length || left.localeCompare(right))[0]; +} + +function pathWithinRoot(path: string, root: string): boolean { + return root === "." || path === root || path.startsWith(`${root}/`); +} + +function transitiveSourcePaths( + rootPaths: readonly string[], + edges: readonly PythonImportEdge[] +): readonly string[] { + const outgoing = new Map(); + for (const edge of edges) { + const targets = outgoing.get(edge.fromPath) ?? []; + targets.push(edge.toPath); + outgoing.set(edge.fromPath, targets); + } + const selected = new Set(rootPaths); + const pending = [...rootPaths]; + while (pending.length > 0) { + const path = pending.shift(); + if (path === undefined) continue; + for (const target of outgoing.get(path) ?? []) { + if (selected.has(target)) continue; + selected.add(target); + pending.push(target); + } + } + return [...selected].sort(); +} diff --git a/packages/validation-python/src/source-files.ts b/packages/validation-python/src/source-files.ts index 344be36..32e79d2 100644 --- a/packages/validation-python/src/source-files.ts +++ b/packages/validation-python/src/source-files.ts @@ -8,8 +8,10 @@ import { type PythonImportEdge, type PythonImportSourceFile } from "./import-analysis.js"; +import { isRelevantPythonConfig } from "./project-config-files.js"; import { resolvePythonProjectContexts, type ResolvePythonProjectContextsOptions } from "./project-context.js"; import { createValidationFileViewPythonWorkspace, type PythonProjectWorkspace } from "./project-workspace.js"; +import { expandPythonSourceClosure } from "./source-closure.js"; export const pythonSourceExtensions = [".py", ".pyi"] as const; @@ -44,7 +46,7 @@ export function createPythonProjectContextResolver( } const targets = requestedTargets === undefined ? await (cached.inputTargets ??= readPythonAfterSources(context).then((sources) => sources.map((source) => source.path))) - : uniqueSorted(requestedTargets.map(normalizeValidationFileViewPath).filter(isPythonSourcePath)); + : uniqueSorted(requestedTargets.map(normalizeValidationFileViewPath).filter(isRelevantPythonTargetPath)); while (true) { const missing = targets.filter((target) => !cached.contexts.has(target)); if (missing.length === 0) return targets.map((target) => requiredProjectContext(cached.contexts, target)); @@ -98,6 +100,10 @@ export function isPythonSourcePath(path: string): boolean { return pythonSourceExtensions.some((extension) => path.endsWith(extension)); } +export function isRelevantPythonTargetPath(path: string): boolean { + return isPythonSourcePath(path) || isRelevantPythonConfig(path); +} + export function toFileNodeId(path: string): string { return `file:${path}`; } @@ -110,6 +116,14 @@ export function pythonInputSet(context: ValidationCheckContext): readonly string ); } +export function pythonProjectInputSet(context: ValidationCheckContext): readonly string[] { + return uniqueSorted( + [...context.fileView.scopeFiles, ...context.fileView.overlays.map((overlay) => overlay.path)] + .map((path) => normalizeValidationFileViewPath(path)) + .filter(isRelevantPythonTargetPath) + ); +} + export async function readPythonAfterSources(context: ValidationCheckContext): Promise { const files: PythonMaterializedSourceFile[] = []; for (const path of pythonInputSet(context)) { @@ -178,7 +192,13 @@ async function materializePythonSourcesUncached( await analyzer.analyze(allSources), new Set(allSourceByPath.keys()) ); - const selectedPaths = await expandSourceClosure(context, rootPaths, repoImports, allSourceByPath, resolveContexts); + const selectedPaths = await expandPythonSourceClosure({ + context, + rootPaths, + edges: repoImports, + sourceByPath: allSourceByPath, + resolveContexts + }); const files = selectedPaths.map((path) => allSourceByPath.get(path)).filter(isDefined); const sourceFileByPath = new Map(files.map((file) => [file.path, file])); const selectedPathSet = new Set(selectedPaths); @@ -191,95 +211,6 @@ async function materializePythonSourcesUncached( }; } -async function expandSourceClosure( - context: ValidationCheckContext, - rootPaths: readonly string[], - edges: readonly PythonImportEdge[], - sourceByPath: ReadonlyMap, - resolveContexts: PythonProjectContextResolver -): Promise { - const projectContexts = new Map(); - let selected = transitiveSourcePaths(rootPaths, edges); - while (true) { - const unresolvedTargets = selected.filter((path) => !projectContexts.has(path)); - if (unresolvedTargets.length > 0) { - for (const projectContext of await resolveContexts(context, unresolvedTargets)) { - projectContexts.set(projectContext.target, projectContext); - } - } - const expanded = transitiveSourcePaths( - includePackageInitializers(selected, sourceByPath, [...projectContexts.values()]), - edges - ); - if (expanded.length === selected.length && expanded.every((path, index) => path === selected[index])) return expanded; - selected = expanded; - } -} - -function includePackageInitializers( - selectedPaths: readonly string[], - sourceByPath: ReadonlyMap, - projectContexts: readonly PythonProjectContext[] -): readonly string[] { - const expanded = new Set(selectedPaths); - for (const path of selectedPaths) { - const sourceRoot = owningSourceRoot(path, projectContexts); - if (sourceRoot === undefined) continue; - let directory = path.slice(0, path.lastIndexOf("/")); - while (directory.length > 0 && directory !== sourceRoot && pathWithinRoot(directory, sourceRoot)) { - const initializers = [`${directory}/__init__.py`, `${directory}/__init__.pyi`] - .filter((candidate) => sourceByPath.has(candidate)); - if (initializers.length === 0) { - const separator = directory.lastIndexOf("/"); - if (separator < 0) break; - directory = directory.slice(0, separator); - continue; - } - // Package markers are structural type-checker inputs, not import expectations. - for (const initializer of initializers) expanded.add(initializer); - const separator = directory.lastIndexOf("/"); - if (separator < 0) break; - directory = directory.slice(0, separator); - } - } - return [...expanded].sort(); -} -function owningSourceRoot(path: string, projectContexts: readonly PythonProjectContext[]): string | undefined { - return projectContexts - .flatMap((projectContext) => projectContext.sourceRoots) - .filter((sourceRoot) => pathWithinRoot(path, sourceRoot)) - .sort((left, right) => right.length - left.length || left.localeCompare(right))[0]; -} - -function pathWithinRoot(path: string, root: string): boolean { - return root === "." || path === root || path.startsWith(`${root}/`); -} - - -function transitiveSourcePaths( - rootPaths: readonly string[], - edges: readonly PythonImportEdge[] -): readonly string[] { - const outgoing = new Map(); - for (const edge of edges) { - const targets = outgoing.get(edge.fromPath) ?? []; - targets.push(edge.toPath); - outgoing.set(edge.fromPath, targets); - } - const selected = new Set(rootPaths); - const pending = [...rootPaths]; - while (pending.length > 0) { - const path = pending.shift(); - if (path === undefined) continue; - for (const target of outgoing.get(path) ?? []) { - if (selected.has(target)) continue; - selected.add(target); - pending.push(target); - } - } - return [...selected].sort(); -} - function emptySourceSet(): PythonMaterializedSourceSet { return { rootPaths: [], diff --git a/packages/validation-python/src/type-check.ts b/packages/validation-python/src/type-check.ts index 0a5f5c9..3d0deb8 100644 --- a/packages/validation-python/src/type-check.ts +++ b/packages/validation-python/src/type-check.ts @@ -9,13 +9,18 @@ import type { ValidationCheckDefinition, ValidationCheckResult } from "@the-open-engine/opcore-validation"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync } from "node:fs"; import { mkdir } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, relative, resolve, sep } from "node:path"; +import { join, relative, resolve } from "node:path"; import { PYTHON_TYPES_CHECK_ID } from "./check-ids.js"; import { pythonCheckAdapter, pythonCheckOwner, supportedPythonValidationScopes } from "./check-constants.js"; import { diagnostic, sortDiagnostics } from "./diagnostics.js"; +import { + removeMaterializedWorkspace, + resolveMaterializedWorkspacePath, + writeMaterializedWorkspaceFile +} from "./materialized-workspace.js"; import { runTool } from "./process.js"; import { pythonInputSet, @@ -157,36 +162,25 @@ async function materializePythonTypeWorkspace( const root = join(tempRoot, "repo"); try { await mkdir(root, { recursive: true }); - for (const source of files) await writeMaterializedFile(root, source.path, source.content); + for (const source of files) await writeMaterializedWorkspaceFile(root, source.path, source.content, "materialized Python workspace"); for (const evidence of project.context.evidence) { if (evidence.role === "layout" || evidence.role === "boundary" && !isConfigPath(evidence.path)) continue; const result = await validation.fileView.readAfter(evidence.path); - if (result.status === "found") await writeMaterializedFile(root, evidence.path, result.content); + if (result.status === "found") await writeMaterializedWorkspaceFile(root, evidence.path, result.content, "materialized Python workspace"); } - const projectCwd = project.context.projectRoot === "." ? root : resolveRepoPath(root, project.context.projectRoot); + const projectCwd = project.context.projectRoot === "." ? root : resolveMaterializedWorkspacePath(root, project.context.projectRoot, "materialized Python workspace"); await mkdir(projectCwd, { recursive: true }); - return { root, projectCwd, cleanup: () => rmSync(tempRoot, { recursive: true, force: true }) }; + return { + root, + projectCwd, + cleanup: () => removeMaterializedWorkspace(tempRoot, "Materialized Python workspace") + }; } catch (error) { - rmSync(tempRoot, { recursive: true, force: true }); + removeMaterializedWorkspace(tempRoot, "Materialized Python workspace"); throw error; } } -async function writeMaterializedFile(root: string, path: string, content: string): Promise { - const absolutePath = resolveRepoPath(root, path); - await mkdir(dirname(absolutePath), { recursive: true }); - writeFileSync(absolutePath, content); -} - -function resolveRepoPath(root: string, path: string): string { - const absolutePath = resolve(root, path); - const relativePath = relative(root, absolutePath); - if (relativePath === "" || relativePath.startsWith("..") || relativePath.split(sep).includes("..")) { - throw new Error(`Repo-relative path escapes materialized Python workspace: ${path}`); - } - return absolutePath; -} - function unresolvedProjectResult(context: PythonProjectContext): ValidationCheckResult { const reason = context.reasons.find((entry) => entry.code === "invalid_config") ?? context.reasons.find((entry) => entry.tool === "python") ?? context.reasons[0]; diff --git a/packages/validation/src/aggregation.ts b/packages/validation/src/aggregation.ts index 0276773..6dfc38e 100644 --- a/packages/validation/src/aggregation.ts +++ b/packages/validation/src/aggregation.ts @@ -10,6 +10,7 @@ import type { ValidationResultManifest, ValidationResultStatus, ValidationSkippedCheck, + PythonValidationCapabilityRun, PythonProjectContext } from "@the-open-engine/opcore-contracts"; import { GRAPH_SCHEMA_VERSION, validateValidationResultPayload } from "@the-open-engine/opcore-contracts"; @@ -30,6 +31,7 @@ export interface AggregateValidationResultsArgs extends CreateValidationManifest failure?: ValidationFailure; refusal?: EditRefusal; pythonProjectContexts?: readonly PythonProjectContext[]; + pythonCapabilityRuns?: readonly PythonValidationCapabilityRun[]; } const failureStatusPriority: readonly ValidationCheckRunStatus[] = [ @@ -55,20 +57,71 @@ export function createValidationManifest(args: CreateValidationManifestArgs): Va export function aggregateValidationResults(args: AggregateValidationResultsArgs): ValidationResult { const diagnostics = sortDiagnostics(args.diagnostics ?? []); const status = args.status ?? deriveStatus(diagnostics, args.runs ?? [], args.skippedChecks ?? []); - const result: ValidationResult = { + const result = withPythonValidationEvidence({ ok: status === "passed", status, diagnostics, manifest: createValidationManifest(args) - }; + }, args); if (args.graphStatus !== undefined) result.graphStatus = args.graphStatus; if (args.refusal !== undefined) result.refusal = args.refusal; - if (args.pythonProjectContexts !== undefined) result.pythonProjectContexts = deduplicatePythonProjectContexts(args.pythonProjectContexts); const failure = args.failure ?? failureForStatus(status); if (failure !== undefined) result.failure = failure; return validateValidationResultPayload(result); } +function withPythonValidationEvidence( + result: ValidationResult, + args: AggregateValidationResultsArgs +): ValidationResult { + attachProjectContexts(result, args.pythonProjectContexts); + attachCapabilityRuns(result, args.pythonCapabilityRuns); + return result; +} + +function attachProjectContexts( + result: ValidationResult, + projectContexts: readonly PythonProjectContext[] | undefined +): void { + if (projectContexts === undefined) return; + const deduplicated = deduplicatePythonProjectContexts(projectContexts); + if (deduplicated.length > 0) result.pythonProjectContexts = deduplicated; +} + +function attachCapabilityRuns( + result: ValidationResult, + capabilityRuns: readonly PythonValidationCapabilityRun[] | undefined +): void { + if (capabilityRuns === undefined) return; + const deduplicated = deduplicatePythonCapabilityRuns(capabilityRuns); + if (deduplicated.length > 0) result.pythonCapabilityRuns = deduplicated; +} + +function deduplicatePythonCapabilityRuns( + runs: readonly PythonValidationCapabilityRun[] +): readonly PythonValidationCapabilityRun[] { + const exact = new Map(); + for (const run of runs) exact.set(JSON.stringify(run), run); + return [...exact.values()].sort(comparePythonCapabilityRuns); +} + +function comparePythonCapabilityRuns( + left: PythonValidationCapabilityRun, + right: PythonValidationCapabilityRun +): number { + return pythonCapabilitySortKey(left).localeCompare(pythonCapabilitySortKey(right)); +} + +function pythonCapabilitySortKey(run: PythonValidationCapabilityRun): string { + return JSON.stringify({ + checkId: run.checkId, + capability: run.capability, + projectKey: run.projectKey ?? "", + contextFingerprint: "contextFingerprint" in run && typeof run.contextFingerprint === "string" ? run.contextFingerprint : "", + receipt: run + }); +} + function deduplicatePythonProjectContexts(contexts: readonly PythonProjectContext[]): readonly PythonProjectContext[] { const byTarget = new Map(); for (const context of contexts) byTarget.set(context.target, context); diff --git a/packages/validation/src/command-adapter.ts b/packages/validation/src/command-adapter.ts index 99f4bff..43c7d05 100644 --- a/packages/validation/src/command-adapter.ts +++ b/packages/validation/src/command-adapter.ts @@ -55,17 +55,19 @@ export function createCheckCommandAdapter(options: ValidationCommandAdapterOptio return async (request) => { try { const parsed = parseCheckCommandOptions(request.args); + const repoRoot = repoRootForCommand(parsed, options); + const availableChecks = registryChecks(options, repoRoot); if (parsed.route === "manifest") { - return manifestCommandResult(request, registryChecks(options, repoRootForCommand(parsed, options)), "check manifest"); + return manifestCommandResult(request, availableChecks, "check manifest"); } const validationRequest = normalizeValidationRequest( { repo: { - repoRoot: repoRootForCommand(parsed, options) + repoRoot }, scope: requireScope(parsed), graph: { - mode: parsed.graphMode, + mode: forcedGraphMode(parsed.checks, parsed.graphMode, availableChecks), provider: defaultValidationGraphProvider }, overlays: [], @@ -91,14 +93,15 @@ export function createValidateCommandAdapter(options: ValidationCommandAdapterOp return async (request) => { try { const parsed = parseValidateCommandOptions(request.args); + const repoRoot = repoRootForCommand(parsed, options); if (parsed.route === "manifest") { - return manifestCommandResult(request, registryChecks(options, repoRootForCommand(parsed, options)), "validate manifest"); + return manifestCommandResult(request, registryChecks(options, repoRoot), "validate manifest"); } if (parsed.route === "pre-write") return preWriteValidationCommandResult(request, parsed, options); const payload = await readRequestPayload(parsed.requestFile); const validatedPayload = validateValidationRequestPayload(payload); const validationRequest = normalizeValidationRequest( - applyValidateCommandOverrides(validatedPayload, parsed), + applyValidateCommandOverrides(validatedPayload, parsed, options), { provider: defaultValidationGraphProvider } @@ -122,7 +125,7 @@ async function preWriteValidationCommandResult( try { const payload = await readRequestPayload(parsed.requestFile); const validatedPayload = validateValidationRequestPayload(payload); - validationRequest = normalizeValidationRequest(applyValidateCommandOverrides(validatedPayload, parsed), { + validationRequest = normalizeValidationRequest(applyValidateCommandOverrides(validatedPayload, parsed, options), { provider: defaultValidationGraphProvider }); result = await runRequestWithTimeout(validationRequest, parsed, options, timeoutMs); @@ -392,7 +395,8 @@ function registryChecks(options: ValidationCommandAdapterOptions, repoRoot: stri function applyValidateCommandOverrides( request: ValidationRequest, - parsed: ParsedValidationCommandOptions + parsed: ParsedValidationCommandOptions, + options: ValidationCommandAdapterOptions ): ValidationRequest { let next = request; if (parsed.repoRoot !== undefined) { @@ -416,9 +420,15 @@ function applyValidateCommandOverrides( }; } if (parsed.checks !== undefined) { + const repoRoot = repoRootForRequest(next, parsed, options); + const availableChecks = registryChecks(options, repoRoot); next = { ...next, - checks: parsed.checks + checks: parsed.checks, + graph: { + ...next.graph, + mode: forcedGraphMode(parsed.checks, next.graph.mode, availableChecks) + } }; } if (parsed.reportMode !== undefined) { @@ -430,6 +440,16 @@ function applyValidateCommandOverrides( return next; } +function forcedGraphMode( + checks: readonly string[] | undefined, + current: ValidationRequest["graph"]["mode"], + availableChecks: readonly ValidationCheckDefinition[] +): ValidationRequest["graph"]["mode"] { + if (checks === undefined) return current; + const byId = new Map(availableChecks.map((check) => [check.id, check])); + return checks.some((checkId) => byId.get(checkId)?.requiresGraph === true) ? "required" : current; +} + function repoIdentityForOverride(repo: ValidationRequest["repo"], repoRoot: string): ValidationRequest["repo"] { const { repoId: _repoId, ...rest } = repo; return { diff --git a/packages/validation/src/registry.ts b/packages/validation/src/registry.ts index ac28368..be753b7 100644 --- a/packages/validation/src/registry.ts +++ b/packages/validation/src/registry.ts @@ -3,6 +3,7 @@ import type { ValidationCheckOutcome, ValidationCheckRunStatus, ValidationDiagnostic, + PythonValidationCapabilityRun, PythonProjectContext, GraphProviderStatus, ValidationRequest, @@ -18,14 +19,17 @@ const diagnosticSeverities = ["info", "warning", "error"] as const; export interface ValidationCheckContext { request: ValidationRequest; + selectedCheckIds?: readonly string[]; scope: ResolvedValidationScope; graphStatus: GraphProviderStatus; graph: ValidationGraphQuerySession; fileView: ValidationFileView; + resources?: unknown; runtime: ValidationRuntimePolicy; } export type ValidationPersistentCacheMode = "enabled" | "disabled"; +export type ValidationInactiveCheckState = "not_applicable" | "disabled"; export interface ValidationRuntimePolicy { persistentCaches: ValidationPersistentCacheMode; @@ -37,6 +41,7 @@ export interface ValidationCheckResult { outcome?: ValidationCheckOutcome; failureMessage?: string; pythonProjectContexts?: readonly PythonProjectContext[]; + pythonCapabilityRuns?: readonly PythonValidationCapabilityRun[]; } export interface ValidationCheckDefinition { @@ -47,6 +52,12 @@ export interface ValidationCheckDefinition { supportedScopes: readonly ValidationScopeKind[]; defaultScopes?: readonly ValidationScopeKind[]; requiresGraph?: boolean; + graphUsage?: "none" | "optional" | "required"; + inactiveResult?: ( + context: ValidationCheckContext, + state: ValidationInactiveCheckState + ) => ValidationCheckResult | readonly ValidationDiagnostic[] | void | Promise; + inactiveStateWhenUnselected?: ValidationInactiveCheckState; graphRequirements?: ( context: ValidationCheckContext ) => readonly ValidationGraphQueryRequirement[] | Promise; @@ -146,36 +157,64 @@ function validateValidationCheckDefinition(definition: ValidationCheckDefinition if (!definition || typeof definition !== "object") { throw new ValidationCheckRegistryError("Validation check definition is required"); } + validateDefinitionIdentity(definition); + validateDefinitionScopes(definition); + validateDefinitionGraphSettings(definition); + validateDefinitionInactiveSettings(definition); + validateDefinitionHooks(definition); +} + +function validateDefinitionIdentity(definition: ValidationCheckDefinition): void { validateValidationCheckId(definition.id, "Validation check id"); validateNonEmptyString(definition.owner, "Validation check owner"); validateNonEmptyString(definition.adapter, "Validation check adapter"); if (!diagnosticSeverities.includes(definition.defaultSeverity)) { throw new ValidationCheckRegistryError(`Unknown validation check defaultSeverity: ${String(definition.defaultSeverity)}`); } +} + +function validateDefinitionScopes(definition: ValidationCheckDefinition): void { if (!Array.isArray(definition.supportedScopes) || definition.supportedScopes.length === 0) { throw new ValidationCheckRegistryError("Validation check supportedScopes must be a non-empty array"); } - for (const scope of definition.supportedScopes) { - if (!validationScopeKinds.includes(scope)) { - throw new ValidationCheckRegistryError(`Unknown validation check supported scope: ${String(scope)}`); - } + for (const scope of definition.supportedScopes) validateKnownScope(scope, "Validation check supported scope"); + if (definition.defaultScopes === undefined) return; + if (!Array.isArray(definition.defaultScopes)) { + throw new ValidationCheckRegistryError("Validation check defaultScopes must be an array when provided"); } - if (definition.defaultScopes !== undefined) { - if (!Array.isArray(definition.defaultScopes)) { - throw new ValidationCheckRegistryError("Validation check defaultScopes must be an array when provided"); - } - for (const scope of definition.defaultScopes) { - if (!validationScopeKinds.includes(scope)) { - throw new ValidationCheckRegistryError(`Unknown validation check default scope: ${String(scope)}`); - } - if (!definition.supportedScopes.includes(scope)) { - throw new ValidationCheckRegistryError(`Validation check default scope must also be supported: ${String(scope)}`); - } + for (const scope of definition.defaultScopes) { + validateKnownScope(scope, "Validation check default scope"); + if (!definition.supportedScopes.includes(scope)) { + throw new ValidationCheckRegistryError(`Validation check default scope must also be supported: ${String(scope)}`); } } +} + +function validateDefinitionGraphSettings(definition: ValidationCheckDefinition): void { if (definition.requiresGraph !== undefined && typeof definition.requiresGraph !== "boolean") { throw new ValidationCheckRegistryError("Validation check requiresGraph must be boolean"); } + if (definition.graphUsage !== undefined && !["none", "optional", "required"].includes(definition.graphUsage)) { + throw new ValidationCheckRegistryError("Validation check graphUsage must be none, optional, or required"); + } +} + +function validateDefinitionInactiveSettings(definition: ValidationCheckDefinition): void { + if (definition.inactiveResult !== undefined && typeof definition.inactiveResult !== "function") { + throw new ValidationCheckRegistryError("Validation check inactiveResult must be a function"); + } + if (definition.inactiveStateWhenUnselected === undefined) return; + if (definition.inactiveResult === undefined) { + throw new ValidationCheckRegistryError("Validation check inactiveStateWhenUnselected requires inactiveResult"); + } + if (definition.inactiveStateWhenUnselected !== "not_applicable" && definition.inactiveStateWhenUnselected !== "disabled") { + throw new ValidationCheckRegistryError( + `Unknown validation check inactiveStateWhenUnselected: ${String(definition.inactiveStateWhenUnselected)}` + ); + } +} + +function validateDefinitionHooks(definition: ValidationCheckDefinition): void { if (definition.graphRequirements !== undefined && typeof definition.graphRequirements !== "function") { throw new ValidationCheckRegistryError("Validation check graphRequirements must be a function"); } @@ -184,6 +223,12 @@ function validateValidationCheckDefinition(definition: ValidationCheckDefinition } } +function validateKnownScope(scope: unknown, label: string): void { + if (!validationScopeKinds.includes(scope as ValidationScopeKind)) { + throw new ValidationCheckRegistryError(`Unknown ${label}: ${String(scope)}`); + } +} + function validateValidationCheckId(checkId: unknown, label: string): string { const value = validateNonEmptyString(checkId, label); if (!validationCheckIdRegex.test(value)) { diff --git a/packages/validation/src/runner.ts b/packages/validation/src/runner.ts index 2d60403..c2969a9 100644 --- a/packages/validation/src/runner.ts +++ b/packages/validation/src/runner.ts @@ -8,6 +8,7 @@ import type { ValidationFailureCategory, ValidationRequest, ValidationResult, + PythonValidationCapabilityRun, PythonProjectContext, ValidationScopeKind, ValidationSkippedCheck @@ -66,6 +67,7 @@ export interface ValidationCheckCompleteEvent { diagnostics: readonly ValidationDiagnostic[]; run?: ValidationCheckRunSummary; skippedCheck?: ValidationSkippedCheck; + pythonCapabilityRuns?: readonly PythonValidationCapabilityRun[]; } export type ValidationCheckCompleteHandler = (event: ValidationCheckCompleteEvent) => void | Promise; @@ -95,6 +97,7 @@ interface CheckExecution { diagnosticsByCheck: Map; skippedChecks: ValidationSkippedCheck[]; pythonProjectContexts: PythonProjectContext[]; + pythonCapabilityRuns: PythonValidationCapabilityRun[]; failureResult?: ValidationResult; } @@ -103,7 +106,9 @@ interface ExecuteChecksArgs { scope: ResolvedValidationScope; graph: ValidationGraphQuerySession; fileView: ValidationFileView; + activeCheckIds: readonly string[]; selectedChecks: readonly ValidationCheckDefinition[]; + inactiveChecks: readonly ValidationCheckDefinition[]; totalStartedAt: number; clock: ValidationClock; runtime: ValidationRuntimePolicy; @@ -116,6 +121,7 @@ interface PreparedValidationArgs { scope: ResolvedValidationScope; graph: ValidationGraphQuerySession; selectedChecks: readonly ValidationCheckDefinition[]; + inactiveChecks: readonly ValidationCheckDefinition[]; totalStartedAt: number; options: RunnerRuntimeOptions; } @@ -131,6 +137,7 @@ interface SingleCheckOutcome { failureStatus?: Extract; providerError?: ValidationGraphProviderError; pythonProjectContexts: readonly PythonProjectContext[]; + pythonCapabilityRuns: readonly PythonValidationCapabilityRun[]; } export function createValidationRunner(options: CreateValidationRunnerOptions): ValidationRunner { @@ -181,6 +188,7 @@ async function runPreparedValidation( } const unsupportedResult = unsupportedScopeResult(selectedChecks, scope.kind, totalStartedAt, options.clock); if (unsupportedResult !== undefined) return unsupportedResult; + const inactiveChecks = selectInactiveValidationChecks(options.registry, selectedChecks, scope.kind); const graph = await resolveGraphSession(request, selectedChecks, options); const graphFailure = requiredGraphFailureResult(request, graph.status, selectedChecks, totalStartedAt, options.clock); if (graphFailure !== undefined) return graphFailure; @@ -190,6 +198,7 @@ async function runPreparedValidation( scope, graph, selectedChecks, + inactiveChecks, totalStartedAt, options }); @@ -199,6 +208,7 @@ async function runPreparedValidation( scope, graph, selectedChecks, + inactiveChecks, totalStartedAt, options }); @@ -244,7 +254,8 @@ async function runIntroducedValidationIncremental(args: PreparedValidationArgs): diagnostics: [], diagnosticsByCheck: new Map(), skippedChecks: [], - pythonProjectContexts: [] + pythonProjectContexts: [], + pythonCapabilityRuns: [] }; const quietOptions = withoutCheckCompleteOptions(args.options); for (const check of args.selectedChecks) { @@ -289,6 +300,7 @@ function mergeCheckExecution(target: CheckExecution, source: CheckExecution): vo target.diagnostics.push(...source.diagnostics); target.skippedChecks.push(...source.skippedChecks); mergePythonProjectContexts(target.pythonProjectContexts, source.pythonProjectContexts); + mergePythonCapabilityRuns(target.pythonCapabilityRuns, source.pythonCapabilityRuns); for (const [checkId, diagnostics] of source.diagnosticsByCheck) { target.diagnosticsByCheck.set(checkId, diagnostics); } @@ -337,7 +349,9 @@ async function executeValidationRequest(args: ExecuteValidationRequestArgs): Pro scope: args.scope, graph: args.graph, fileView, + activeCheckIds: args.selectedChecks.map((check) => check.id), selectedChecks: args.selectedChecks, + inactiveChecks: args.inactiveChecks, totalStartedAt: args.totalStartedAt, clock: args.options.clock, runtime: args.options.runtime, @@ -387,79 +401,134 @@ function requiredGraphFailureResult( } async function executeSelectedChecks(args: ExecuteChecksArgs): Promise { - const execution: CheckExecution = { + const execution = emptyCheckExecution(); + try { + for (const check of args.selectedChecks) { + const loopAction = await executeSelectedCheck(check, args, execution); + if (loopAction === "return") return execution; + if (loopAction === "break") break; + } + await executeInactiveChecks(args, execution); + return execution; + } finally { + aggregateExecutionFailure(args, execution); + } +} + +function emptyCheckExecution(): CheckExecution { + return { runs: [], diagnostics: [], diagnosticsByCheck: new Map(), skippedChecks: [], - pythonProjectContexts: [] + pythonProjectContexts: [], + pythonCapabilityRuns: [] }; - for (const check of args.selectedChecks) { - const skippedCheck = skippedGraphCheck(check, args.graph.status); - if (skippedCheck !== undefined) { - execution.skippedChecks.push(skippedCheck); - await emitCheckComplete(args, { - schemaVersion: 1, - kind: "validation.check", - checkId: check.id, - status: "skipped", - diagnostics: [], - skippedCheck - }); - continue; - } - const preloadFailure = await preloadCheckGraphRequirements(check, args, execution); - if (preloadFailure !== undefined) { - if ("status" in preloadFailure) { - execution.failureResult = preloadFailure; - await emitCheckComplete(args, failureCheckCompleteEvent(check, preloadFailure, execution.diagnostics)); - return execution; - } - execution.skippedChecks.push(preloadFailure); - await emitCheckComplete(args, { - schemaVersion: 1, - kind: "validation.check", - checkId: check.id, - status: "skipped", - diagnostics: [], - skippedCheck: preloadFailure - }); - continue; - } - const outcome = await runSingleCheck(check, args); - if (outcome.providerError !== undefined) { - if (args.request.graph.mode === "required") { - if (outcome.run !== undefined) execution.runs.push(outcome.run); - execution.failureResult = checkProviderFailureResult(check, args, execution, outcome.providerError); - await emitCheckComplete(args, outcomeCheckCompleteEvent(check, outcome)); - return execution; - } - const skippedCheck = skippedGraphProviderError(check, outcome.providerError); - execution.skippedChecks.push(skippedCheck); - await emitCheckComplete(args, { - schemaVersion: 1, - kind: "validation.check", - checkId: check.id, - status: "skipped", - diagnostics: [], - skippedCheck - }); - continue; - } +} + +async function executeSelectedCheck( + check: ValidationCheckDefinition, + args: ExecuteChecksArgs, + execution: CheckExecution +): Promise<"continue" | "break" | "return"> { + const skippedCheck = skippedGraphCheck(check, args.graph.status); + if (skippedCheck !== undefined) { + await recordSkippedCheck(args, execution, check.id, skippedCheck); + return "continue"; + } + const preloadResult = await handlePreloadFailure(check, args, execution); + if (preloadResult !== undefined) return preloadResult; + const outcome = await runSingleCheck(check, args); + const providerResult = await handleProviderError(check, args, execution, outcome); + if (providerResult !== undefined) return providerResult; + await recordCheckOutcome(args, execution, check, outcome); + if (outcome.failureStatus !== undefined && outcome.failureMessage !== undefined) { + execution.failureResult = checkRunFailureResult(check, args, execution, outcome.failureStatus, outcome.failureMessage); + return "return"; + } + return args.failFast && outcome.run?.status === "policy_failure" ? "break" : "continue"; +} + +async function handlePreloadFailure( + check: ValidationCheckDefinition, + args: ExecuteChecksArgs, + execution: CheckExecution +): Promise<"continue" | "return" | undefined> { + const preloadFailure = await preloadCheckGraphRequirements(check, args, execution); + if (preloadFailure === undefined) return undefined; + if ("status" in preloadFailure) { + execution.failureResult = preloadFailure; + await emitCheckComplete(args, failureCheckCompleteEvent(check, preloadFailure, execution.diagnostics)); + return "return"; + } + await recordSkippedCheck(args, execution, check.id, preloadFailure); + return "continue"; +} + +async function handleProviderError( + check: ValidationCheckDefinition, + args: ExecuteChecksArgs, + execution: CheckExecution, + outcome: SingleCheckOutcome +): Promise<"continue" | "return" | undefined> { + if (outcome.providerError === undefined) return undefined; + if (args.request.graph.mode === "required") { if (outcome.run !== undefined) execution.runs.push(outcome.run); - mergePythonProjectContexts(execution.pythonProjectContexts, outcome.pythonProjectContexts); - execution.diagnosticsByCheck.set(check.id, [...outcome.diagnostics]); - execution.diagnostics.push(...outcome.diagnostics); + execution.failureResult = checkProviderFailureResult(check, args, execution, outcome.providerError); await emitCheckComplete(args, outcomeCheckCompleteEvent(check, outcome)); - if (outcome.failureStatus !== undefined && outcome.failureMessage !== undefined) { - execution.failureResult = checkRunFailureResult(check, args, execution, outcome.failureStatus, outcome.failureMessage); - return execution; - } - if (args.failFast && outcome.run?.status === "policy_failure") { - return execution; - } + return "return"; } - return execution; + await recordSkippedCheck(args, execution, check.id, skippedGraphProviderError(check, outcome.providerError)); + return "continue"; +} + +async function recordSkippedCheck( + args: ExecuteChecksArgs, + execution: CheckExecution, + checkId: string, + skippedCheck: ValidationSkippedCheck +): Promise { + execution.skippedChecks.push(skippedCheck); + await emitCheckComplete(args, { + schemaVersion: 1, + kind: "validation.check", + checkId, + status: "skipped", + diagnostics: [], + skippedCheck + }); +} + +async function recordCheckOutcome( + args: ExecuteChecksArgs, + execution: CheckExecution, + check: ValidationCheckDefinition, + outcome: SingleCheckOutcome +): Promise { + if (outcome.run !== undefined) execution.runs.push(outcome.run); + mergePythonProjectContexts(execution.pythonProjectContexts, outcome.pythonProjectContexts); + mergePythonCapabilityRuns(execution.pythonCapabilityRuns, outcome.pythonCapabilityRuns); + execution.diagnosticsByCheck.set(check.id, [...outcome.diagnostics]); + execution.diagnostics.push(...outcome.diagnostics); + await emitCheckComplete(args, outcomeCheckCompleteEvent(check, outcome)); +} + +function aggregateExecutionFailure(args: ExecuteChecksArgs, execution: CheckExecution): void { + if (execution.failureResult === undefined) return; + const failure = execution.failureResult; + execution.failureResult = aggregateForChecks(args.selectedChecks, { + runs: execution.runs, + skippedChecks: execution.skippedChecks, + diagnostics: execution.diagnostics, + pythonProjectContexts: execution.pythonProjectContexts, + pythonCapabilityRuns: execution.pythonCapabilityRuns, + generatedAt: args.clock.isoNow(), + durationMs: elapsed(args.totalStartedAt, args.clock.nowMs()), + ...(failure.graphStatus === undefined ? {} : { graphStatus: failure.graphStatus }), + status: failure.status, + ...(failure.failure === undefined ? {} : { failure: failure.failure }), + ...(failure.refusal === undefined ? {} : { refusal: failure.refusal }) + }); } function outcomeCheckCompleteEvent(check: ValidationCheckDefinition, outcome: SingleCheckOutcome): ValidationCheckCompleteEvent { @@ -469,7 +538,8 @@ function outcomeCheckCompleteEvent(check: ValidationCheckDefinition, outcome: Si checkId: check.id, status: outcome.run?.status ?? "passed", diagnostics: outcome.diagnostics, - ...(outcome.run === undefined ? {} : { run: outcome.run }) + ...(outcome.run === undefined ? {} : { run: outcome.run }), + ...(outcome.pythonCapabilityRuns.length === 0 ? {} : { pythonCapabilityRuns: outcome.pythonCapabilityRuns }) }; } @@ -508,6 +578,7 @@ function introducedExecution(before: CheckExecution, after: CheckExecution): Che diagnosticsByCheck, skippedChecks: after.skippedChecks, pythonProjectContexts: after.pythonProjectContexts, + pythonCapabilityRuns: after.pythonCapabilityRuns, failureResult: after.failureResult }; } @@ -589,6 +660,7 @@ async function runSingleCheck(check: ValidationCheckDefinition, args: ExecuteChe return { diagnostics, pythonProjectContexts: normalized.pythonProjectContexts ?? [], + pythonCapabilityRuns: normalized.pythonCapabilityRuns ?? [], failureMessage, failureStatus, run: { @@ -605,6 +677,7 @@ async function runSingleCheck(check: ValidationCheckDefinition, args: ExecuteChe return { diagnostics: [], pythonProjectContexts: [], + pythonCapabilityRuns: [], providerError: error, run: { checkId: check.id, @@ -618,6 +691,7 @@ async function runSingleCheck(check: ValidationCheckDefinition, args: ExecuteChe return { diagnostics: [], pythonProjectContexts: [], + pythonCapabilityRuns: [], failureMessage: errorMessage(error), failureStatus: "infrastructure_failure", run: { @@ -631,9 +705,20 @@ async function runSingleCheck(check: ValidationCheckDefinition, args: ExecuteChe } } +async function executeInactiveChecks(args: ExecuteChecksArgs, execution: CheckExecution): Promise { + for (const check of args.inactiveChecks) { + if (check.inactiveResult === undefined || check.inactiveStateWhenUnselected === undefined) continue; + const result = await check.inactiveResult(checkContext(args), check.inactiveStateWhenUnselected); + const normalized = normalizeCheckResult(result); + mergePythonProjectContexts(execution.pythonProjectContexts, normalized.pythonProjectContexts ?? []); + mergePythonCapabilityRuns(execution.pythonCapabilityRuns, normalized.pythonCapabilityRuns ?? []); + } +} + function checkContext(args: ExecuteChecksArgs) { return { request: args.request, + selectedCheckIds: args.activeCheckIds, scope: args.scope, graphStatus: args.graph.status, graph: args.graph, @@ -674,6 +759,7 @@ function checkRunFailureResult( skippedChecks: execution.skippedChecks, diagnostics: execution.diagnostics, pythonProjectContexts: execution.pythonProjectContexts, + pythonCapabilityRuns: execution.pythonCapabilityRuns, generatedAt: args.clock.isoNow(), durationMs: elapsed(args.totalStartedAt, args.clock.nowMs()), graphStatus: args.graph.status, @@ -697,6 +783,7 @@ function checkProviderFailureResult( skippedChecks: execution.skippedChecks, diagnostics: execution.diagnostics, pythonProjectContexts: execution.pythonProjectContexts, + pythonCapabilityRuns: execution.pythonCapabilityRuns, generatedAt: args.clock.isoNow(), durationMs: elapsed(args.totalStartedAt, args.clock.nowMs()), graphStatus: error.status, @@ -720,6 +807,7 @@ function graphRequirementFailureResult( skippedChecks: execution.skippedChecks, diagnostics: execution.diagnostics, pythonProjectContexts: execution.pythonProjectContexts, + pythonCapabilityRuns: execution.pythonCapabilityRuns, generatedAt: args.clock.isoNow(), durationMs: elapsed(args.totalStartedAt, args.clock.nowMs()), graphStatus: args.graph.status, @@ -744,6 +832,7 @@ function finalValidationResult( skippedChecks: execution.skippedChecks, diagnostics: execution.diagnostics, pythonProjectContexts: execution.pythonProjectContexts, + pythonCapabilityRuns: execution.pythonCapabilityRuns, generatedAt: clock.isoNow(), durationMs: elapsed(totalStartedAt, clock.nowMs()), graphStatus @@ -829,7 +918,8 @@ function normalizeCheckResult(result: ValidationCheckResult | readonly Validatio status: result.status, outcome: result.outcome, failureMessage: result.failureMessage, - pythonProjectContexts: result.pythonProjectContexts + pythonProjectContexts: result.pythonProjectContexts, + pythonCapabilityRuns: result.pythonCapabilityRuns }; } @@ -839,6 +929,26 @@ function mergePythonProjectContexts(target: PythonProjectContext[], source: read target.splice(0, target.length, ...[...byTarget.values()].sort((left, right) => left.target.localeCompare(right.target))); } +function mergePythonCapabilityRuns(target: PythonValidationCapabilityRun[], source: readonly PythonValidationCapabilityRun[]): void { + const byFingerprint = new Map(target.map((run) => [JSON.stringify(run), run])); + for (const run of source) byFingerprint.set(JSON.stringify(run), run); + target.splice(0, target.length, ...[...byFingerprint.values()]); +} + +function selectInactiveValidationChecks( + registry: ValidationCheckRegistry, + selectedChecks: readonly ValidationCheckDefinition[], + scopeKind: ValidationScopeKind +): readonly ValidationCheckDefinition[] { + const selectedIds = new Set(selectedChecks.map((check) => check.id)); + return registry.checks.filter((check) => + !selectedIds.has(check.id) && + check.supportedScopes.includes(scopeKind) && + check.inactiveResult !== undefined && + check.inactiveStateWhenUnselected !== undefined + ); +} + function isValidationDiagnosticArray( result: ValidationCheckResult | readonly ValidationDiagnostic[] ): result is readonly ValidationDiagnostic[] { diff --git a/scripts/check-python-pytest-authority.mjs b/scripts/check-python-pytest-authority.mjs new file mode 100644 index 0000000..0bbdea2 --- /dev/null +++ b/scripts/check-python-pytest-authority.mjs @@ -0,0 +1,271 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { createHash, randomBytes } from "node:crypto"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + renameSync, + rmSync, + writeFileSync +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createStagedOpcorePackage } from "./stage-opcore-bundle.mjs"; + +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const opcoreBin = join(repoRoot, "packages/opcore/dist/index.js"); +const fixtureRoot = join(repoRoot, "packages/fixtures/validation-python/pytest-authority"); +const tempRoot = mkdtempSync(join(tmpdir(), "opcore-pytest-authority-")); +const workspacePrefix = `opcore-python-pytest-workspace-${randomBytes(6).toString("hex")}-`; + +try { + const venvRoot = join(tempRoot, "venv"); + run("python3", ["-m", "venv", venvRoot], { cwd: repoRoot }); + const venvPython = join(venvRoot, "bin/python"); + const venvBin = join(venvRoot, "bin"); + run(venvPython, ["-m", "pip", "install", "--disable-pip-version-check", "pytest==8.4.1"], { cwd: repoRoot }); + const pytestVersion = run(venvPython, ["-m", "pytest", "--version"], { cwd: repoRoot }).stdout.trim(); + const env = { + ...process.env, + OPCORE_INTERNAL_PYTEST_WORKSPACE_PREFIX: workspacePrefix, + VIRTUAL_ENV: venvRoot, + PATH: `${venvBin}:${process.env.PATH ?? ""}` + }; + + const baseTempRoots = currentPytestTempRoots(); + const scenarios = [ + runFixtureScenario("pass", env), + runFixtureScenario("fail", env, { expectedExitCodes: [1] }), + runFixtureScenario("collection-fail", env, { expectedExitCodes: [1] }), + runFixtureScenario("timeout", env, { expectedExitCodes: [1] }), + runFixtureScenario("manifest", env), + runFixtureScenario("workspace-isolation", env) + ]; + const packedInstall = runPackedInstallScenario(env); + installAmbientPlugin(venvPython); + scenarios.push(runFixtureScenario("pass", env, { label: "ambient-plugin-isolation" })); + assertNoPytestTempLeaks(baseTempRoots); + + console.log(JSON.stringify({ + pytestVersion, + scenarios, + packedInstall + }, null, 2)); +} finally { + rmSync(tempRoot, { recursive: true, force: true }); +} + +function runFixtureScenario(name, env, options = {}) { + const label = options.label ?? name; + const repo = materializeFixture(name, label); + const beforeDigest = repoDigest(repo); + const beforeTempRoots = currentPytestTempRoots(); + run("node", [opcoreBin, "graph", "build", "--repo", repo, "--json"], { cwd: repoRoot, env }); + const result = run( + "node", + [opcoreBin, "check", "files", "src/app.py", "--repo", repo, "--checks", "python.pytest", "--json"], + { cwd: repoRoot, env, allowedExitCodes: options.expectedExitCodes ?? [0] } + ); + const parsed = JSON.parse(result.stdout); + const capabilityRun = parsed.validationResult?.pythonCapabilityRuns?.[0]; + assert.equal(capabilityRun?.capability, "pytest", label); + assert.equal(capabilityRun?.cleanup?.ok, true, label); + assert.equal(capabilityRun?.cleanup?.attempted, true, label); + assert.equal(repoDigest(repo), beforeDigest, `${label} mutated source repo`); + assertNoNewPytestTempRoots(beforeTempRoots, label); + verifyScenario(label, parsed.validationResult, capabilityRun); + return summarize(label, parsed.validationResult, capabilityRun); +} + +function runPackedInstallScenario(env) { + const repo = materializeFixture("pass", "packed-install"); + const beforeDigest = repoDigest(repo); + run("node", [opcoreBin, "graph", "build", "--repo", repo, "--json"], { cwd: repoRoot, env }); + + const packDestination = join(tempRoot, "packed-install-artifacts"); + mkdirSync(packDestination, { recursive: true }); + const staged = createStagedOpcorePackage(tempRoot); + try { + const pack = JSON.parse(run("npm", ["pack", "--json", "--pack-destination", packDestination], { + cwd: staged.packageDir + }).stdout)[0]; + const consumer = join(tempRoot, "packed-consumer"); + mkdirSync(consumer, { recursive: true }); + run("npm", ["init", "-y"], { cwd: consumer }); + run("npm", ["install", "--silent", join(packDestination, pack.filename)], { cwd: consumer }); + const installedBin = join(consumer, "node_modules", ".bin", "opcore"); + const beforeTempRoots = currentPytestTempRoots(); + const result = run( + installedBin, + ["check", "files", "src/app.py", "--repo", repo, "--checks", "python.pytest", "--json"], + { cwd: consumer, env } + ); + const parsed = JSON.parse(result.stdout); + const capabilityRun = parsed.validationResult?.pythonCapabilityRuns?.[0]; + assert.equal(parsed.validationResult?.status, "passed"); + assert.equal(capabilityRun?.outcome, "passed"); + assert.equal(repoDigest(repo), beforeDigest, "packed install mutated source repo"); + assertNoNewPytestTempRoots(beforeTempRoots, "packed-install"); + return { + status: parsed.validationResult.status, + outcome: capabilityRun.outcome, + selectionMode: capabilityRun.selectionMode, + cleanup: capabilityRun.cleanup, + tarball: pack.filename + }; + } finally { + staged.cleanup(); + } +} + +function verifyScenario(label, validationResult, capabilityRun) { + switch (label) { + case "pass": + case "ambient-plugin-isolation": + case "workspace-isolation": + assert.equal(validationResult?.status, "passed", label); + assert.equal(capabilityRun?.outcome, "passed", label); + assert.equal(capabilityRun?.counts?.passedCount > 0, true, label); + if (label === "workspace-isolation") { + assert.equal(capabilityRun?.collection?.stage, "collection"); + assert.equal(capabilityRun?.execution?.stage, "execution"); + } + break; + case "fail": + assert.equal(validationResult?.status, "policy_failure", label); + assert.equal(capabilityRun?.outcome, "findings", label); + assert.equal((capabilityRun?.counts?.failedCount ?? 0) + (capabilityRun?.counts?.errorCount ?? 0) > 0, true, label); + break; + case "collection-fail": + assert.equal(validationResult?.status, "policy_failure", label); + assert.equal(capabilityRun?.collection?.termination, "exited", label); + assert.equal(capabilityRun?.outcome !== "passed", true, label); + assert.equal(Object.hasOwn(capabilityRun, "execution"), false, label); + break; + case "timeout": + assert.equal(validationResult?.status, "policy_failure", label); + assert.equal(capabilityRun?.outcome, "timeout", label); + assert.equal(capabilityRun?.execution?.termination, "timeout", label); + break; + case "manifest": + assert.equal(validationResult?.status, "passed", label); + assert.equal(capabilityRun?.outcome, "passed", label); + assert.equal(capabilityRun?.selectionMode, "manifest", label); + assert.equal((capabilityRun?.counts?.passedCount ?? 0) > 100, true, label); + break; + default: + throw new Error(`Unknown scenario: ${label}`); + } +} + +function summarize(label, validationResult, capabilityRun) { + return { + name: label, + status: validationResult.status, + outcome: capabilityRun.outcome, + activation: capabilityRun.activation, + selectionMode: capabilityRun.selectionMode, + counts: capabilityRun.counts, + cleanup: capabilityRun.cleanup, + collection: capabilityRun.collection, + execution: capabilityRun.execution + }; +} + +function installAmbientPlugin(venvPython) { + const sitePackages = run(venvPython, ["-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], { cwd: repoRoot }).stdout.trim(); + const distInfo = join(sitePackages, "opcore_bad_pytest_plugin-0.0.0.dist-info"); + mkdirSync(distInfo, { recursive: true }); + writeFileSync(join(sitePackages, "opcore_bad_pytest_plugin.py"), "raise RuntimeError('ambient pytest plugin loaded')\n"); + writeFileSync(join(distInfo, "METADATA"), [ + "Metadata-Version: 2.1", + "Name: opcore-bad-pytest-plugin", + "Version: 0.0.0", + "" + ].join("\n")); + writeFileSync(join(distInfo, "entry_points.txt"), [ + "[pytest11]", + "opcore_bad_plugin = opcore_bad_pytest_plugin", + "" + ].join("\n")); +} + +function materializeFixture(name, destinationName = name) { + const source = join(fixtureRoot, name); + const repo = join(tempRoot, destinationName); + cpSync(source, repo, { recursive: true }); + for (const path of walkFiles(repo)) { + if (!path.endsWith(".fixture")) continue; + renameSync(path, path.slice(0, -".fixture".length)); + } + const localBin = join(repo, ".venv", "bin"); + mkdirSync(localBin, { recursive: true }); + for (const tool of ["python", "python3", "pytest"]) { + const wrapperPath = join(localBin, tool); + writeFileSync(wrapperPath, `#!/bin/sh\nexec "${join(tempRoot, "venv", "bin", tool)}" "$@"\n`); + chmodSync(wrapperPath, 0o755); + } + return repo; +} + +function currentPytestTempRoots() { + return new Set( + readdirSync(tmpdir(), { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && entry.name.startsWith(workspacePrefix)) + .map((entry) => join(tmpdir(), entry.name)) + ); +} + +function assertNoNewPytestTempRoots(before, label) { + const after = currentPytestTempRoots(); + const leaked = [...after].filter((path) => !before.has(path)); + assert.deepEqual(leaked, [], `${label} leaked pytest temp roots`); +} + +function assertNoPytestTempLeaks(before) { + const after = currentPytestTempRoots(); + const leaked = [...after].filter((path) => !before.has(path)); + assert.deepEqual(leaked, [], "pytest authority proof leaked temp roots"); +} + +function repoDigest(root) { + const hash = createHash("sha256"); + for (const path of walkFiles(root)) { + hash.update(path.slice(root.length + 1)); + hash.update("\0"); + hash.update(readFileSync(path)); + hash.update("\0"); + } + return `sha256:${hash.digest("hex")}`; +} + +function walkFiles(root) { + const entries = []; + for (const dirent of readdirSync(root, { withFileTypes: true })) { + if ([".git", ".opcore", ".pytest_cache", "__pycache__"].includes(dirent.name)) continue; + const path = join(root, dirent.name); + if (dirent.isDirectory()) entries.push(...walkFiles(path)); + else if (dirent.isFile()) entries.push(path); + } + return entries.sort(); +} + +function run(command, args, options) { + const result = spawnSync(command, args, { + cwd: options.cwd, + env: options.env, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"] + }); + const allowedExitCodes = options.allowedExitCodes ?? [0]; + if (!allowedExitCodes.includes(result.status ?? -1)) { + throw new Error(`${command} ${args.join(" ")} failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } + return result; +} diff --git a/scripts/ci/run-local-ci-equivalent.sh b/scripts/ci/run-local-ci-equivalent.sh index 3da54bb..fadda62 100755 --- a/scripts/ci/run-local-ci-equivalent.sh +++ b/scripts/ci/run-local-ci-equivalent.sh @@ -6,6 +6,33 @@ cd "${repo_root}" generated_artifact_backup="" +terminate_descendant_processes() { + local pid="$1" + local signal="${2:-TERM}" + local child + + while IFS= read -r child; do + [ -n "${child}" ] || continue + terminate_descendant_processes "${child}" "${signal}" + done < <(pgrep -P "${pid}" || true) + + kill "-${signal}" "${pid}" 2>/dev/null || true +} + +cleanup_local_ci() { + local status="$1" + local child + + while IFS= read -r child; do + [ -n "${child}" ] || continue + terminate_descendant_processes "${child}" TERM + done < <(pgrep -P "$$" || true) + + restore_generated_artifacts + rm -f "${changed_file_list}" + exit "${status}" +} + run_step() { printf '\n==> %s\n' "$*" "$@" @@ -90,7 +117,7 @@ run_docs_or_agent_gate() { } changed_file_list="$(mktemp "${TMPDIR:-/tmp}/opcore-local-ci-changed.XXXXXX")" -trap 'restore_generated_artifacts; rm -f "${changed_file_list}"' EXIT +trap 'cleanup_local_ci "$?"' EXIT collect_changed_files > "${changed_file_list}" if docs_or_agent_only_changes "${changed_file_list}"; then diff --git a/scripts/generate-cutover-receipt.mjs b/scripts/generate-cutover-receipt.mjs index 5e2724c..cbf0fa3 100644 --- a/scripts/generate-cutover-receipt.mjs +++ b/scripts/generate-cutover-receipt.mjs @@ -515,7 +515,7 @@ function runPythonToolDegradationNegativeChecks(tempRoot, opcoreBin, env, comman throw new Error("python-relevant-tests-no-pytest did not pass with graph-backed relevant-test evidence"); } const relevantTestCodes = relevantTestsParsed.validationResult.diagnostics?.map((diagnostic) => diagnostic.code) ?? []; - if (!relevantTestCodes.some((code) => code === "PY_RELEVANT_TESTS_FOUND" || code === "PY_RELEVANT_TESTS_ABSENT")) { + if (!relevantTestCodes.some((code) => code === "PY_RELEVANT_TEST_CANDIDATES_FOUND" || code === "PY_RELEVANT_TEST_CANDIDATES_ABSENT")) { throw new Error("python-relevant-tests-no-pytest did not report Python relevant-test graph evidence"); } const statusResult = run(opcoreBin, ["status", "--json"], { diff --git a/scripts/generate-release-receipt.mjs b/scripts/generate-release-receipt.mjs index 3b5cb2a..eef0e20 100644 --- a/scripts/generate-release-receipt.mjs +++ b/scripts/generate-release-receipt.mjs @@ -9,6 +9,7 @@ import { statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { @@ -187,7 +188,7 @@ function packageSourceFor(packageName, stagedPackages) { }; } if (!stagedPackages.has(packageName)) { - stagedPackages.set(packageName, createStagedOpcorePackage(join(repoRoot, packDestination))); + stagedPackages.set(packageName, createStagedOpcorePackage(tmpdir())); } const staged = stagedPackages.get(packageName); return { repoRoot: staged.stageRoot, packageRoot: staged.packageRoot, packageDir: staged.packageDir }; diff --git a/scripts/lib/launch-claim-scrub.mjs b/scripts/lib/launch-claim-scrub.mjs index 58d5d1d..1df6a6b 100644 --- a/scripts/lib/launch-claim-scrub.mjs +++ b/scripts/lib/launch-claim-scrub.mjs @@ -118,13 +118,19 @@ export function collectNpmPackTextEntries(repoRoot, packageInfos) { } export function collectPackageTarballTextEntries(tarballPath, labelPrefix = "npm-pack") { - const list = runTar(["-tf", tarballPath], `list ${tarballPath}`) - .split(/\r?\n/) - .filter((path) => path.length > 0 && isScrubbableTextPath(path)); - return list.map((path) => ({ - label: `${labelPrefix}:${path}`, - text: runTar(["-xOf", tarballPath, path], `read ${tarballPath}:${path}`) - })); + const extractionRoot = mkdtempSync(join(tmpdir(), "opcore-launch-scrub-tar-")); + try { + runTar(["-xf", tarballPath, "-C", extractionRoot], `extract ${tarballPath}`); + return walkFiles(extractionRoot) + .filter((path) => isScrubbableTextPath(path)) + .map((path) => ({ + label: `${labelPrefix}:${relative(extractionRoot, path).split("\\").join("/")}`, + path, + text: readFileSync(path, "utf8") + })); + } finally { + rmSync(extractionRoot, { recursive: true, force: true }); + } } export function collectInstalledPackageTextEntries(projectRoot, packageNames) { diff --git a/tests/asp-provider.test.mjs b/tests/asp-provider.test.mjs index 12feb6e..111d374 100644 --- a/tests/asp-provider.test.mjs +++ b/tests/asp-provider.test.mjs @@ -37,6 +37,7 @@ const allCheckIds = [ "python.import-graph", "python.dead-code", "python.relevant-tests", + "python.pytest", "docs.existence", "docs.staleness", "docs.freshness", diff --git a/tests/command-router.test.mjs b/tests/command-router.test.mjs index c2d069d..137ee23 100644 --- a/tests/command-router.test.mjs +++ b/tests/command-router.test.mjs @@ -38,7 +38,8 @@ const requiredDefaultCheckIds = [ "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]; describe("Opcore command router", () => { diff --git a/tests/conformance.test.mjs b/tests/conformance.test.mjs index 0d07ca4..294256b 100644 --- a/tests/conformance.test.mjs +++ b/tests/conformance.test.mjs @@ -394,7 +394,8 @@ describe("conformance fixture metadata", () => { "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]); assert.deepEqual(validation.degradedTools, ["mypy", "pyright", "ruff", "pytest"]); }); diff --git a/tests/contracts.test.mjs b/tests/contracts.test.mjs index e59d2a5..b40065c 100644 --- a/tests/contracts.test.mjs +++ b/tests/contracts.test.mjs @@ -866,6 +866,7 @@ describe("Opcore shared contracts", () => { it("validates canonical Python project contexts and rejects invalid outcomes and provenance", () => { const context = validPythonProjectContext(); assert.equal(validatePythonProjectContext(context).schemaId, PYTHON_PROJECT_CONTEXT_SCHEMA_ID); + assert.equal(validatePythonProjectContext({ ...context, target: "services/api/pyproject.toml" }).target, "services/api/pyproject.toml"); assert.equal( validateValidationResultPayload(validValidationResult({ pythonProjectContexts: [context] })).pythonProjectContexts[0].projectRoot, "services/api" diff --git a/tests/fixtures/package-packlists.json b/tests/fixtures/package-packlists.json index a946aea..9894434 100644 --- a/tests/fixtures/package-packlists.json +++ b/tests/fixtures/package-packlists.json @@ -400,6 +400,9 @@ "node_modules/@the-open-engine/opcore-validation-python/dist/index.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/index.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/index.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/materialized-workspace.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/materialized-workspace.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/materialized-workspace.js", "node_modules/@the-open-engine/opcore-validation-python/dist/process.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/process.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/process.js", @@ -421,12 +424,39 @@ "node_modules/@the-open-engine/opcore-validation-python/dist/protocol-validation.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/protocol-validation.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/protocol-validation.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-check.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-check.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-check.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-hook-source.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-hook-source.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-hook-source.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-process.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-process.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-process.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-project-runner.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-project-runner.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-project-runner.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-protocol.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-protocol.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-protocol.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-result.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-result.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-result.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-types.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-types.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-types.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-workspace.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-workspace.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/pytest-workspace.js", "node_modules/@the-open-engine/opcore-validation-python/dist/relevant-tests-check.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/relevant-tests-check.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/relevant-tests-check.js", "node_modules/@the-open-engine/opcore-validation-python/dist/source-files.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/source-files.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/source-files.js", + "node_modules/@the-open-engine/opcore-validation-python/dist/source-closure.d.ts", + "node_modules/@the-open-engine/opcore-validation-python/dist/source-closure.d.ts.map", + "node_modules/@the-open-engine/opcore-validation-python/dist/source-closure.js", "node_modules/@the-open-engine/opcore-validation-python/dist/source-hygiene-check.d.ts", "node_modules/@the-open-engine/opcore-validation-python/dist/source-hygiene-check.d.ts.map", "node_modules/@the-open-engine/opcore-validation-python/dist/source-hygiene-check.js", diff --git a/tests/gate-negative-cutover.test.mjs b/tests/gate-negative-cutover.test.mjs new file mode 100644 index 0000000..688959f --- /dev/null +++ b/tests/gate-negative-cutover.test.mjs @@ -0,0 +1,2 @@ +process.env.OPCORE_GATE_NEGATIVE_GROUP = "cutover"; +await import("./gate-negative-fixtures.test.mjs"); diff --git a/tests/gate-negative-fixtures.test.mjs b/tests/gate-negative-fixtures.test.mjs index ad0d351..5ef4f1a 100644 --- a/tests/gate-negative-fixtures.test.mjs +++ b/tests/gate-negative-fixtures.test.mjs @@ -1,6 +1,6 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; -import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { constants, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -19,7 +19,6 @@ import { bundledExternalRuntimePackageNames } from "../scripts/release-package-d import { externalRuntimePackageDir } from "../scripts/stage-opcore-bundle.mjs"; const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const releaseDocsLockTimeoutMs = 900000; const copiedRepoSkips = new Set([ ".git", "node_modules", @@ -30,15 +29,23 @@ const copiedRepoSkips = new Set([ ".codex", ".gemini", ".opencode", + ".opcore", ".lattice", ".code-review-graph", ".rox-cache", ".robustness-engine-cache", ".receipt-test.lock" ]); +const tempRepoTemplateCache = new Map(); +const activeGroup = process.env.OPCORE_GATE_NEGATIVE_GROUP ?? ""; +function gateTest(group, name, fn) { + if (activeGroup === group) it(name, fn); +} + +if (activeGroup) { describe("negative gate fixtures", () => { - it("rejects tracked TypeScript build info", () => { + gateTest("plain", "rejects tracked TypeScript build info", () => { const repo = tempRepo(); writeFileSync(join(repo, "packages/contracts/tsconfig.tsbuildinfo"), "{}\n"); run(repo, "git", ["add", "-f", "packages/contracts/tsconfig.tsbuildinfo"]); @@ -47,7 +54,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /Generated TypeScript build info must not be checked in/); }); - it("rejects Python CRG provenance markers", () => { + gateTest("plain", "rejects Python CRG provenance markers", () => { const repo = tempRepo(); writeFileSync(join(repo, "pyproject.toml"), "[project]\nname = \"code-review-graph\"\n"); run(repo, "git", ["add", "pyproject.toml"]); @@ -56,7 +63,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /Forbidden Python packaging file/); }); - it("rejects provenance receipt checks without build artifacts", () => { + gateTest("plain", "rejects provenance receipt checks without build artifacts", () => { const repo = tempRepo(); const workflowPath = join(repo, ".github/workflows/provenance.yml"); const workflow = readFileSync(workflowPath, "utf8") @@ -68,7 +75,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /provenance\.yml.*(build.*release receipt|release-receipt:check.*build)/i); }); - it("rejects high-confidence secrets in release receipt scan", () => { + gateTest("release-a", "rejects high-confidence secrets in release receipt scan", () => { const repo = tempRepo({ includeDist: true }); writeFileSync(join(repo, "secret.txt"), `OPENAI_API_KEY=${JSON.stringify(`sk-${"a".repeat(40)}`)}\n`); @@ -76,7 +83,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /Secret\/history scan.*secret\.txt|openai_api_key/i); }); - it("allows reviewed path-scoped secret false positives", () => { + gateTest("release-a", "allows reviewed path-scoped secret false positives", () => { const repo = tempRepo({ includeDist: true }); writeFileSync(join(repo, "tmp-allowlisted-secret.txt"), ["token", " = ", JSON.stringify("documented-placeholder-value"), "\n"].join("")); writeFileSync( @@ -105,7 +112,7 @@ describe("negative gate fixtures", () => { assert.equal(scan.findingCount, 0); }); - it("rejects secret allowlist entries without reviewed metadata", () => { + gateTest("release-a", "rejects secret allowlist entries without reviewed metadata", () => { const repo = tempRepo({ includeDist: true }); writeFileSync( join(repo, "docs/release/secret-scan-allowlist.json"), @@ -128,7 +135,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /allowlist.*reviewer|allowlist.*reason|allowlist.*expiresAt/i); }); - it("rejects secret allowlist entries without path or commit scope", () => { + gateTest("release-a", "rejects secret allowlist entries without path or commit scope", () => { const repo = tempRepo({ includeDist: true }); writeFileSync( join(repo, "docs/release/secret-scan-allowlist.json"), @@ -153,7 +160,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /allowlist.*path|allowlist.*commit/i); }); - it("rejects unexpected package files in release package inspection", () => { + gateTest("release-a", "rejects unexpected package files in release package inspection", () => { const repo = tempRepo({ includeDist: true }); const manifestPath = join(repo, "packages/edit/package.json"); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); @@ -165,7 +172,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /packed files mismatch|EXTRA\.md/); }); - it("rejects old public bins in release package inspection", () => { + gateTest("release-a", "rejects old public bins in release package inspection", () => { const repo = tempRepo({ includeDist: true }); const manifestPath = join(repo, "packages/opcore/package.json"); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); @@ -176,7 +183,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /forbidden old public bin crg/); }); - it("rejects canonical ASP server manifest launch claim overreach", () => { + gateTest("release-b", "rejects canonical ASP server manifest launch claim overreach", () => { const repo = tempRepo({ includeDist: true }); const manifestPath = join(repo, "packages/asp-provider/dist/manifests/asp-server.json"); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); @@ -188,7 +195,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /packages\/asp-provider\/dist\/manifests\/asp-server\.json/); }); - it("rejects generic Opcore replacement overclaims in launch docs", () => { + gateTest("release-b", "rejects generic Opcore replacement overclaims in launch docs", () => { const repo = tempRepo({ includeDist: true }); writeFileSync(join(repo, "docs/quickstart.md"), "Opcore replaces your linters.\n"); @@ -198,7 +205,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /docs\/quickstart\.md/); }); - it("rejects blended quality score overclaims in launch docs", () => { + gateTest("release-b", "rejects blended quality score overclaims in launch docs", () => { const repo = tempRepo({ includeDist: true }); writeFileSync(join(repo, "docs/quickstart.md"), "Opcore reports a blended quality score.\n"); @@ -208,7 +215,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /docs\/quickstart\.md/); }); - it("rejects bad descriptor artifact references in release descriptor inspection", () => { + gateTest("release-b", "rejects bad descriptor artifact references in release descriptor inspection", () => { const repo = tempRepo({ includeDist: true }); const descriptorPath = join(repo, "packages/opcore/dist/descriptors/opcore.managed-tool.json"); const descriptor = JSON.parse(readFileSync(descriptorPath, "utf8")); @@ -221,7 +228,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /packaged descriptor artifact|missing\.json/); }); - it("rejects missing native checksum evidence in release package inspection", () => { + gateTest("release-b", "rejects missing native checksum evidence in release package inspection", () => { const repo = tempRepo({ includeDist: true }); const packageName = graphCoreNativePackageNameForTarget(`${process.platform}-${process.arch}`); const checksumPath = join( @@ -236,7 +243,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /packed files mismatch|checksum|sha256/); }); - it("rejects current-tool markers in cutover descriptor inspection", () => { + gateTest("cutover", "rejects current-tool markers in cutover descriptor inspection", () => { const repo = tempRepo({ includeDist: true }); const descriptorPath = join(repo, "packages/opcore/dist/descriptors/opcore.managed-tool.json"); const descriptor = JSON.parse(readFileSync(descriptorPath, "utf8")); @@ -247,7 +254,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /private runtime|forbidden marker|\.ace/); }); - it("rejects cutover receipts with advertised not_implemented commands", () => { + gateTest("cutover", "rejects cutover receipts with advertised not_implemented commands", () => { const repo = tempRepo({ includeDist: true }); const receiptPath = join(repo, "bad-cutover-receipt.json"); writeFileSync(receiptPath, `${JSON.stringify(minimalCutoverReceipt(repo, { status: "not_implemented", exitCode: 2 }), null, 2)}\n`); @@ -258,7 +265,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /not_implemented/); }); - it("rejects cutover receipts missing required command evidence", () => { + gateTest("cutover", "rejects cutover receipts missing required command evidence", () => { const repo = tempRepo({ includeDist: true }); const receiptPath = join(repo, "bad-cutover-missing-command.json"); const receipt = minimalCutoverReceipt(repo); @@ -271,7 +278,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /command receipts.*inspect-search/); }); - it("rejects old bin fallback in installed cutover projects", () => { + gateTest("cutover", "rejects old bin fallback in installed cutover projects", () => { const repo = tempRepo({ includeDist: true }); const project = join(repo, "tmp-installed-project"); mkdirSync(join(project, "node_modules/.bin"), { recursive: true }); @@ -286,7 +293,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /old public bin.*lattice/); }); - it("rejects sibling file dependencies", () => { + gateTest("plain", "rejects sibling file dependencies", () => { const repo = tempRepo(); const manifestPath = join(repo, "packages/graph/package.json"); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); @@ -297,7 +304,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /file dependencies must stay inside packages|must not reference sibling repo/); }); - it("rejects parent-directory TypeScript outputs", () => { + gateTest("plain", "rejects parent-directory TypeScript outputs", () => { const repo = tempRepo(); const tsconfigPath = join(repo, "packages/validation/tsconfig.json"); const tsconfig = JSON.parse(readFileSync(tsconfigPath, "utf8")); @@ -308,7 +315,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /must not reference parent directories or absolute paths/); }); - it("rejects reserved graph implementation package paths", () => { + gateTest("plain", "rejects reserved graph implementation package paths", () => { const repo = tempRepo(); const readmePath = join(repo, "reserved-graph-name.md"); writeFileSync(readmePath, `${["packages", "crg"].join("/")} is reserved for removed implementation references\n`); @@ -317,7 +324,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /reserved graph naming references/); }); - it("rejects reserved graph implementation package names", () => { + gateTest("plain", "rejects reserved graph implementation package names", () => { const repo = tempRepo(); const manifestPath = join(repo, "packages/graph/package.json"); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); @@ -328,7 +335,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /reserved graph naming references/); }); - it("rejects reserved graph provider literals", () => { + gateTest("plain", "rejects reserved graph provider literals", () => { const repo = tempRepo(); const sourcePath = join(repo, "packages/graph/src/reserved-provider.ts"); writeFileSync(sourcePath, `export const status = { provider: ${JSON.stringify("crg")} };\n`); @@ -337,7 +344,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /reserved graph naming references/); }); - it("rejects reserved graph providerName metadata", () => { + gateTest("plain", "rejects reserved graph providerName metadata", () => { const repo = tempRepo(); const sourcePath = join(repo, "packages/graph/src/reserved-provider-name-metadata.ts"); writeFileSync(sourcePath, `export const status = { providerName: ${JSON.stringify("crg")} };\n`); @@ -347,7 +354,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /legacy graph provider name metadata/); }); - it("rejects reserved graph provider name constants", () => { + gateTest("plain", "rejects reserved graph provider name constants", () => { const repo = tempRepo(); const sourcePath = join(repo, "packages/graph/src/bad-provider-name.ts"); const legacyGraphTool = "cr" + "g"; @@ -358,7 +365,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /legacy graph provider name constant/); }); - it("rejects stale CONTRIBUTING graph naming", () => { + gateTest("plain", "rejects stale CONTRIBUTING graph naming", () => { const repo = tempRepo(); const contributingPath = join(repo, "CONTRIBUTING.md"); const legacyGraphTool = "cr" + "g"; @@ -378,7 +385,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /reserved graph naming references/); }); - it("rejects edit importing graph-core native artifact loaders", () => { + gateTest("plain", "rejects edit importing graph-core native artifact loaders", () => { const repo = tempRepo(); const sourcePath = join(repo, "packages/edit/src/bad-graph-loader.ts"); writeFileSync(sourcePath, `import { resolveGraphCoreArtifact } from "@the-open-engine/opcore-graph";\nvoid resolveGraphCoreArtifact;\n`); @@ -387,7 +394,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /bad-graph-loader\.ts/); }); - it("rejects validation relying on graph sqlite internals", () => { + gateTest("plain", "rejects validation relying on graph sqlite internals", () => { const repo = tempRepo(); const sourcePath = join(repo, "packages/validation/src/bad-graph-sqlite.ts"); writeFileSync(sourcePath, `export const graphSqliteInternal = "graph sqlite internal reader";\n`); @@ -396,7 +403,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /bad-graph-sqlite\.ts/); }); - it("rejects Cargo package names containing crg", () => { + gateTest("plain", "rejects Cargo package names containing crg", () => { const repo = tempRepo(); const manifestPath = join(repo, "crates/graph-core/Cargo.toml"); const manifest = readFileSync(manifestPath, "utf8").replace( @@ -409,7 +416,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /must not use crg in Rust package/); }); - it("rejects Rox code-quality coverage without all Rust crate paths", () => { + gateTest("plain", "rejects Rox code-quality coverage without all Rust crate paths", () => { const repo = tempRepo(); const roxPath = join(repo, "rox.json"); const rox = JSON.parse(readFileSync(roxPath, "utf8")); @@ -420,7 +427,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /checks\.codeQuality\.include must include "crates\/"/); }); - it("rejects Rox code-quality coverage without existing TypeScript and script scopes", () => { + gateTest("plain", "rejects Rox code-quality coverage without existing TypeScript and script scopes", () => { const repo = tempRepo(); const roxPath = join(repo, "rox.json"); const rox = JSON.parse(readFileSync(roxPath, "utf8")); @@ -431,7 +438,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /checks\.codeQuality\.include must include "packages\/"/); }); - it("rejects scoped Rust quality scripts that run against the whole repo", () => { + gateTest("plain", "rejects scoped Rust quality scripts that run against the whole repo", () => { const repo = tempRepo(); const manifestPath = join(repo, "package.json"); const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); @@ -442,7 +449,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /must run scoped Rust graph function metrics script/); }); - it("rejects repo-wide Rox without scoped Rust graph metrics", () => { + gateTest("plain", "rejects repo-wide Rox without scoped Rust graph metrics", () => { const repo = tempRepo(); const roxPath = join(repo, "rox.json"); const rox = JSON.parse(readFileSync(roxPath, "utf8")); @@ -453,7 +460,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /must run scoped Rust graph function metrics/); }); - it("rejects Rox all-mode Rust metrics without crate package scope", () => { + gateTest("plain", "rejects Rox all-mode Rust metrics without crate package scope", () => { const repo = tempRepo(); const roxPath = join(repo, "rox.json"); const rox = JSON.parse(readFileSync(roxPath, "utf8")); @@ -464,7 +471,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /packages must include "crates"/); }); - it("rejects Rox code-quality coverage without changed-file modes", () => { + gateTest("plain", "rejects Rox code-quality coverage without changed-file modes", () => { const repo = tempRepo(); const roxPath = join(repo, "rox.json"); const rox = JSON.parse(readFileSync(roxPath, "utf8")); @@ -475,7 +482,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /checks\.codeQuality\.when\.modes must include "changed"/); }); - it("rejects graph-core crates without workspace lint opt-in", () => { + gateTest("plain", "rejects graph-core crates without workspace lint opt-in", () => { const repo = tempRepo(); const manifestPath = join(repo, "crates/graph-core/Cargo.toml"); const manifest = readFileSync(manifestPath, "utf8").replace(/\n\[lints]\nworkspace = true\n?/, "\n"); @@ -485,7 +492,7 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /crates\/graph-core\/Cargo\.toml must include \[lints]/); }); - it("rejects Cargo workspace manifests without clippy lint policy", () => { + gateTest("plain", "rejects Cargo workspace manifests without clippy lint policy", () => { const repo = tempRepo(); const manifestPath = join(repo, "Cargo.toml"); const manifest = readFileSync(manifestPath, "utf8").replace(/\n\[workspace\.lints\.clippy][\s\S]*$/, "\n"); @@ -495,23 +502,37 @@ describe("negative gate fixtures", () => { assert.match(stderrAndStdout(result), /Cargo\.toml must include \[workspace\.lints\.clippy]/); }); }); +} function tempRepo(options = {}) { - const tempRoot = mkdtempSync(join(tmpdir(), "lattice-gate-")); - const repo = join(tempRoot, "repo"); - withReleaseDocsLock(() => { - cpSync(repoRoot, repo, { - recursive: true, - filter(source) { - const rel = relative(repoRoot, source); - if (rel === "") return true; - return !rel.split(/[\\/]/).some((segment) => copiedRepoSkips.has(segment) || (segment === "dist" && !options.includeDist)); - } - }); + const repo = tempRepoTemplate(options); + resetTempRepo(repo); + return repo; +} + +function tempRepoTemplate(options = {}) { + const includeDist = options.includeDist === true; + const cacheKey = includeDist ? "include-dist" : "default"; + const cached = tempRepoTemplateCache.get(cacheKey); + if (cached && existsSync(cached)) return cached; + + const templateRoot = mkdtempSync(join(tmpdir(), `lattice-gate-template-${cacheKey}-`)); + const repo = join(templateRoot, "repo"); + cpSync(repoRoot, repo, { + mode: constants.COPYFILE_FICLONE, + recursive: true, + filter(source) { + const rel = relative(repoRoot, source); + if (rel === "") return true; + if (rel.endsWith(".tsbuildinfo")) return false; + return !rel.split(/[\\/]/).some((segment) => copiedRepoSkips.has(segment) || (segment === "dist" && !includeDist)); + } }); - if (options.includeDist) copyBundledExternalRuntimePackages(repo); + if (includeDist) copyBundledExternalRuntimePackages(repo); run(repo, "git", ["init", "--quiet"]); stageRepoEntries(repo); + run(repo, "git", ["-c", "user.name=fixture", "-c", "user.email=fixture@example.invalid", "commit", "--quiet", "-m", "fixture baseline"]); + tempRepoTemplateCache.set(cacheKey, repo); return repo; } @@ -520,38 +541,17 @@ function copyBundledExternalRuntimePackages(repo) { const source = externalRuntimePackageDir(packageName); const destination = join(repo, "node_modules", ...packageName.split("/")); mkdirSync(dirname(destination), { recursive: true }); - cpSync(source, destination, { recursive: true }); + cpSync(source, destination, { mode: constants.COPYFILE_FICLONE, recursive: true }); } } function stageRepoEntries(repo) { - const files = run(repo, "git", ["ls-files", "--others", "--exclude-standard", "-z"]).stdout.split("\0").filter(Boolean); - for (let index = 0; index < files.length; index += 100) { - run(repo, "git", ["add", "--", ...files.slice(index, index + 100)]); - } -} - -function withReleaseDocsLock(runLocked) { - const lockPath = join(repoRoot, "docs/release/.receipt-test.lock"); - const deadline = Date.now() + releaseDocsLockTimeoutMs; - while (Date.now() < deadline) { - try { - mkdirSync(lockPath); - try { - return runLocked(); - } finally { - rmSync(lockPath, { recursive: true, force: true }); - } - } catch (error) { - if (error?.code !== "EEXIST") throw error; - sleep(50); - } - } - throw new Error(`timed out waiting for ${lockPath}`); + run(repo, "git", ["add", "-A", "-f", "."]); } -function sleep(ms) { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +function resetTempRepo(repo) { + run(repo, "git", ["reset", "--hard", "--quiet", "HEAD"]); + run(repo, "git", ["clean", "-fdx", "--quiet"]); } function run(cwd, command, args, options = {}) { diff --git a/tests/gate-negative-plain.test.mjs b/tests/gate-negative-plain.test.mjs new file mode 100644 index 0000000..1496406 --- /dev/null +++ b/tests/gate-negative-plain.test.mjs @@ -0,0 +1,2 @@ +process.env.OPCORE_GATE_NEGATIVE_GROUP = "plain"; +await import("./gate-negative-fixtures.test.mjs"); diff --git a/tests/gate-negative-release-a.test.mjs b/tests/gate-negative-release-a.test.mjs new file mode 100644 index 0000000..4b120a1 --- /dev/null +++ b/tests/gate-negative-release-a.test.mjs @@ -0,0 +1,2 @@ +process.env.OPCORE_GATE_NEGATIVE_GROUP = "release-a"; +await import("./gate-negative-fixtures.test.mjs"); diff --git a/tests/gate-negative-release-b.test.mjs b/tests/gate-negative-release-b.test.mjs new file mode 100644 index 0000000..926c32d --- /dev/null +++ b/tests/gate-negative-release-b.test.mjs @@ -0,0 +1,2 @@ +process.env.OPCORE_GATE_NEGATIVE_GROUP = "release-b"; +await import("./gate-negative-fixtures.test.mjs"); diff --git a/tests/installed-bins.test.mjs b/tests/installed-bins.test.mjs index f9e1429..6aea69d 100644 --- a/tests/installed-bins.test.mjs +++ b/tests/installed-bins.test.mjs @@ -165,6 +165,7 @@ describe("installed package bins", () => { "python.import-graph", "python.dead-code", "python.relevant-tests", + "python.pytest", "docs.existence", "docs.staleness", "docs.freshness", diff --git a/tests/opcore-metrics.test.mjs b/tests/opcore-metrics.test.mjs index 90e626d..678ab14 100644 --- a/tests/opcore-metrics.test.mjs +++ b/tests/opcore-metrics.test.mjs @@ -429,8 +429,8 @@ function validationResult() { category: "test", severity: "info", path: "pkg/untested.py", - code: "PY_RELEVANT_TESTS_ABSENT", - message: "No TESTED_BY graph evidence found." + code: "PY_RELEVANT_TEST_CANDIDATES_ABSENT", + message: "No TESTED_BY graph candidate evidence found." }, { category: "graph", @@ -483,7 +483,8 @@ function validationResult() { "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ], generatedAt: "2026-06-25T00:00:00.000Z" } diff --git a/tests/schema-contracts.test.mjs b/tests/schema-contracts.test.mjs index 5675a3c..c366209 100644 --- a/tests/schema-contracts.test.mjs +++ b/tests/schema-contracts.test.mjs @@ -834,6 +834,7 @@ describe("Opcore JSON schema wire constraints", () => { it("validates Python project-context wire identity and typed outcome vocabulary", () => { const context = pythonProjectContextWith(); assert.equal(isValidDefinition("PythonProjectContext", context), true); + assert.equal(isValidDefinition("PythonProjectContext", { ...context, target: "services/api/pyproject.toml" }), true); assert.equal(isValidDefinition("ValidationResult", validationResultWith({ pythonProjectContexts: [context] })), true); assert.equal(isValidDefinition("PythonProjectContext", { ...context, schemaId: "python.context.v0" }), false); assert.equal(isValidDefinition("PythonProjectContext", { ...context, outcome: "ready" }), false); diff --git a/tests/validation-cli.test.mjs b/tests/validation-cli.test.mjs index cbc4d34..10a8a35 100644 --- a/tests/validation-cli.test.mjs +++ b/tests/validation-cli.test.mjs @@ -41,7 +41,8 @@ const pythonCheckIds = [ "python.types", "python.import-graph", "python.dead-code", - "python.relevant-tests" + "python.relevant-tests", + "python.pytest" ]; const docsCheckIds = [ "docs.existence", @@ -57,7 +58,8 @@ const docsCheckIds = [ ]; const cloneCheckIds = ["clone.duplication"]; const typeScriptExecutableDefaultCheckIds = typeScriptCheckIds.filter((checkId) => checkId !== "typescript.lint"); -const executableDefaultCheckIds = [...typeScriptExecutableDefaultCheckIds, ...rustCheckIds, ...pythonCheckIds, ...cloneCheckIds]; +const pythonExecutableDefaultCheckIds = pythonCheckIds.filter((checkId) => checkId !== "python.pytest"); +const executableDefaultCheckIds = [...typeScriptExecutableDefaultCheckIds, ...rustCheckIds, ...pythonExecutableDefaultCheckIds, ...cloneCheckIds]; const defaultCheckIds = [...typeScriptCheckIds, ...rustCheckIds, ...pythonCheckIds, ...docsCheckIds, ...cloneCheckIds]; describe("validation CLI", () => { @@ -856,6 +858,96 @@ describe("validation CLI", () => { } }); + it("returns a typed disabled python.pytest capability run instead of an unknown check failure", () => { + const temp = mkdtempSync(join(tmpdir(), "opcore-validation-cli-python-pytest-disabled-")); + try { + mkdirSync(join(temp, "src"), { recursive: true }); + writeFileSync(join(temp, "src/app.py"), "VALUE = 1\n"); + writeRepoConfigObject(temp, { + schemaVersion: 1, + kind: "opcore_init_config", + validation: { + checks: { + disabled: ["python.pytest"] + } + } + }); + + const result = JSON.parse(runRaw(["check", "files", "--files", "src/app.py", "--repo", temp, "--checks", "python.pytest", "--json"], [0]).stdout); + + assert.equal(result.validationResult.status, "passed"); + assert.deepEqual(result.validationResult.manifest.checks, ["python.pytest"]); + assert.deepEqual(result.validationResult.pythonCapabilityRuns, [ + { + capability: "pytest", + checkId: "python.pytest", + activation: "disabled", + outcome: "disabled", + message: "python.pytest is disabled by repo policy.", + selectionMode: "none" + } + ]); + } finally { + rmSync(temp, { recursive: true, force: true }); + } + }); + + it("emits an unrequested not_applicable python.pytest capability run for Python-targeted explicit checks", async () => { + const temp = mkdtempSync(join(tmpdir(), "opcore-validation-cli-python-pytest-unrequested-")); + try { + mkdirSync(join(temp, "src"), { recursive: true }); + writeFileSync(join(temp, "src/app.py"), "VALUE = 1\n"); + + const result = await routeCommand( + ["check", "files", "--files", "src/app.py", "--repo", temp, "--checks", "python.syntax", "--json"], + "opcore" + ); + + assert.equal(result.status, "ok"); + assert.equal(result.validationResult.status, "passed"); + assert.deepEqual(result.validationResult.manifest.checks, ["python.syntax"]); + assert.deepEqual(result.validationResult.pythonCapabilityRuns, [ + { + capability: "pytest", + checkId: "python.pytest", + activation: "not_applicable", + outcome: "not_applicable", + message: "Python pytest execution is opt-in." + } + ]); + assert.deepEqual( + result.validationResult.manifest.runs.map((run) => run.checkId), + ["python.syntax"] + ); + } finally { + rmSync(temp, { recursive: true, force: true }); + } + }); + + it("does not emit unrequested python.pytest capability evidence for unrelated explicit checks", async () => { + const temp = mkdtempSync(join(tmpdir(), "opcore-validation-cli-unrelated-checks-")); + try { + mkdirSync(join(temp, "src"), { recursive: true }); + writeFileSync(join(temp, "src/app.ts"), "export const value = 1;\n"); + + const result = await routeCommand( + ["check", "files", "--files", "src/app.ts", "--repo", temp, "--checks", "typescript.syntax", "--json"], + "opcore" + ); + + assert.equal(result.status, "ok"); + assert.equal(result.validationResult.status, "passed"); + assert.deepEqual(result.validationResult.manifest.checks, ["typescript.syntax"]); + assert.deepEqual(result.validationResult.pythonCapabilityRuns ?? [], []); + assert.deepEqual( + result.validationResult.manifest.runs.map((run) => run.checkId), + ["typescript.syntax"] + ); + } finally { + rmSync(temp, { recursive: true, force: true }); + } + }); + it("uses native docs enabled flags for default docs checks", () => { const temp = mkdtempSync(join(tmpdir(), "opcore-native-default-docs-checks-")); try { diff --git a/tests/validation-python.test.mjs b/tests/validation-python.test.mjs index 897254c..22d648e 100644 --- a/tests/validation-python.test.mjs +++ b/tests/validation-python.test.mjs @@ -8,6 +8,7 @@ import { createValidationCheckRegistry, createValidationRunner } from "../packag import { PYTHON_DEAD_CODE_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, + PYTHON_PYTEST_CHECK_ID, PYTHON_RELEVANT_TESTS_CHECK_ID, PYTHON_SOURCE_HYGIENE_CHECK_ID, PYTHON_SYNTAX_CHECK_ID, @@ -55,12 +56,14 @@ describe("validation-python adapter", () => { PYTHON_TYPES_CHECK_ID, PYTHON_IMPORT_GRAPH_CHECK_ID, PYTHON_DEAD_CODE_CHECK_ID, - PYTHON_RELEVANT_TESTS_CHECK_ID + PYTHON_RELEVANT_TESTS_CHECK_ID, + PYTHON_PYTEST_CHECK_ID ] ); assert.equal(registry.byId.get(PYTHON_SYNTAX_CHECK_ID)?.requiresGraph, false); assert.equal(registry.byId.get(PYTHON_SOURCE_HYGIENE_CHECK_ID)?.requiresGraph, false); assert.equal(registry.byId.get(PYTHON_IMPORT_GRAPH_CHECK_ID)?.requiresGraph, true); + assert.deepEqual(registry.byId.get(PYTHON_PYTEST_CHECK_ID)?.defaultScopes, []); }); it("fails syntax and type checks closed when no canonical context resolver is injected", async () => { @@ -876,7 +879,7 @@ describe("validation-python adapter", () => { assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code).sort(), [ "PY_DEAD_CODE_UNSUPPORTED", - "PY_RELEVANT_TESTS_ABSENT" + "PY_RELEVANT_TEST_CANDIDATES_ABSENT" ]); }); @@ -954,7 +957,7 @@ describe("validation-python adapter", () => { })); assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); - assert.deepEqual(result.pythonProjectContexts, []); + assert.deepEqual(result.pythonProjectContexts ?? [], []); }); it("shares one canonical source resolver across graph-dependent Python checks", async () => { @@ -1114,7 +1117,177 @@ describe("validation-python adapter", () => { ); assert.equal(result.status, "passed"); - assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code), ["PY_RELEVANT_TESTS_FOUND"]); + assert.deepEqual(result.diagnostics.map((diagnostic) => diagnostic.code), ["PY_RELEVANT_TEST_CANDIDATES_FOUND"]); + }); + + it("materializes pytest support files and preserves exact regular-file modes", async () => { + const repoRoot = mkdtempSync(join(tmpdir(), "opcore-validation-python-pytest-support-")); + try { + writePassingPythonProtocolShim(repoRoot); + writePytestShim(repoRoot, { + collectionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_support_files\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ], + executionShellLines: [ + "node -e \"const fs=require('fs'); const payload=JSON.parse(fs.readFileSync('tests/data/payload.json','utf8')); if(payload.value!==7) { console.error('support file missing'); process.exit(9); } if((fs.statSync('tests/private.txt').mode & 0o777)!==0o640) { console.error('private mode mismatch'); process.exit(10); } if((fs.statSync('tests/run-me.sh').mode & 0o777)!==0o750) { console.error('exec mode mismatch'); process.exit(11); }\"", + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_support_files\"}'", + "emit '{\"type\":\"test_report\",\"nodeid\":\"tests/test_app.py::test_support_files\",\"when\":\"call\",\"outcome\":\"passed\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ] + }); + const files = { + "pyproject.toml": "[project]\nname='fixture'\n", + "src/app.py": "VALUE = 7\n", + "tests/test_app.py": "def test_support_files():\n assert True\n", + "tests/data/payload.json": "{\"value\":7}\n", + "tests/private.txt": "secret\n", + "tests/run-me.sh": "#!/bin/sh\nexit 0\n" + }; + writeRepoFiles(repoRoot, files); + chmodSync(join(repoRoot, "tests/private.txt"), 0o640); + chmodSync(join(repoRoot, "tests/run-me.sh"), 0o750); + const result = await runner({ + files, + checks: createPythonValidationChecks({ + repoRoot, + env: { PATH: "" }, + nodeWorkspace: createNodePythonProjectWorkspace(repoRoot), + processProbe: successfulProbe(), + toolArgv: { pytest: [join(repoRoot, ".venv", "bin", "pytest")] } + }), + graphProviderClient: graphClient({ + factQuery: (query) => + availableFactResult( + query, + [], + query.selector.kind === "edges" && query.selector.edgeKinds?.includes("TESTED_BY") + ? [{ kind: "TESTED_BY", from: "file:src/app.py", to: "file:tests/test_app.py" }] + : [] + ) + }) + }).runValidation(request({ + repo: { repoRoot }, + checks: [PYTHON_PYTEST_CHECK_ID], + scope: { kind: "files", files: ["src/app.py"] } + })); + + assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); + assert.equal(result.pythonCapabilityRuns[0]?.outcome, "passed"); + assert.equal(result.pythonCapabilityRuns[0]?.counts?.passedCount, 1); + } finally { + rmSync(repoRoot, { recursive: true, force: true }); + } + }); + + it("executes python.pytest for config-only targets inside the canonical project", async () => { + const repoRoot = mkdtempSync(join(tmpdir(), "opcore-validation-python-pytest-config-only-")); + try { + writePassingPythonProtocolShim(repoRoot); + writePytestShim(repoRoot, { + collectionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_config_only\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ], + executionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_config_only\"}'", + "emit '{\"type\":\"test_report\",\"nodeid\":\"tests/test_app.py::test_config_only\",\"when\":\"call\",\"outcome\":\"passed\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ] + }); + const files = { + "pyproject.toml": "[project]\nname='fixture'\n", + "src/app.py": "VALUE = 7\n", + "tests/test_app.py": "def test_config_only():\n assert True\n" + }; + writeRepoFiles(repoRoot, files); + const result = await runner({ + files, + checks: createPythonValidationChecks({ + repoRoot, + env: { PATH: "" }, + nodeWorkspace: createNodePythonProjectWorkspace(repoRoot), + processProbe: successfulProbe(), + toolArgv: { pytest: [join(repoRoot, ".venv", "bin", "pytest")] } + }), + graphProviderClient: graphClient({ + factQuery: (query) => + availableFactResult( + query, + [], + query.selector.kind === "edges" && query.selector.edgeKinds?.includes("TESTED_BY") + ? [{ kind: "TESTED_BY", from: "file:src/app.py", to: "file:tests/test_app.py" }] + : [] + ) + }) + }).runValidation(request({ + repo: { repoRoot }, + checks: [PYTHON_PYTEST_CHECK_ID], + scope: { kind: "files", files: ["pyproject.toml"] } + })); + + assert.equal(result.status, "passed", JSON.stringify(result, null, 2)); + assert.equal(result.pythonCapabilityRuns[0]?.outcome, "passed"); + assert.deepEqual(result.pythonCapabilityRuns[0]?.candidatePaths, ["tests/test_app.py"]); + assert.equal(result.pythonCapabilityRuns[0]?.counts?.passedCount, 1); + assert.equal(result.pythonProjectContexts[0]?.projectRoot, "."); + } finally { + rmSync(repoRoot, { recursive: true, force: true }); + } + }); + + it("fails python.pytest when execution hook events do not match the collected node ids", async () => { + const repoRoot = mkdtempSync(join(tmpdir(), "opcore-validation-python-pytest-mismatch-")); + try { + writePassingPythonProtocolShim(repoRoot); + writePytestShim(repoRoot, { + collectionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_expected\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ], + executionShellLines: [ + "emit '{\"type\":\"collected\",\"nodeid\":\"tests/test_app.py::test_expected\"}'", + "emit '{\"type\":\"test_report\",\"nodeid\":\"tests/test_app.py::test_other\",\"when\":\"call\",\"outcome\":\"passed\"}'", + "emit '{\"type\":\"session_finish\",\"exitstatus\":0}'" + ] + }); + const files = { + "pyproject.toml": "[project]\nname='fixture'\n", + "src/app.py": "VALUE = 1\n", + "tests/test_app.py": "def test_expected():\n assert True\n" + }; + writeRepoFiles(repoRoot, files); + const result = await runner({ + files, + checks: createPythonValidationChecks({ + repoRoot, + env: { PATH: "" }, + nodeWorkspace: createNodePythonProjectWorkspace(repoRoot), + processProbe: successfulProbe(), + toolArgv: { pytest: [join(repoRoot, ".venv", "bin", "pytest")] } + }), + graphProviderClient: graphClient({ + factQuery: (query) => + availableFactResult( + query, + [], + query.selector.kind === "edges" && query.selector.edgeKinds?.includes("TESTED_BY") + ? [{ kind: "TESTED_BY", from: "file:src/app.py", to: "file:tests/test_app.py" }] + : [] + ) + }) + }).runValidation(request({ + repo: { repoRoot }, + checks: [PYTHON_PYTEST_CHECK_ID], + scope: { kind: "files", files: ["src/app.py"] } + })); + + assert.equal(result.status, "policy_failure", JSON.stringify(result, null, 2)); + assert.equal(result.pythonCapabilityRuns[0]?.outcome, "tool_failure"); + assert.equal(result.diagnostics.every((diagnostic) => diagnostic.code === "PYTHON_PYTEST_PROTOCOL_MISMATCH"), true); + } finally { + rmSync(repoRoot, { recursive: true, force: true }); + } }); it("passes clean Python validation fixture checks", async () => { @@ -2472,6 +2645,68 @@ function writeToolShim(repoRoot, name, content) { chmodSync(shimPath, 0o755); } +function writePytestShim(repoRoot, options) { + writeToolShim( + repoRoot, + "pytest", + [ + "#!/bin/sh", + "set -eu", + "output=\"$OPCORE_PYTEST_HOOK_OUTPUT\"", + "collect=0", + "for arg in \"$@\"; do", + " if [ \"$arg\" = \"--version\" ]; then echo 'pytest 8.4.1'; exit 0; fi", + " if [ \"$arg\" = \"--collect-only\" ]; then collect=1; fi", + "done", + "emit() { printf '%s\\n' \"$1\" >> \"$output\"; }", + "if [ \"$collect\" = \"1\" ]; then", + ...options.collectionShellLines.map((line) => ` ${line}`), + ` exit ${options.collectionExitCode ?? 0}`, + "fi", + ...options.executionShellLines.map((line) => line), + `exit ${options.executionExitCode ?? 0}`, + "" + ].join("\n") + ); +} + +function writeRepoFiles(repoRoot, files) { + for (const [path, content] of Object.entries(files)) { + mkdirSync(dirname(join(repoRoot, path)), { recursive: true }); + writeFileSync(join(repoRoot, path), content); + } +} + +function pyrightShim(version, diagnostics, exitCode = diagnostics.some((entry) => entry.severity === "error") ? 1 : 0) { + const payload = pyrightPayload(version, diagnostics); + return rawPyrightShim(version, [ + "cat <<'OPCORE_PYRIGHT_JSON'", + JSON.stringify(payload), + "OPCORE_PYRIGHT_JSON", + `exit ${exitCode}` + ].join("\n")); +} + +function pyrightPayload(version, diagnostics) { + const summary = { + filesAnalyzed: 1, + errorCount: diagnostics.filter((entry) => entry.severity === "error").length, + warningCount: diagnostics.filter((entry) => entry.severity === "warning").length, + informationCount: diagnostics.filter((entry) => entry.severity === "information").length, + timeInSec: 0.01 + }; + return { version, time: "0", generalDiagnostics: diagnostics, summary }; +} + +function rawPyrightShim(version, body) { + return [ + "#!/bin/sh", + `for arg in \"$@\"; do if [ \"$arg\" = \"--version\" ]; then echo 'pyright ${version}'; exit 0; fi; done`, + body, + "" + ].join("\n"); +} + function writePythonProtocolShim(repoRoot, compilerBranch) { const bin = join(repoRoot, ".venv", "bin"); mkdirSync(bin, { recursive: true });